feat(datasource): add RSS feed connector with full-text ingestion

Enable private RSS feeds as a knowledge base data source with custom
headers, article body fetching, and Markdown output for incremental sync.
This commit is contained in:
wizardchen
2026-06-23 14:58:45 +08:00
committed by lyingbug
parent 94ce8fedb1
commit e40ad802d7
21 changed files with 1328 additions and 131 deletions
+6
View File
@@ -3,6 +3,7 @@ module github.com/Tencent/WeKnora
go 1.26.0
require (
codeberg.org/readeck/go-readability/v2 v2.1.2
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.1
github.com/PuerkitoBio/goquery v1.12.0
@@ -35,6 +36,7 @@ require (
github.com/mark3labs/mcp-go v0.52.0
github.com/milvus-io/milvus/client/v2 v2.6.4
github.com/minio/minio-go/v7 v7.1.0
github.com/mmcdole/gofeed v1.3.0
github.com/neo4j/neo4j-go-driver/v6 v6.0.0
github.com/ollama/ollama v0.23.2
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
@@ -176,6 +178,7 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
@@ -183,6 +186,7 @@ require (
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/flatbuffers v25.12.19+incompatible // indirect
@@ -194,6 +198,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/itlightning/dateparse v0.2.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -227,6 +232,7 @@ require (
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect