feat(datasource): add RSS feed connector with full-text ingestion
Enable private RSS feeds as a knowledge base data source with custom headers, article body fetching, and Markdown output for incremental sync.
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
readability "codeberg.org/readeck/go-readability/v2"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
// requestTimeout bounds a single feed or article fetch.
|
||||
requestTimeout = 20 * time.Second
|
||||
|
||||
// maxFeedSize caps the feed body to avoid memory blowups on hostile feeds.
|
||||
maxFeedSize = 10 * 1024 * 1024 // 10 MB
|
||||
|
||||
// maxArticleSize caps a single article page body.
|
||||
maxArticleSize = 5 * 1024 * 1024 // 5 MB
|
||||
|
||||
// defaultUserAgent is sent on every request; some feeds reject empty UAs.
|
||||
defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
|
||||
)
|
||||
|
||||
// client performs SSRF-safe HTTP fetches with optional custom headers.
|
||||
type client struct {
|
||||
httpClient *http.Client
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
func newClient(headers map[string]string) *client {
|
||||
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
cfg.Timeout = requestTimeout
|
||||
return &client{
|
||||
httpClient: utils.NewSSRFSafeHTTPClient(cfg),
|
||||
headers: headers,
|
||||
}
|
||||
}
|
||||
|
||||
// fetch retrieves rawURL with SSRF validation and size limiting. The user's
|
||||
// custom headers are applied first, then a default User-Agent / Accept if the
|
||||
// user did not override them.
|
||||
func (c *client) fetch(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {
|
||||
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
|
||||
return nil, fmt.Errorf("URL rejected: %w", err)
|
||||
}
|
||||
if _, err := url.Parse(rawURL); err != nil {
|
||||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for k, v := range c.headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
if req.Header.Get("Accept") == "" {
|
||||
req.Header.Set("Accept",
|
||||
"application/rss+xml, application/atom+xml, application/xml, text/xml, application/json, text/html;q=0.9, */*;q=0.8")
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body failed: %w", err)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// fetchFeed retrieves the raw bytes of a feed document.
|
||||
func (c *client) fetchFeed(ctx context.Context, feedURL string) ([]byte, error) {
|
||||
return c.fetch(ctx, feedURL, maxFeedSize)
|
||||
}
|
||||
|
||||
// extractArticle fetches an article page and returns the readability-cleaned
|
||||
// main content as HTML, plus the extracted title (may be empty). Returns an
|
||||
// error if the page can't be fetched or no readable content is found, so the
|
||||
// caller can fall back to feed-provided content.
|
||||
func (c *client) extractArticle(ctx context.Context, articleURL string) (contentHTML, title string, err error) {
|
||||
body, err := c.fetch(ctx, articleURL, maxArticleSize)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
pageURL, _ := url.Parse(articleURL)
|
||||
article, err := readability.FromReader(bytes.NewReader(body), pageURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("readability parse: %w", err)
|
||||
}
|
||||
if article.Node == nil {
|
||||
return "", "", fmt.Errorf("no readable content extracted")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := article.RenderHTML(&buf); err != nil {
|
||||
return "", "", fmt.Errorf("render article html: %w", err)
|
||||
}
|
||||
return buf.String(), article.Title(), nil
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
// Compile-time proof that *Connector satisfies the datasource.Connector interface.
|
||||
var _ datasource.Connector = (*Connector)(nil)
|
||||
|
||||
// Connector implements datasource.Connector for RSS/Atom/JSON feeds.
|
||||
type Connector struct{}
|
||||
|
||||
// NewConnector creates a new RSS connector.
|
||||
func NewConnector() *Connector { return &Connector{} }
|
||||
|
||||
// Type returns the connector type identifier.
|
||||
func (c *Connector) Type() string { return types.ConnectorTypeRSS }
|
||||
|
||||
// Validate verifies that every configured feed URL is reachable and parses as
|
||||
// a valid feed.
|
||||
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
for _, feedURL := range cfg.feedURLList() {
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch feed %s: %w", feedURL, err)
|
||||
}
|
||||
if _, err := parser.Parse(bytes.NewReader(data)); err != nil {
|
||||
return fmt.Errorf("parse feed %s: %w", feedURL, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors has nothing to do: feeds are a flat list with no
|
||||
// nesting, so a selection has no ancestors to reveal.
|
||||
func (c *Connector) ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// ListResources returns one resource per configured feed URL. The feed is
|
||||
// fetched so the resource can carry its real title; a feed that fails to fetch
|
||||
// still appears (named by URL) with an error note, so the user can deselect it
|
||||
// instead of the whole listing failing.
|
||||
func (c *Connector) ListResources(
|
||||
ctx context.Context, config *types.DataSourceConfig, parentID string,
|
||||
) ([]types.Resource, error) {
|
||||
// Feeds are flat: a lazy-load request for a specific parent has nothing extra.
|
||||
if parentID != "" {
|
||||
return []types.Resource{}, nil
|
||||
}
|
||||
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
feedURLs := cfg.feedURLList()
|
||||
out := make([]types.Resource, 0, len(feedURLs))
|
||||
for _, feedURL := range feedURLs {
|
||||
res := types.Resource{
|
||||
ExternalID: feedURL,
|
||||
Type: "feed",
|
||||
Name: feedURL,
|
||||
URL: feedURL,
|
||||
}
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[RSS] list: fetch %s failed: %v", feedURL, err)
|
||||
res.Description = "fetch failed: " + err.Error()
|
||||
out = append(out, res)
|
||||
continue
|
||||
}
|
||||
feed, err := parser.Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[RSS] list: parse %s failed: %v", feedURL, err)
|
||||
res.Description = "parse failed: " + err.Error()
|
||||
out = append(out, res)
|
||||
continue
|
||||
}
|
||||
if title := strings.TrimSpace(feed.Title); title != "" {
|
||||
res.Name = title
|
||||
}
|
||||
res.Description = strings.TrimSpace(feed.Description)
|
||||
if feed.Link != "" {
|
||||
res.URL = feed.Link
|
||||
}
|
||||
if feed.UpdatedParsed != nil {
|
||||
res.ModifiedAt = *feed.UpdatedParsed
|
||||
}
|
||||
res.Metadata = map[string]interface{}{"item_count": len(feed.Items)}
|
||||
out = append(out, res)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FetchAll performs a full sync of the specified feeds (or all configured feeds
|
||||
// when resourceIDs is empty).
|
||||
func (c *Connector) FetchAll(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]types.FetchedItem, error) {
|
||||
items, _, err := c.walk(ctx, config, resourceIDs, nil, false)
|
||||
return items, err
|
||||
}
|
||||
|
||||
// FetchIncremental returns only items whose fingerprint changed since the prior
|
||||
// cursor. Deletions are intentionally not emitted (feeds drop old items as a
|
||||
// matter of course).
|
||||
func (c *Connector) FetchIncremental(
|
||||
ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor,
|
||||
) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
var prev *rssCursor
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
var p rssCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &p)
|
||||
prev = &p
|
||||
}
|
||||
|
||||
items, newCursor, err := c.walk(ctx, config, config.ResourceIDs, prev, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
cursorMap := make(map[string]interface{})
|
||||
b, _ := json.Marshal(newCursor)
|
||||
_ = json.Unmarshal(b, &cursorMap)
|
||||
|
||||
return items, &types.SyncCursor{
|
||||
LastSyncTime: newCursor.LastSyncTime,
|
||||
ConnectorCursor: cursorMap,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// walk is the shared implementation for FetchAll / FetchIncremental.
|
||||
//
|
||||
// When incremental is true, items whose fingerprint matches the prior cursor
|
||||
// are skipped (no article re-fetch). The returned cursor always reflects the
|
||||
// full current item set so a later sync can detect changes.
|
||||
func (c *Connector) walk(
|
||||
ctx context.Context,
|
||||
config *types.DataSourceConfig,
|
||||
resourceIDs []string,
|
||||
prev *rssCursor,
|
||||
incremental bool,
|
||||
) ([]types.FetchedItem, *rssCursor, error) {
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Default to all configured feeds when no explicit selection was made.
|
||||
feedURLs := resourceIDs
|
||||
if len(feedURLs) == 0 {
|
||||
feedURLs = cfg.feedURLList()
|
||||
}
|
||||
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
newCursor := &rssCursor{
|
||||
LastSyncTime: time.Now().UTC(),
|
||||
FeedItems: make(map[string]map[string]string),
|
||||
}
|
||||
var out []types.FetchedItem
|
||||
|
||||
for _, feedURL := range feedURLs {
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fetch feed %s: %w", feedURL, err)
|
||||
}
|
||||
feed, err := parser.Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse feed %s: %w", feedURL, err)
|
||||
}
|
||||
|
||||
newCursor.FeedItems[feedURL] = make(map[string]string)
|
||||
var prevItems map[string]string
|
||||
if incremental && prev != nil {
|
||||
prevItems = prev.FeedItems[feedURL]
|
||||
}
|
||||
|
||||
var kept, skipped int
|
||||
for _, item := range feed.Items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
|
||||
if itemID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
feedContent := firstNonEmpty(item.Content, item.Description)
|
||||
fp := itemFingerprint(item.UpdatedParsed, item.PublishedParsed, feedContent)
|
||||
newCursor.FeedItems[feedURL][itemID] = fp
|
||||
|
||||
if incremental && prevItems != nil && prevItems[itemID] == fp {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
|
||||
out = append(out, c.buildItem(ctx, cli, feed, item, feedURL, itemID, feedContent))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[RSS] feed %s: items=%d fetched=%d skipped=%d",
|
||||
feedURL, len(feed.Items), kept, skipped)
|
||||
}
|
||||
|
||||
if !incremental {
|
||||
return out, newCursor, nil
|
||||
}
|
||||
return out, newCursor, nil
|
||||
}
|
||||
|
||||
// buildItem assembles a FetchedItem for a single feed entry, resolving the
|
||||
// best available content (full-text article > feed content) and converting it
|
||||
// to Markdown.
|
||||
func (c *Connector) buildItem(
|
||||
ctx context.Context,
|
||||
cli *client,
|
||||
feed *gofeed.Feed,
|
||||
item *gofeed.Item,
|
||||
feedURL, itemID, feedContent string,
|
||||
) types.FetchedItem {
|
||||
title := firstNonEmpty(item.Title, "untitled")
|
||||
|
||||
// Prefer full article text; fall back to feed-provided content on failure.
|
||||
contentHTML := feedContent
|
||||
if strings.TrimSpace(item.Link) != "" {
|
||||
if articleHTML, articleTitle, err := cli.extractArticle(ctx, item.Link); err == nil {
|
||||
contentHTML = articleHTML
|
||||
if item.Title == "" && articleTitle != "" {
|
||||
title = articleTitle
|
||||
}
|
||||
} else {
|
||||
logger.Warnf(ctx, "[RSS] full-text fetch failed for %s (using feed content): %v", item.Link, err)
|
||||
}
|
||||
}
|
||||
|
||||
content := htmlToMarkdown(contentHTML)
|
||||
|
||||
updatedAt := time.Now().UTC()
|
||||
switch {
|
||||
case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
|
||||
updatedAt = *item.UpdatedParsed
|
||||
case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
|
||||
updatedAt = *item.PublishedParsed
|
||||
}
|
||||
|
||||
author := ""
|
||||
if item.Author != nil {
|
||||
author = item.Author.Name
|
||||
}
|
||||
|
||||
return types.FetchedItem{
|
||||
ExternalID: itemID,
|
||||
Title: title,
|
||||
Content: []byte(content),
|
||||
ContentType: "text/markdown",
|
||||
FileName: sanitizeFileName(title) + ".md",
|
||||
URL: item.Link,
|
||||
UpdatedAt: updatedAt,
|
||||
SourceResourceID: feedURL,
|
||||
Metadata: map[string]string{
|
||||
"channel": types.ChannelRSS,
|
||||
"feed_url": feedURL,
|
||||
"feed_title": feed.Title,
|
||||
"guid": item.GUID,
|
||||
"link": item.Link,
|
||||
"author": author,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// htmlToMarkdown converts HTML to Markdown, returning the trimmed original on
|
||||
// conversion failure so we never silently drop content.
|
||||
func htmlToMarkdown(html string) string {
|
||||
if strings.TrimSpace(html) == "" {
|
||||
return ""
|
||||
}
|
||||
md, err := htmltomd.ConvertString(html)
|
||||
if err != nil || strings.TrimSpace(md) == "" {
|
||||
return strings.TrimSpace(html)
|
||||
}
|
||||
return strings.TrimSpace(md)
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1)
|
||||
// are reachable. Production keeps the default strict SSRF policy.
|
||||
func TestMain(m *testing.M) {
|
||||
_ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,::1")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
const longArticleBody = `<p>This is the first paragraph of a reasonably long article that the ` +
|
||||
`readability extractor should detect as the main content of the page. It contains ` +
|
||||
`enough words to clear the minimum content threshold used by the algorithm.</p>` +
|
||||
`<p>The second paragraph continues the discussion with more sentences so that the ` +
|
||||
`scoring heuristics confidently select this block over navigation and footer noise.</p>` +
|
||||
`<p>A third paragraph adds further substance, ensuring the article is unmistakably ` +
|
||||
`the dominant readable region of the document under test.</p>`
|
||||
|
||||
// fakeFeed spins up an httptest server serving an RSS feed and article pages.
|
||||
type fakeFeed struct {
|
||||
server *httptest.Server
|
||||
feedTitle string
|
||||
itemContent string // optional <description>/content for items
|
||||
}
|
||||
|
||||
func newFakeFeed(t *testing.T) *fakeFeed {
|
||||
t.Helper()
|
||||
f := &fakeFeed{feedTitle: "Test Feed"}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/article/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, `<!DOCTYPE html><html><head><title>%s</title></head>`+
|
||||
`<body><nav>menu</nav><article><h1>Heading</h1>%s</article><footer>foot</footer></body></html>`,
|
||||
"Full "+r.URL.Path, longArticleBody)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/feed.xml", func(w http.ResponseWriter, r *http.Request) {
|
||||
if auth := r.Header.Get("X-Test-Auth"); f.itemContent == "needs-auth" && auth != "secret" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/rss+xml")
|
||||
base := "http://" + r.Host
|
||||
desc := "summary fallback"
|
||||
fmt.Fprintf(w, `<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>%s</title>
|
||||
<link>%s</link>
|
||||
<description>A test feed</description>
|
||||
<item>
|
||||
<title>Article One</title>
|
||||
<link>%s/article/a1</link>
|
||||
<guid>guid-1</guid>
|
||||
<pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate>
|
||||
<description>%s</description>
|
||||
</item>
|
||||
<item>
|
||||
<title>Article Two</title>
|
||||
<link>%s/article/a2</link>
|
||||
<guid>guid-2</guid>
|
||||
<pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
|
||||
<description>%s</description>
|
||||
</item>
|
||||
</channel></rss>`, f.feedTitle, base, base, desc, base, desc)
|
||||
})
|
||||
|
||||
f.server = httptest.NewServer(mux)
|
||||
t.Cleanup(f.server.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeFeed) feedURL() string { return f.server.URL + "/feed.xml" }
|
||||
|
||||
func makeConfig(feedURLs string, headers string) *types.DataSourceConfig {
|
||||
creds := map[string]interface{}{"feed_urls": feedURLs}
|
||||
if headers != "" {
|
||||
creds["auth_headers"] = headers
|
||||
}
|
||||
return &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Credentials: creds,
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Type(t *testing.T) {
|
||||
if NewConnector().Type() != types.ConnectorTypeRSS {
|
||||
t.Fatalf("Type() = %q, want %q", NewConnector().Type(), types.ConnectorTypeRSS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_RequiresFeedURLs(t *testing.T) {
|
||||
if _, err := parseConfig(makeConfig(" ", "")); err == nil {
|
||||
t.Fatal("expected error when feed_urls is blank")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_FeedURLList_SplitsAndDedupes(t *testing.T) {
|
||||
cfg := &Config{FeedURLs: "https://a.com/f, https://b.com/f\nhttps://a.com/f\n"}
|
||||
got := cfg.feedURLList()
|
||||
want := []string{"https://a.com/f", "https://b.com/f"}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Fatalf("feedURLList = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ParseHeaders(t *testing.T) {
|
||||
cfg := &Config{AuthHeaders: "Authorization: Bearer x\nX-Foo: bar \nbroken-line\n: noname"}
|
||||
got := cfg.parseHeaders()
|
||||
if got["Authorization"] != "Bearer x" {
|
||||
t.Errorf("Authorization = %q", got["Authorization"])
|
||||
}
|
||||
if got["X-Foo"] != "bar" {
|
||||
t.Errorf("X-Foo = %q", got["X-Foo"])
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("expected 2 headers, got %d: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Validate_Success(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Validate_PrivateFeedWithHeader(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
f.itemContent = "needs-auth"
|
||||
|
||||
// Without the header → 401.
|
||||
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err == nil {
|
||||
t.Fatal("expected error without auth header")
|
||||
}
|
||||
// With the header → success.
|
||||
if err := NewConnector().Validate(
|
||||
context.Background(), makeConfig(f.feedURL(), "X-Test-Auth: secret"),
|
||||
); err != nil {
|
||||
t.Fatalf("Validate with header error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_ListResources(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
res, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources error: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 resource, got %d", len(res))
|
||||
}
|
||||
if res[0].ExternalID != f.feedURL() {
|
||||
t.Errorf("ExternalID = %q, want %q", res[0].ExternalID, f.feedURL())
|
||||
}
|
||||
if res[0].Name != "Test Feed" {
|
||||
t.Errorf("Name = %q, want %q", res[0].Name, "Test Feed")
|
||||
}
|
||||
|
||||
// Non-empty parentID → empty (feeds are flat).
|
||||
children, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "feed-x")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources(parent) error: %v", err)
|
||||
}
|
||||
if len(children) != 0 {
|
||||
t.Fatalf("expected no children, got %d", len(children))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_FetchAll_FullTextMarkdown(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
items, err := NewConnector().FetchAll(context.Background(), makeConfig(f.feedURL(), ""), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll error: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected 2 items, got %d", len(items))
|
||||
}
|
||||
|
||||
it := items[0]
|
||||
if it.ExternalID != "guid-1" {
|
||||
t.Errorf("ExternalID = %q, want guid-1", it.ExternalID)
|
||||
}
|
||||
if it.ContentType != "text/markdown" {
|
||||
t.Errorf("ContentType = %q, want text/markdown", it.ContentType)
|
||||
}
|
||||
if it.Metadata["channel"] != types.ChannelRSS {
|
||||
t.Errorf("channel = %q, want %q", it.Metadata["channel"], types.ChannelRSS)
|
||||
}
|
||||
// Full text from the article page should be present (not the short summary).
|
||||
if !strings.Contains(string(it.Content), "first paragraph") {
|
||||
t.Errorf("expected full article text in content, got: %q", string(it.Content))
|
||||
}
|
||||
if !strings.HasSuffix(it.FileName, ".md") {
|
||||
t.Errorf("FileName = %q, want .md suffix", it.FileName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_FetchIncremental_SkipsUnchanged(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
cfg := makeConfig(f.feedURL(), "")
|
||||
cfg.ResourceIDs = []string{f.feedURL()}
|
||||
|
||||
// First sync: everything is new.
|
||||
items, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first FetchIncremental error: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected 2 items on first sync, got %d", len(items))
|
||||
}
|
||||
if cursor == nil {
|
||||
t.Fatal("expected non-nil cursor")
|
||||
}
|
||||
|
||||
// Second sync with the returned cursor: nothing changed → no items.
|
||||
items2, _, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("second FetchIncremental error: %v", err)
|
||||
}
|
||||
if len(items2) != 0 {
|
||||
t.Fatalf("expected 0 items on unchanged second sync, got %d", len(items2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
|
||||
got, err := NewConnector().ResolveResourceAncestors(
|
||||
context.Background(), makeConfig("https://a.com/f", ""), []string{"x"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveResourceAncestors error: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("expected empty ancestors, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package rss implements the RSS/Atom data source connector for WeKnora.
|
||||
//
|
||||
// It syncs articles from one or more RSS/Atom/JSON feeds into a WeKnora
|
||||
// knowledge base. Each configured feed URL is treated as a selectable
|
||||
// resource; each feed item becomes a knowledge entry whose body is the
|
||||
// article's full text rendered as Markdown.
|
||||
//
|
||||
// Capabilities:
|
||||
// - Private feeds: optional custom HTTP headers (e.g. "Authorization: Bearer …")
|
||||
// are attached to every request, both to the feed and to article pages.
|
||||
// - Full text: when an item exposes a link, the article page is fetched and
|
||||
// run through a readability extractor; the cleaned HTML is converted to
|
||||
// Markdown. Feed-provided content (content:encoded / description) is used
|
||||
// as a fallback.
|
||||
// - Incremental: a per-feed fingerprint (item updated/published time, or a
|
||||
// content hash when no timestamp is present) detects changed items so only
|
||||
// new/updated articles are re-fetched. Deletions are NOT synced — feeds
|
||||
// routinely drop old items, which would otherwise look like deletions.
|
||||
//
|
||||
// All outbound requests go through the SSRF-safe HTTP client so a malicious
|
||||
// feed cannot redirect WeKnora to internal services.
|
||||
package rss
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Config holds RSS-specific configuration parsed from DataSourceConfig.Credentials.
|
||||
//
|
||||
// Both fields live under Credentials (rather than Settings) because the
|
||||
// existing data-source editor UI only renders credential inputs, and because
|
||||
// AuthHeaders may carry secrets that must be encrypted at rest. FeedURLs are
|
||||
// not secret but ride along for the same single-input flow.
|
||||
type Config struct {
|
||||
// FeedURLs is a newline- or comma-separated list of feed URLs.
|
||||
FeedURLs string `json:"feed_urls"`
|
||||
|
||||
// AuthHeaders is an optional newline-separated list of custom request
|
||||
// headers in "Name: Value" form, applied to every outbound request.
|
||||
AuthHeaders string `json:"auth_headers,omitempty"`
|
||||
}
|
||||
|
||||
// parseConfig extracts and validates RSS configuration from the credentials map.
|
||||
func parseConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("%w: config is nil", datasource.ErrInvalidConfig)
|
||||
}
|
||||
credBytes, err := json.Marshal(config.Credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal credentials: %w", err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(credBytes, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse rss credentials: %w", err)
|
||||
}
|
||||
if len(cfg.feedURLList()) == 0 {
|
||||
return nil, fmt.Errorf("%w: feed_urls is required", datasource.ErrInvalidCredentials)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// feedURLList splits FeedURLs on newlines and commas, trims, dedupes (order
|
||||
// preserved), and drops blanks.
|
||||
func (c *Config) feedURLList() []string {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
raw := strings.FieldsFunc(c.FeedURLs, func(r rune) bool {
|
||||
return r == '\n' || r == '\r' || r == ','
|
||||
})
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, u := range raw {
|
||||
u = strings.TrimSpace(u)
|
||||
if u == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[u]; ok {
|
||||
continue
|
||||
}
|
||||
seen[u] = struct{}{}
|
||||
out = append(out, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseHeaders turns the newline-separated "Name: Value" AuthHeaders blob into
|
||||
// a map. Lines without a colon, or with an empty name, are skipped.
|
||||
func (c *Config) parseHeaders() map[string]string {
|
||||
if c == nil || strings.TrimSpace(c.AuthHeaders) == "" {
|
||||
return nil
|
||||
}
|
||||
headers := make(map[string]string)
|
||||
for _, line := range strings.Split(c.AuthHeaders, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(line, ":")
|
||||
if idx <= 0 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(line[:idx])
|
||||
value := strings.TrimSpace(line[idx+1:])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
headers[name] = value
|
||||
}
|
||||
if len(headers) == 0 {
|
||||
return nil
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// rssCursor stores incremental sync state.
|
||||
//
|
||||
// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is the
|
||||
// item's updated/published timestamp (RFC3339) or, when no timestamp exists,
|
||||
// a "h:<sha256-prefix>" content hash. An item is considered unchanged when its
|
||||
// fingerprint matches the stored one.
|
||||
type rssCursor struct {
|
||||
LastSyncTime time.Time `json:"last_sync_time"`
|
||||
FeedItems map[string]map[string]string `json:"feed_items,omitempty"`
|
||||
}
|
||||
|
||||
// itemFingerprint returns a change-detection token for a feed item. It prefers
|
||||
// the explicit updated/published timestamp; absent that, it hashes the content
|
||||
// so edits to timestamp-less items are still detected.
|
||||
func itemFingerprint(updated, published *time.Time, content string) string {
|
||||
if updated != nil && !updated.IsZero() {
|
||||
return updated.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if published != nil && !published.IsZero() {
|
||||
return published.UTC().Format(time.RFC3339)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
return "h:" + hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty trimmed string among the args.
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters invalid in filenames and truncates to a
|
||||
// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
|
||||
func sanitizeFileName(name string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
"\n", " ", "\r", " ", "\t", " ",
|
||||
)
|
||||
result := strings.TrimSpace(replacer.Replace(name))
|
||||
if result == "" {
|
||||
return "untitled"
|
||||
}
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -53,6 +53,10 @@ func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) UpdateSyncState(ctx context.Context, ds *types.DataSource) error {
|
||||
return r.Update(ctx, ds)
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -117,6 +121,10 @@ func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) UpdateResult(ctx context.Context, log *types.SyncLog) error {
|
||||
return r.Update(ctx, log)
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) CancelPendingByDataSource(_ context.Context, dsID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user