fix(datasource): harden RSS connector and editor UX after review

Scope auth headers to feed fetches, use feed-scoped external IDs and
content-based incremental fingerprints, tolerate partial feed failures,
store feed URLs in settings, and fix credential configured detection.
This commit is contained in:
wizardchen
2026-06-23 15:45:47 +08:00
committed by lyingbug
parent e40ad802d7
commit 3cb0e906a7
12 changed files with 430 additions and 96 deletions
+2 -2
View File
@@ -83,8 +83,8 @@ func (h *DataSourceCredentialsHandler) Put(c *gin.Context) {
return
}
configured := false
if parsed, err := updated.ParseConfig(); err == nil && parsed != nil && parsed.HasCredentials() {
configured = true
if parsed, err := updated.ParseConfig(); err == nil && parsed != nil {
configured = parsed.HasConfiguredCredentials(updated.Type)
}
resp := dto.CredentialsResponse{
Fields: map[string]dto.CredentialFieldMetadata{
+29 -1
View File
@@ -2,6 +2,7 @@ package dto
import (
"encoding/json"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
@@ -65,7 +66,8 @@ func NewDataSourceResponse(ds *types.DataSource) *DataSourceResponse {
ResourceIDs: parsed.ResourceIDs,
Settings: parsed.Settings,
}
configured = parsed.HasCredentials()
enrichRSSFeedURLsInSettings(ds.Type, parsed, cfgDTO)
configured = parsed.HasConfiguredCredentials(ds.Type)
}
return &DataSourceResponse{
ID: ds.ID,
@@ -94,6 +96,32 @@ func NewDataSourceResponse(ds *types.DataSource) *DataSourceResponse {
}
}
// enrichRSSFeedURLsInSettings copies feed_urls from credentials into settings
// for API responses. Feed URLs are not secrets but may still live in the
// encrypted credentials blob on rows created before they moved to settings.
func enrichRSSFeedURLsInSettings(dsType string, parsed *types.DataSourceConfig, cfgDTO *DataSourceConfigDTO) {
if dsType != types.ConnectorTypeRSS || parsed == nil || cfgDTO == nil {
return
}
if cfgDTO.Settings != nil {
if v, ok := cfgDTO.Settings["feed_urls"].(string); ok && strings.TrimSpace(v) != "" {
return
}
}
raw, ok := parsed.Credentials["feed_urls"]
if !ok {
return
}
feedURLs, ok := raw.(string)
if !ok || strings.TrimSpace(feedURLs) == "" {
return
}
if cfgDTO.Settings == nil {
cfgDTO.Settings = make(map[string]interface{})
}
cfgDTO.Settings["feed_urls"] = feedURLs
}
func NewDataSourceResponses(dss []*types.DataSource) []*DataSourceResponse {
out := make([]*DataSourceResponse, 0, len(dss))
for _, d := range dss {
+50
View File
@@ -53,6 +53,56 @@ func TestDataSourceResponse_NilSafe(t *testing.T) {
assert.Equal(t, []*DataSourceResponse{}, NewDataSourceResponses(nil))
}
func TestDataSourceResponse_RSSFeedURLsFromCredentials(t *testing.T) {
cfg := types.DataSourceConfig{
Type: types.ConnectorTypeRSS,
Credentials: map[string]interface{}{
"feed_urls": "https://example.com/a.xml\nhttps://example.com/b.xml",
},
ResourceIDs: []string{"https://example.com/a.xml"},
}
blob, _ := cfg.ToJSON()
ds := &types.DataSource{
ID: "ds-rss",
Name: "my-rss",
Type: types.ConnectorTypeRSS,
Config: blob,
}
resp := NewDataSourceResponse(ds)
assert.NotNil(t, resp.Config)
assert.Equal(t, "https://example.com/a.xml\nhttps://example.com/b.xml", resp.Config.Settings["feed_urls"])
assert.False(t, resp.Credentials["credentials"].Configured)
body, err := json.Marshal(resp)
assert.NoError(t, err)
s := string(body)
assert.Contains(t, s, "https://example.com/a.xml")
assert.NotContains(t, s, "Bearer secret")
}
func TestDataSourceResponse_RSSAuthHeadersConfigured(t *testing.T) {
cfg := types.DataSourceConfig{
Type: types.ConnectorTypeRSS,
Credentials: map[string]interface{}{
"auth_headers": "Authorization: Bearer secret",
},
Settings: map[string]interface{}{
"feed_urls": "https://example.com/feed.xml",
},
}
blob, _ := cfg.ToJSON()
ds := &types.DataSource{
ID: "ds-rss-auth",
Name: "my-rss",
Type: types.ConnectorTypeRSS,
Config: blob,
}
resp := NewDataSourceResponse(ds)
assert.True(t, resp.Credentials["credentials"].Configured)
body, err := json.Marshal(resp)
assert.NoError(t, err)
assert.NotContains(t, string(body), "Bearer secret")
}
func TestDataSourceResponse_NoConfig(t *testing.T) {
ds := &types.DataSource{ID: "x", Name: "x"}
body, err := json.Marshal(NewDataSourceResponse(ds))