fix(datasource): harden RSS connector and editor UX after review
Scope auth headers to feed fetches, use feed-scoped external IDs and content-based incremental fingerprints, tolerate partial feed failures, store feed URLs in settings, and fix credential configured detection.
This commit is contained in:
@@ -83,8 +83,8 @@ func (h *DataSourceCredentialsHandler) Put(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
configured := false
|
||||
if parsed, err := updated.ParseConfig(); err == nil && parsed != nil && parsed.HasCredentials() {
|
||||
configured = true
|
||||
if parsed, err := updated.ParseConfig(); err == nil && parsed != nil {
|
||||
configured = parsed.HasConfiguredCredentials(updated.Type)
|
||||
}
|
||||
resp := dto.CredentialsResponse{
|
||||
Fields: map[string]dto.CredentialFieldMetadata{
|
||||
|
||||
@@ -2,6 +2,7 @@ package dto
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -65,7 +66,8 @@ func NewDataSourceResponse(ds *types.DataSource) *DataSourceResponse {
|
||||
ResourceIDs: parsed.ResourceIDs,
|
||||
Settings: parsed.Settings,
|
||||
}
|
||||
configured = parsed.HasCredentials()
|
||||
enrichRSSFeedURLsInSettings(ds.Type, parsed, cfgDTO)
|
||||
configured = parsed.HasConfiguredCredentials(ds.Type)
|
||||
}
|
||||
return &DataSourceResponse{
|
||||
ID: ds.ID,
|
||||
@@ -94,6 +96,32 @@ func NewDataSourceResponse(ds *types.DataSource) *DataSourceResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// enrichRSSFeedURLsInSettings copies feed_urls from credentials into settings
|
||||
// for API responses. Feed URLs are not secrets but may still live in the
|
||||
// encrypted credentials blob on rows created before they moved to settings.
|
||||
func enrichRSSFeedURLsInSettings(dsType string, parsed *types.DataSourceConfig, cfgDTO *DataSourceConfigDTO) {
|
||||
if dsType != types.ConnectorTypeRSS || parsed == nil || cfgDTO == nil {
|
||||
return
|
||||
}
|
||||
if cfgDTO.Settings != nil {
|
||||
if v, ok := cfgDTO.Settings["feed_urls"].(string); ok && strings.TrimSpace(v) != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
raw, ok := parsed.Credentials["feed_urls"]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
feedURLs, ok := raw.(string)
|
||||
if !ok || strings.TrimSpace(feedURLs) == "" {
|
||||
return
|
||||
}
|
||||
if cfgDTO.Settings == nil {
|
||||
cfgDTO.Settings = make(map[string]interface{})
|
||||
}
|
||||
cfgDTO.Settings["feed_urls"] = feedURLs
|
||||
}
|
||||
|
||||
func NewDataSourceResponses(dss []*types.DataSource) []*DataSourceResponse {
|
||||
out := make([]*DataSourceResponse, 0, len(dss))
|
||||
for _, d := range dss {
|
||||
|
||||
@@ -53,6 +53,56 @@ func TestDataSourceResponse_NilSafe(t *testing.T) {
|
||||
assert.Equal(t, []*DataSourceResponse{}, NewDataSourceResponses(nil))
|
||||
}
|
||||
|
||||
func TestDataSourceResponse_RSSFeedURLsFromCredentials(t *testing.T) {
|
||||
cfg := types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Credentials: map[string]interface{}{
|
||||
"feed_urls": "https://example.com/a.xml\nhttps://example.com/b.xml",
|
||||
},
|
||||
ResourceIDs: []string{"https://example.com/a.xml"},
|
||||
}
|
||||
blob, _ := cfg.ToJSON()
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-rss",
|
||||
Name: "my-rss",
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Config: blob,
|
||||
}
|
||||
resp := NewDataSourceResponse(ds)
|
||||
assert.NotNil(t, resp.Config)
|
||||
assert.Equal(t, "https://example.com/a.xml\nhttps://example.com/b.xml", resp.Config.Settings["feed_urls"])
|
||||
assert.False(t, resp.Credentials["credentials"].Configured)
|
||||
body, err := json.Marshal(resp)
|
||||
assert.NoError(t, err)
|
||||
s := string(body)
|
||||
assert.Contains(t, s, "https://example.com/a.xml")
|
||||
assert.NotContains(t, s, "Bearer secret")
|
||||
}
|
||||
|
||||
func TestDataSourceResponse_RSSAuthHeadersConfigured(t *testing.T) {
|
||||
cfg := types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Credentials: map[string]interface{}{
|
||||
"auth_headers": "Authorization: Bearer secret",
|
||||
},
|
||||
Settings: map[string]interface{}{
|
||||
"feed_urls": "https://example.com/feed.xml",
|
||||
},
|
||||
}
|
||||
blob, _ := cfg.ToJSON()
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-rss-auth",
|
||||
Name: "my-rss",
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Config: blob,
|
||||
}
|
||||
resp := NewDataSourceResponse(ds)
|
||||
assert.True(t, resp.Credentials["credentials"].Configured)
|
||||
body, err := json.Marshal(resp)
|
||||
assert.NoError(t, err)
|
||||
assert.NotContains(t, string(body), "Bearer secret")
|
||||
}
|
||||
|
||||
func TestDataSourceResponse_NoConfig(t *testing.T) {
|
||||
ds := &types.DataSource{ID: "x", Name: "x"}
|
||||
body, err := json.Marshal(NewDataSourceResponse(ds))
|
||||
|
||||
Reference in New Issue
Block a user