fix(datasource): harden RSS connector and editor UX after review

Scope auth headers to feed fetches, use feed-scoped external IDs and
content-based incremental fingerprints, tolerate partial feed failures,
store feed URLs in settings, and fix credential configured detection.
This commit is contained in:
wizardchen
2026-06-23 15:45:47 +08:00
committed by lyingbug
parent e40ad802d7
commit 3cb0e906a7
12 changed files with 430 additions and 96 deletions
@@ -80,6 +80,12 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data
}
// Validate configuration
if cfg, err := ds.ParseConfig(); err == nil && cfg != nil {
cfg.StripNonSecretCredentials(ds.Type)
if blob, err := cfg.ToJSON(); err == nil {
ds.Config = blob
}
}
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
return nil, err
}
@@ -176,6 +182,7 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
} else {
merged.Credentials = nil
}
merged.StripNonSecretCredentials(ds.Type)
if blob, err := merged.ToJSON(); err == nil {
ds.Config = blob
}
@@ -192,7 +199,7 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
if mergedCfg != nil && existingParsedCfg != nil {
configActuallyChanged = !reflect.DeepEqual(*mergedCfg, *existingParsedCfg)
}
hasCreds := mergedCfg != nil && mergedCfg.HasCredentials()
hasCreds := mergedCfg != nil && mergedCfg.HasConfiguredCredentials(ds.Type)
if hasCreds && (ds.Type != existing.Type || configActuallyChanged) {
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
return nil, err
@@ -236,6 +243,7 @@ func (s *DataSourceService) UpdateDataSourceCredentials(
parsed = &types.DataSourceConfig{Type: existing.Type}
}
parsed.Credentials = credentials
parsed.StripNonSecretCredentials(existing.Type)
blob, err := parsed.ToJSON()
if err != nil {
return nil, err
@@ -269,9 +277,18 @@ func (s *DataSourceService) ClearDataSourceCredentials(ctx context.Context, id s
if err != nil {
return err
}
if parsed == nil || !parsed.HasCredentials() {
if parsed == nil {
return nil
}
parsed.StripNonSecretCredentials(existing.Type)
if !parsed.HasConfiguredCredentials(existing.Type) {
blob, err := parsed.ToJSON()
if err != nil {
return err
}
existing.Config = blob
return s.dsRepo.Update(ctx, existing)
}
parsed.Credentials = nil
blob, err := parsed.ToJSON()
if err != nil {
@@ -562,6 +579,16 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
return nil
}
if _, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID); err != nil {
logger.Warnf(ctx, "knowledge base not found (likely deleted), cancelling sync: kb=%s ds=%s err=%v",
ds.KnowledgeBaseID, payload.DataSourceID, err)
syncLog.Status = types.SyncLogStatusCanceled
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = "knowledge base has been deleted"
_ = s.syncLogRepo.Update(ctx, syncLog)
return nil
}
wasPaused := ds.Status == types.DataSourceStatusPaused
// Get connector