fix(datasource): harden RSS connector and editor UX after review
Scope auth headers to feed fetches, use feed-scoped external IDs and content-based incremental fingerprints, tolerate partial feed failures, store feed URLs in settings, and fix credential configured detection.
This commit is contained in:
@@ -80,6 +80,12 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if cfg, err := ds.ParseConfig(); err == nil && cfg != nil {
|
||||
cfg.StripNonSecretCredentials(ds.Type)
|
||||
if blob, err := cfg.ToJSON(); err == nil {
|
||||
ds.Config = blob
|
||||
}
|
||||
}
|
||||
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -176,6 +182,7 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
|
||||
} else {
|
||||
merged.Credentials = nil
|
||||
}
|
||||
merged.StripNonSecretCredentials(ds.Type)
|
||||
if blob, err := merged.ToJSON(); err == nil {
|
||||
ds.Config = blob
|
||||
}
|
||||
@@ -192,7 +199,7 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
|
||||
if mergedCfg != nil && existingParsedCfg != nil {
|
||||
configActuallyChanged = !reflect.DeepEqual(*mergedCfg, *existingParsedCfg)
|
||||
}
|
||||
hasCreds := mergedCfg != nil && mergedCfg.HasCredentials()
|
||||
hasCreds := mergedCfg != nil && mergedCfg.HasConfiguredCredentials(ds.Type)
|
||||
if hasCreds && (ds.Type != existing.Type || configActuallyChanged) {
|
||||
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
|
||||
return nil, err
|
||||
@@ -236,6 +243,7 @@ func (s *DataSourceService) UpdateDataSourceCredentials(
|
||||
parsed = &types.DataSourceConfig{Type: existing.Type}
|
||||
}
|
||||
parsed.Credentials = credentials
|
||||
parsed.StripNonSecretCredentials(existing.Type)
|
||||
blob, err := parsed.ToJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -269,9 +277,18 @@ func (s *DataSourceService) ClearDataSourceCredentials(ctx context.Context, id s
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if parsed == nil || !parsed.HasCredentials() {
|
||||
if parsed == nil {
|
||||
return nil
|
||||
}
|
||||
parsed.StripNonSecretCredentials(existing.Type)
|
||||
if !parsed.HasConfiguredCredentials(existing.Type) {
|
||||
blob, err := parsed.ToJSON()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing.Config = blob
|
||||
return s.dsRepo.Update(ctx, existing)
|
||||
}
|
||||
parsed.Credentials = nil
|
||||
blob, err := parsed.ToJSON()
|
||||
if err != nil {
|
||||
@@ -562,6 +579,16 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID); err != nil {
|
||||
logger.Warnf(ctx, "knowledge base not found (likely deleted), cancelling sync: kb=%s ds=%s err=%v",
|
||||
ds.KnowledgeBaseID, payload.DataSourceID, err)
|
||||
syncLog.Status = types.SyncLogStatusCanceled
|
||||
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
||||
syncLog.ErrorMessage = "knowledge base has been deleted"
|
||||
_ = s.syncLogRepo.Update(ctx, syncLog)
|
||||
return nil
|
||||
}
|
||||
|
||||
wasPaused := ds.Status == types.DataSourceStatusPaused
|
||||
|
||||
// Get connector
|
||||
|
||||
Reference in New Issue
Block a user