From ba8c96d83a3ba1b931c8bae861f0670c26ed9968 Mon Sep 17 00:00:00 2001 From: langcaiye Date: Thu, 4 Jun 2026 16:16:05 +0800 Subject: [PATCH] fix(datasource): fail sync when all fetched items fail --- .../application/repository/datasource_repo.go | 56 +++++++++++ .../repository/datasource_repo_test.go | 87 ++++++++++++++++ .../application/service/datasource_service.go | 98 +++++++++++++------ .../service/datasource_service_test.go | 49 ++++++++++ internal/types/interfaces/datasource.go | 6 ++ 5 files changed, 268 insertions(+), 28 deletions(-) create mode 100644 internal/application/repository/datasource_repo_test.go create mode 100644 internal/application/service/datasource_service_test.go diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go index c04fcb17..87faf8c2 100644 --- a/internal/application/repository/datasource_repo.go +++ b/internal/application/repository/datasource_repo.go @@ -81,6 +81,32 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource) return nil } +// UpdateSyncState updates only fields managed by sync execution. GORM's +// Updates(struct) skips zero values, so use a map here to persist cleared error +// messages without broadening the generic Update method. +func (r *DataSourceRepository) UpdateSyncState(ctx context.Context, ds *types.DataSource) error { + if ds == nil { + return errors.New("data source is nil") + } + if ds.ID == "" { + return errors.New("data source id is empty") + } + if err := r.db.WithContext(ctx). + Model(&types.DataSource{}). + Where("id = ?", ds.ID). + Updates(map[string]interface{}{ + "status": ds.Status, + "last_sync_at": ds.LastSyncAt, + "last_sync_cursor": ds.LastSyncCursor, + "last_sync_result": ds.LastSyncResult, + "error_message": ds.ErrorMessage, + "updated_at": time.Now().UTC(), + }).Error; err != nil { + return err + } + return nil +} + // Delete performs a soft delete func (r *DataSourceRepository) Delete(ctx context.Context, id string) error { if id == "" { @@ -221,6 +247,36 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro return nil } +// UpdateResult updates only fields produced by sync execution. Use an explicit +// map so empty error messages are written when a later sync succeeds. +func (r *SyncLogRepository) UpdateResult(ctx context.Context, log *types.SyncLog) error { + if log == nil { + return errors.New("sync log is nil") + } + if log.ID == "" { + return errors.New("sync log id is empty") + } + if err := r.db.WithContext(ctx). + Model(&types.SyncLog{}). + Where("id = ?", log.ID). + Updates(map[string]interface{}{ + "status": log.Status, + "finished_at": log.FinishedAt, + "items_total": log.ItemsTotal, + "items_created": log.ItemsCreated, + "items_updated": log.ItemsUpdated, + "items_deleted": log.ItemsDeleted, + "items_skipped": log.ItemsSkipped, + "items_failed": log.ItemsFailed, + "error_message": log.ErrorMessage, + "result": log.Result, + "updated_at": time.Now().UTC(), + }).Error; err != nil { + return err + } + return nil +} + // CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled. func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error { if dsID == "" { diff --git a/internal/application/repository/datasource_repo_test.go b/internal/application/repository/datasource_repo_test.go new file mode 100644 index 00000000..03cb0e03 --- /dev/null +++ b/internal/application/repository/datasource_repo_test.go @@ -0,0 +1,87 @@ +package repository + +import ( + "context" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func setupDataSourceRepoTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.DataSource{}, &types.SyncLog{})) + return db +} + +func TestDataSourceRepositoryUpdateSyncStateClearsErrorMessage(t *testing.T) { + db := setupDataSourceRepoTestDB(t) + repo := NewDataSourceRepository(db) + now := time.Now().UTC() + result := types.JSON(`{"total":0}`) + + ds := &types.DataSource{ + ID: "ds-1", + TenantID: 1, + KnowledgeBaseID: "kb-1", + Name: "Feishu", + Type: types.ConnectorTypeFeishu, + Status: types.DataSourceStatusError, + ErrorMessage: "previous failure", + } + require.NoError(t, repo.Create(context.Background(), ds)) + + ds.Status = types.DataSourceStatusActive + ds.ErrorMessage = "" + ds.LastSyncAt = &now + ds.LastSyncResult = result + require.NoError(t, repo.UpdateSyncState(context.Background(), ds)) + + var stored types.DataSource + require.NoError(t, db.First(&stored, "id = ?", ds.ID).Error) + assert.Equal(t, types.DataSourceStatusActive, stored.Status) + assert.Empty(t, stored.ErrorMessage) + assert.Equal(t, result.ToString(), stored.LastSyncResult.ToString()) + require.NotNil(t, stored.LastSyncAt) +} + +func TestSyncLogRepositoryUpdateResultClearsErrorMessage(t *testing.T) { + db := setupDataSourceRepoTestDB(t) + repo := NewSyncLogRepository(db) + finishedAt := time.Now().UTC() + result := types.JSON(`{"total":0}`) + + log := &types.SyncLog{ + ID: "log-1", + DataSourceID: "ds-1", + TenantID: 1, + Status: types.SyncLogStatusFailed, + ErrorMessage: "previous failure", + ItemsTotal: 1, + ItemsFailed: 1, + } + require.NoError(t, repo.Create(context.Background(), log)) + + log.Status = types.SyncLogStatusSuccess + log.ErrorMessage = "" + log.FinishedAt = &finishedAt + log.ItemsTotal = 0 + log.ItemsFailed = 0 + log.Result = result + require.NoError(t, repo.UpdateResult(context.Background(), log)) + + var stored types.SyncLog + require.NoError(t, db.First(&stored, "id = ?", log.ID).Error) + assert.Equal(t, types.SyncLogStatusSuccess, stored.Status) + assert.Empty(t, stored.ErrorMessage) + assert.Zero(t, stored.ItemsTotal) + assert.Zero(t, stored.ItemsFailed) + assert.Equal(t, result.ToString(), stored.Result.ToString()) + require.NotNil(t, stored.FinishedAt) +} diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index de4ccc90..2ecc025b 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -666,15 +666,12 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e } } - // Update sync log with results - syncLog.ItemsTotal = result.Total - syncLog.ItemsCreated = result.Created - syncLog.ItemsUpdated = result.Updated - syncLog.ItemsDeleted = result.Deleted - syncLog.ItemsSkipped = result.Skipped - syncLog.ItemsFailed = result.Failed - syncLog.Status = types.SyncLogStatusSuccess - syncLog.FinishedAt = timePtr(time.Now().UTC()) + resultJSON, _ := result.ToJSON() + if err := allFetchedItemsFailedError(result); err != nil { + logger.Errorf(ctx, "data source sync failed while processing fetched items: %v", err) + s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused) + return err + } // Update cursor for next incremental sync if nextCursor != nil { @@ -683,25 +680,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e } ds.LastSyncAt = timePtr(time.Now().UTC()) - if wasPaused { - ds.Status = types.DataSourceStatusPaused - } else { - ds.Status = types.DataSourceStatusActive - } - ds.ErrorMessage = "" - - // Store result - resultJSON, _ := result.ToJSON() - ds.LastSyncResult = resultJSON - syncLog.Result = resultJSON - - // Update database - if err := s.dsRepo.Update(ctx, ds); err != nil { - logger.Errorf(ctx, "failed to update data source: %v", err) - } - if err := s.syncLogRepo.Update(ctx, syncLog); err != nil { - logger.Errorf(ctx, "failed to update sync log: %v", err) - } + s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusSuccess, "", wasPaused) logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d", payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted) @@ -709,6 +688,69 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e return nil } +func (s *DataSourceService) updateSyncRunResult( + ctx context.Context, + ds *types.DataSource, + syncLog *types.SyncLog, + result *types.SyncResult, + resultJSON types.JSON, + status string, + errorMessage string, + wasPaused bool, +) { + syncLog.ItemsTotal = result.Total + syncLog.ItemsCreated = result.Created + syncLog.ItemsUpdated = result.Updated + syncLog.ItemsDeleted = result.Deleted + syncLog.ItemsSkipped = result.Skipped + syncLog.ItemsFailed = result.Failed + syncLog.Status = status + syncLog.FinishedAt = timePtr(time.Now().UTC()) + syncLog.ErrorMessage = errorMessage + syncLog.Result = resultJSON + if err := s.syncLogRepo.UpdateResult(ctx, syncLog); err != nil { + logger.Errorf(ctx, "failed to update sync log: %v", err) + } + + if status == types.SyncLogStatusFailed { + if !wasPaused { + ds.Status = types.DataSourceStatusError + } + } else if wasPaused { + ds.Status = types.DataSourceStatusPaused + } else { + ds.Status = types.DataSourceStatusActive + } + ds.ErrorMessage = errorMessage + ds.LastSyncResult = resultJSON + if err := s.dsRepo.UpdateSyncState(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to update data source: %v", err) + } +} + +func allFetchedItemsFailedError(result *types.SyncResult) error { + if result == nil || result.Total == 0 { + return nil + } + if result.Failed != result.Total || result.Created != 0 || result.Updated != 0 || + result.Deleted != 0 || result.Skipped != 0 { + return nil + } + + detail := "" + if len(result.Errors) > 0 { + detail = result.Errors[0] + const maxDetailLen = 500 + if len(detail) > maxDetailLen { + detail = detail[:maxDetailLen] + "..." + } + } + if detail == "" { + return fmt.Errorf("all fetched items failed during sync (%d/%d)", result.Failed, result.Total) + } + return fmt.Errorf("all fetched items failed during sync (%d/%d): %s", result.Failed, result.Total, detail) +} + // ValidateCredentials tests connectivity using raw credentials without persisting anything. func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error { connector, err := s.connectorRegistry.Get(connectorType) diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.go new file mode 100644 index 00000000..60999369 --- /dev/null +++ b/internal/application/service/datasource_service_test.go @@ -0,0 +1,49 @@ +package service + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAllFetchedItemsFailedError(t *testing.T) { + err := allFetchedItemsFailedError(&types.SyncResult{ + Total: 2, + Failed: 2, + Errors: []string{"doc one: export failed"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "all fetched items failed during sync (2/2)") + assert.Contains(t, err.Error(), "doc one: export failed") +} + +func TestAllFetchedItemsFailedErrorIgnoresPartialFailure(t *testing.T) { + err := allFetchedItemsFailedError(&types.SyncResult{ + Total: 3, + Created: 1, + Failed: 2, + }) + require.NoError(t, err) +} + +func TestAllFetchedItemsFailedErrorIgnoresSkippedItems(t *testing.T) { + err := allFetchedItemsFailedError(&types.SyncResult{ + Total: 3, + Skipped: 3, + }) + require.NoError(t, err) +} + +func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) { + err := allFetchedItemsFailedError(&types.SyncResult{ + Total: 1, + Failed: 1, + Errors: []string{strings.Repeat("x", 600)}, + }) + require.Error(t, err) + assert.LessOrEqual(t, len(err.Error()), 560) + assert.Contains(t, err.Error(), "...") +} diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go index fe7b4926..f9235a84 100644 --- a/internal/types/interfaces/datasource.go +++ b/internal/types/interfaces/datasource.go @@ -79,6 +79,9 @@ type DataSourceRepository interface { // Update updates an existing data source Update(ctx context.Context, ds *types.DataSource) error + // UpdateSyncState updates only fields produced by a sync run. + UpdateSyncState(ctx context.Context, ds *types.DataSource) error + // Delete performs a soft delete Delete(ctx context.Context, id string) error @@ -107,6 +110,9 @@ type SyncLogRepository interface { // Update updates an existing sync log entry Update(ctx context.Context, log *types.SyncLog) error + // UpdateResult updates only fields produced by a sync run. + UpdateResult(ctx context.Context, log *types.SyncLog) error + // CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled. CancelPendingByDataSource(ctx context.Context, dsID string) error