From 53bd412cf6bc642c84add7cf6fc3b917a2fde989 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 30 Mar 2026 15:12:00 +0800 Subject: [PATCH] fix(datasource): enhance error handling in ProcessSync method - Improved logging for missing data sources, providing clearer warnings when a data source is not found. - Updated sync log status to 'canceled' when a data source is deleted, ensuring accurate tracking of sync operations. - Adjusted error handling to prevent unnecessary error returns, enhancing overall process stability. --- .../application/repository/datasource_repo.go | 18 ++++++++++++++++++ .../application/service/datasource_service.go | 17 ++++++++++++++--- internal/types/interfaces/datasource.go | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go index a888b18a..97ce9ee6 100644 --- a/internal/application/repository/datasource_repo.go +++ b/internal/application/repository/datasource_repo.go @@ -3,6 +3,7 @@ package repository import ( "context" "errors" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -220,6 +221,23 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro return nil } +// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled. +func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error { + if dsID == "" { + return errors.New("data source id is empty") + } + now := time.Now() + return r.db.WithContext(ctx). + Model(&types.SyncLog{}). + Where("data_source_id = ?", dsID). + Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}). + Updates(map[string]interface{}{ + "status": types.SyncLogStatusCanceled, + "finished_at": &now, + "error_message": "data source deleted", + }).Error +} + // CleanupOldLogs deletes sync logs older than the retention period func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error { if retentionDays <= 0 { diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 3eeb98a5..10e10cbd 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -172,6 +172,11 @@ func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) err // Remove cron schedule s.scheduler.Remove(id) + // Cancel any pending/running sync logs so queued asynq tasks won't retry + if err := s.syncLogRepo.CancelPendingByDataSource(ctx, id); err != nil { + logger.Warnf(ctx, "failed to cancel pending sync logs for ds=%s: %v", id, err) + } + logger.Infof(ctx, "data source deleted: id=%s", id) return nil } @@ -369,15 +374,21 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e // Get data source ds, err := s.GetDataSource(ctx, payload.DataSourceID) if err != nil { - logger.Errorf(ctx, "failed to get data source: %v", err) - return err + logger.Warnf(ctx, "data source not found (likely deleted), cancelling sync: ds=%s err=%v", payload.DataSourceID, err) + if syncLog, slErr := s.syncLogRepo.FindByID(ctx, payload.SyncLogID); slErr == nil && syncLog != nil { + syncLog.Status = types.SyncLogStatusCanceled + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = "data source has been deleted" + _ = s.syncLogRepo.Update(ctx, syncLog) + } + return nil } // Get sync log syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID) if err != nil { logger.Errorf(ctx, "failed to get sync log: %v", err) - return err + return nil } // Get connector diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go index 71172c88..ec5b0d4c 100644 --- a/internal/types/interfaces/datasource.go +++ b/internal/types/interfaces/datasource.go @@ -95,6 +95,9 @@ type SyncLogRepository interface { // Update updates an existing sync log entry Update(ctx context.Context, log *types.SyncLog) error + // CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled. + CancelPendingByDataSource(ctx context.Context, dsID string) error + // CleanupOldLogs deletes sync logs older than the retention period CleanupOldLogs(ctx context.Context, retentionDays int) error }