fix(datasource): enhance error handling in ProcessSync method

- Improved logging for missing data sources, providing clearer warnings when a data source is not found.
- Updated sync log status to 'canceled' when a data source is deleted, ensuring accurate tracking of sync operations.
- Adjusted error handling to prevent unnecessary error returns, enhancing overall process stability.
This commit is contained in:
wizardchen
2026-03-30 15:12:00 +08:00
committed by lyingbug
parent 36dac0120a
commit 53bd412cf6
3 changed files with 35 additions and 3 deletions
@@ -3,6 +3,7 @@ package repository
import (
"context"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -220,6 +221,23 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro
return nil
}
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error {
if dsID == "" {
return errors.New("data source id is empty")
}
now := time.Now()
return r.db.WithContext(ctx).
Model(&types.SyncLog{}).
Where("data_source_id = ?", dsID).
Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}).
Updates(map[string]interface{}{
"status": types.SyncLogStatusCanceled,
"finished_at": &now,
"error_message": "data source deleted",
}).Error
}
// CleanupOldLogs deletes sync logs older than the retention period
func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error {
if retentionDays <= 0 {
@@ -172,6 +172,11 @@ func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) err
// Remove cron schedule
s.scheduler.Remove(id)
// Cancel any pending/running sync logs so queued asynq tasks won't retry
if err := s.syncLogRepo.CancelPendingByDataSource(ctx, id); err != nil {
logger.Warnf(ctx, "failed to cancel pending sync logs for ds=%s: %v", id, err)
}
logger.Infof(ctx, "data source deleted: id=%s", id)
return nil
}
@@ -369,15 +374,21 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
// Get data source
ds, err := s.GetDataSource(ctx, payload.DataSourceID)
if err != nil {
logger.Errorf(ctx, "failed to get data source: %v", err)
return err
logger.Warnf(ctx, "data source not found (likely deleted), cancelling sync: ds=%s err=%v", payload.DataSourceID, err)
if syncLog, slErr := s.syncLogRepo.FindByID(ctx, payload.SyncLogID); slErr == nil && syncLog != nil {
syncLog.Status = types.SyncLogStatusCanceled
syncLog.FinishedAt = timePtr(time.Now())
syncLog.ErrorMessage = "data source has been deleted"
_ = s.syncLogRepo.Update(ctx, syncLog)
}
return nil
}
// Get sync log
syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID)
if err != nil {
logger.Errorf(ctx, "failed to get sync log: %v", err)
return err
return nil
}
// Get connector
+3
View File
@@ -95,6 +95,9 @@ type SyncLogRepository interface {
// Update updates an existing sync log entry
Update(ctx context.Context, log *types.SyncLog) error
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
CancelPendingByDataSource(ctx context.Context, dsID string) error
// CleanupOldLogs deletes sync logs older than the retention period
CleanupOldLogs(ctx context.Context, retentionDays int) error
}