fix(datasource): enhance error handling in ProcessSync method
- Improved logging for missing data sources, providing clearer warnings when a data source is not found. - Updated sync log status to 'canceled' when a data source is deleted, ensuring accurate tracking of sync operations. - Adjusted error handling to prevent unnecessary error returns, enhancing overall process stability.
This commit is contained in:
@@ -3,6 +3,7 @@ package repository
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -220,6 +221,23 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
|
||||
func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error {
|
||||
if dsID == "" {
|
||||
return errors.New("data source id is empty")
|
||||
}
|
||||
now := time.Now()
|
||||
return r.db.WithContext(ctx).
|
||||
Model(&types.SyncLog{}).
|
||||
Where("data_source_id = ?", dsID).
|
||||
Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}).
|
||||
Updates(map[string]interface{}{
|
||||
"status": types.SyncLogStatusCanceled,
|
||||
"finished_at": &now,
|
||||
"error_message": "data source deleted",
|
||||
}).Error
|
||||
}
|
||||
|
||||
// CleanupOldLogs deletes sync logs older than the retention period
|
||||
func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error {
|
||||
if retentionDays <= 0 {
|
||||
|
||||
@@ -172,6 +172,11 @@ func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) err
|
||||
// Remove cron schedule
|
||||
s.scheduler.Remove(id)
|
||||
|
||||
// Cancel any pending/running sync logs so queued asynq tasks won't retry
|
||||
if err := s.syncLogRepo.CancelPendingByDataSource(ctx, id); err != nil {
|
||||
logger.Warnf(ctx, "failed to cancel pending sync logs for ds=%s: %v", id, err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "data source deleted: id=%s", id)
|
||||
return nil
|
||||
}
|
||||
@@ -369,15 +374,21 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
// Get data source
|
||||
ds, err := s.GetDataSource(ctx, payload.DataSourceID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get data source: %v", err)
|
||||
return err
|
||||
logger.Warnf(ctx, "data source not found (likely deleted), cancelling sync: ds=%s err=%v", payload.DataSourceID, err)
|
||||
if syncLog, slErr := s.syncLogRepo.FindByID(ctx, payload.SyncLogID); slErr == nil && syncLog != nil {
|
||||
syncLog.Status = types.SyncLogStatusCanceled
|
||||
syncLog.FinishedAt = timePtr(time.Now())
|
||||
syncLog.ErrorMessage = "data source has been deleted"
|
||||
_ = s.syncLogRepo.Update(ctx, syncLog)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get sync log
|
||||
syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get sync log: %v", err)
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get connector
|
||||
|
||||
@@ -95,6 +95,9 @@ type SyncLogRepository interface {
|
||||
// Update updates an existing sync log entry
|
||||
Update(ctx context.Context, log *types.SyncLog) error
|
||||
|
||||
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
|
||||
CancelPendingByDataSource(ctx context.Context, dsID string) error
|
||||
|
||||
// CleanupOldLogs deletes sync logs older than the retention period
|
||||
CleanupOldLogs(ctx context.Context, retentionDays int) error
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user