fix(datasource): fail sync when all fetched items fail

This commit is contained in:
langcaiye
2026-06-04 16:16:05 +08:00
committed by lyingbug
parent 4369d156e1
commit ba8c96d83a
5 changed files with 268 additions and 28 deletions
@@ -81,6 +81,32 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)
return nil
}
// UpdateSyncState updates only fields managed by sync execution. GORM's
// Updates(struct) skips zero values, so use a map here to persist cleared error
// messages without broadening the generic Update method.
func (r *DataSourceRepository) UpdateSyncState(ctx context.Context, ds *types.DataSource) error {
if ds == nil {
return errors.New("data source is nil")
}
if ds.ID == "" {
return errors.New("data source id is empty")
}
if err := r.db.WithContext(ctx).
Model(&types.DataSource{}).
Where("id = ?", ds.ID).
Updates(map[string]interface{}{
"status": ds.Status,
"last_sync_at": ds.LastSyncAt,
"last_sync_cursor": ds.LastSyncCursor,
"last_sync_result": ds.LastSyncResult,
"error_message": ds.ErrorMessage,
"updated_at": time.Now().UTC(),
}).Error; err != nil {
return err
}
return nil
}
// Delete performs a soft delete
func (r *DataSourceRepository) Delete(ctx context.Context, id string) error {
if id == "" {
@@ -221,6 +247,36 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro
return nil
}
// UpdateResult updates only fields produced by sync execution. Use an explicit
// map so empty error messages are written when a later sync succeeds.
func (r *SyncLogRepository) UpdateResult(ctx context.Context, log *types.SyncLog) error {
if log == nil {
return errors.New("sync log is nil")
}
if log.ID == "" {
return errors.New("sync log id is empty")
}
if err := r.db.WithContext(ctx).
Model(&types.SyncLog{}).
Where("id = ?", log.ID).
Updates(map[string]interface{}{
"status": log.Status,
"finished_at": log.FinishedAt,
"items_total": log.ItemsTotal,
"items_created": log.ItemsCreated,
"items_updated": log.ItemsUpdated,
"items_deleted": log.ItemsDeleted,
"items_skipped": log.ItemsSkipped,
"items_failed": log.ItemsFailed,
"error_message": log.ErrorMessage,
"result": log.Result,
"updated_at": time.Now().UTC(),
}).Error; err != nil {
return err
}
return nil
}
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error {
if dsID == "" {
@@ -0,0 +1,87 @@
package repository
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func setupDataSourceRepoTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.DataSource{}, &types.SyncLog{}))
return db
}
func TestDataSourceRepositoryUpdateSyncStateClearsErrorMessage(t *testing.T) {
db := setupDataSourceRepoTestDB(t)
repo := NewDataSourceRepository(db)
now := time.Now().UTC()
result := types.JSON(`{"total":0}`)
ds := &types.DataSource{
ID: "ds-1",
TenantID: 1,
KnowledgeBaseID: "kb-1",
Name: "Feishu",
Type: types.ConnectorTypeFeishu,
Status: types.DataSourceStatusError,
ErrorMessage: "previous failure",
}
require.NoError(t, repo.Create(context.Background(), ds))
ds.Status = types.DataSourceStatusActive
ds.ErrorMessage = ""
ds.LastSyncAt = &now
ds.LastSyncResult = result
require.NoError(t, repo.UpdateSyncState(context.Background(), ds))
var stored types.DataSource
require.NoError(t, db.First(&stored, "id = ?", ds.ID).Error)
assert.Equal(t, types.DataSourceStatusActive, stored.Status)
assert.Empty(t, stored.ErrorMessage)
assert.Equal(t, result.ToString(), stored.LastSyncResult.ToString())
require.NotNil(t, stored.LastSyncAt)
}
func TestSyncLogRepositoryUpdateResultClearsErrorMessage(t *testing.T) {
db := setupDataSourceRepoTestDB(t)
repo := NewSyncLogRepository(db)
finishedAt := time.Now().UTC()
result := types.JSON(`{"total":0}`)
log := &types.SyncLog{
ID: "log-1",
DataSourceID: "ds-1",
TenantID: 1,
Status: types.SyncLogStatusFailed,
ErrorMessage: "previous failure",
ItemsTotal: 1,
ItemsFailed: 1,
}
require.NoError(t, repo.Create(context.Background(), log))
log.Status = types.SyncLogStatusSuccess
log.ErrorMessage = ""
log.FinishedAt = &finishedAt
log.ItemsTotal = 0
log.ItemsFailed = 0
log.Result = result
require.NoError(t, repo.UpdateResult(context.Background(), log))
var stored types.SyncLog
require.NoError(t, db.First(&stored, "id = ?", log.ID).Error)
assert.Equal(t, types.SyncLogStatusSuccess, stored.Status)
assert.Empty(t, stored.ErrorMessage)
assert.Zero(t, stored.ItemsTotal)
assert.Zero(t, stored.ItemsFailed)
assert.Equal(t, result.ToString(), stored.Result.ToString())
require.NotNil(t, stored.FinishedAt)
}
@@ -666,15 +666,12 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
}
}
// Update sync log with results
syncLog.ItemsTotal = result.Total
syncLog.ItemsCreated = result.Created
syncLog.ItemsUpdated = result.Updated
syncLog.ItemsDeleted = result.Deleted
syncLog.ItemsSkipped = result.Skipped
syncLog.ItemsFailed = result.Failed
syncLog.Status = types.SyncLogStatusSuccess
syncLog.FinishedAt = timePtr(time.Now().UTC())
resultJSON, _ := result.ToJSON()
if err := allFetchedItemsFailedError(result); err != nil {
logger.Errorf(ctx, "data source sync failed while processing fetched items: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
return err
}
// Update cursor for next incremental sync
if nextCursor != nil {
@@ -683,25 +680,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
}
ds.LastSyncAt = timePtr(time.Now().UTC())
if wasPaused {
ds.Status = types.DataSourceStatusPaused
} else {
ds.Status = types.DataSourceStatusActive
}
ds.ErrorMessage = ""
// Store result
resultJSON, _ := result.ToJSON()
ds.LastSyncResult = resultJSON
syncLog.Result = resultJSON
// Update database
if err := s.dsRepo.Update(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to update data source: %v", err)
}
if err := s.syncLogRepo.Update(ctx, syncLog); err != nil {
logger.Errorf(ctx, "failed to update sync log: %v", err)
}
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusSuccess, "", wasPaused)
logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d",
payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted)
@@ -709,6 +688,69 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
return nil
}
func (s *DataSourceService) updateSyncRunResult(
ctx context.Context,
ds *types.DataSource,
syncLog *types.SyncLog,
result *types.SyncResult,
resultJSON types.JSON,
status string,
errorMessage string,
wasPaused bool,
) {
syncLog.ItemsTotal = result.Total
syncLog.ItemsCreated = result.Created
syncLog.ItemsUpdated = result.Updated
syncLog.ItemsDeleted = result.Deleted
syncLog.ItemsSkipped = result.Skipped
syncLog.ItemsFailed = result.Failed
syncLog.Status = status
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = errorMessage
syncLog.Result = resultJSON
if err := s.syncLogRepo.UpdateResult(ctx, syncLog); err != nil {
logger.Errorf(ctx, "failed to update sync log: %v", err)
}
if status == types.SyncLogStatusFailed {
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
} else if wasPaused {
ds.Status = types.DataSourceStatusPaused
} else {
ds.Status = types.DataSourceStatusActive
}
ds.ErrorMessage = errorMessage
ds.LastSyncResult = resultJSON
if err := s.dsRepo.UpdateSyncState(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to update data source: %v", err)
}
}
func allFetchedItemsFailedError(result *types.SyncResult) error {
if result == nil || result.Total == 0 {
return nil
}
if result.Failed != result.Total || result.Created != 0 || result.Updated != 0 ||
result.Deleted != 0 || result.Skipped != 0 {
return nil
}
detail := ""
if len(result.Errors) > 0 {
detail = result.Errors[0]
const maxDetailLen = 500
if len(detail) > maxDetailLen {
detail = detail[:maxDetailLen] + "..."
}
}
if detail == "" {
return fmt.Errorf("all fetched items failed during sync (%d/%d)", result.Failed, result.Total)
}
return fmt.Errorf("all fetched items failed during sync (%d/%d): %s", result.Failed, result.Total, detail)
}
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
connector, err := s.connectorRegistry.Get(connectorType)
@@ -0,0 +1,49 @@
package service
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAllFetchedItemsFailedError(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 2,
Failed: 2,
Errors: []string{"doc one: export failed"},
})
require.Error(t, err)
assert.Contains(t, err.Error(), "all fetched items failed during sync (2/2)")
assert.Contains(t, err.Error(), "doc one: export failed")
}
func TestAllFetchedItemsFailedErrorIgnoresPartialFailure(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 3,
Created: 1,
Failed: 2,
})
require.NoError(t, err)
}
func TestAllFetchedItemsFailedErrorIgnoresSkippedItems(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 3,
Skipped: 3,
})
require.NoError(t, err)
}
func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 1,
Failed: 1,
Errors: []string{strings.Repeat("x", 600)},
})
require.Error(t, err)
assert.LessOrEqual(t, len(err.Error()), 560)
assert.Contains(t, err.Error(), "...")
}
+6
View File
@@ -79,6 +79,9 @@ type DataSourceRepository interface {
// Update updates an existing data source
Update(ctx context.Context, ds *types.DataSource) error
// UpdateSyncState updates only fields produced by a sync run.
UpdateSyncState(ctx context.Context, ds *types.DataSource) error
// Delete performs a soft delete
Delete(ctx context.Context, id string) error
@@ -107,6 +110,9 @@ type SyncLogRepository interface {
// Update updates an existing sync log entry
Update(ctx context.Context, log *types.SyncLog) error
// UpdateResult updates only fields produced by a sync run.
UpdateResult(ctx context.Context, log *types.SyncLog) error
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
CancelPendingByDataSource(ctx context.Context, dsID string) error