fix(datasource): fail sync when all fetched items fail
This commit is contained in:
@@ -81,6 +81,32 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateSyncState updates only fields managed by sync execution. GORM's
|
||||
// Updates(struct) skips zero values, so use a map here to persist cleared error
|
||||
// messages without broadening the generic Update method.
|
||||
func (r *DataSourceRepository) UpdateSyncState(ctx context.Context, ds *types.DataSource) error {
|
||||
if ds == nil {
|
||||
return errors.New("data source is nil")
|
||||
}
|
||||
if ds.ID == "" {
|
||||
return errors.New("data source id is empty")
|
||||
}
|
||||
if err := r.db.WithContext(ctx).
|
||||
Model(&types.DataSource{}).
|
||||
Where("id = ?", ds.ID).
|
||||
Updates(map[string]interface{}{
|
||||
"status": ds.Status,
|
||||
"last_sync_at": ds.LastSyncAt,
|
||||
"last_sync_cursor": ds.LastSyncCursor,
|
||||
"last_sync_result": ds.LastSyncResult,
|
||||
"error_message": ds.ErrorMessage,
|
||||
"updated_at": time.Now().UTC(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete performs a soft delete
|
||||
func (r *DataSourceRepository) Delete(ctx context.Context, id string) error {
|
||||
if id == "" {
|
||||
@@ -221,6 +247,36 @@ func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateResult updates only fields produced by sync execution. Use an explicit
|
||||
// map so empty error messages are written when a later sync succeeds.
|
||||
func (r *SyncLogRepository) UpdateResult(ctx context.Context, log *types.SyncLog) error {
|
||||
if log == nil {
|
||||
return errors.New("sync log is nil")
|
||||
}
|
||||
if log.ID == "" {
|
||||
return errors.New("sync log id is empty")
|
||||
}
|
||||
if err := r.db.WithContext(ctx).
|
||||
Model(&types.SyncLog{}).
|
||||
Where("id = ?", log.ID).
|
||||
Updates(map[string]interface{}{
|
||||
"status": log.Status,
|
||||
"finished_at": log.FinishedAt,
|
||||
"items_total": log.ItemsTotal,
|
||||
"items_created": log.ItemsCreated,
|
||||
"items_updated": log.ItemsUpdated,
|
||||
"items_deleted": log.ItemsDeleted,
|
||||
"items_skipped": log.ItemsSkipped,
|
||||
"items_failed": log.ItemsFailed,
|
||||
"error_message": log.ErrorMessage,
|
||||
"result": log.Result,
|
||||
"updated_at": time.Now().UTC(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
|
||||
func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID string) error {
|
||||
if dsID == "" {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupDataSourceRepoTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.DataSource{}, &types.SyncLog{}))
|
||||
return db
|
||||
}
|
||||
|
||||
func TestDataSourceRepositoryUpdateSyncStateClearsErrorMessage(t *testing.T) {
|
||||
db := setupDataSourceRepoTestDB(t)
|
||||
repo := NewDataSourceRepository(db)
|
||||
now := time.Now().UTC()
|
||||
result := types.JSON(`{"total":0}`)
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-1",
|
||||
TenantID: 1,
|
||||
KnowledgeBaseID: "kb-1",
|
||||
Name: "Feishu",
|
||||
Type: types.ConnectorTypeFeishu,
|
||||
Status: types.DataSourceStatusError,
|
||||
ErrorMessage: "previous failure",
|
||||
}
|
||||
require.NoError(t, repo.Create(context.Background(), ds))
|
||||
|
||||
ds.Status = types.DataSourceStatusActive
|
||||
ds.ErrorMessage = ""
|
||||
ds.LastSyncAt = &now
|
||||
ds.LastSyncResult = result
|
||||
require.NoError(t, repo.UpdateSyncState(context.Background(), ds))
|
||||
|
||||
var stored types.DataSource
|
||||
require.NoError(t, db.First(&stored, "id = ?", ds.ID).Error)
|
||||
assert.Equal(t, types.DataSourceStatusActive, stored.Status)
|
||||
assert.Empty(t, stored.ErrorMessage)
|
||||
assert.Equal(t, result.ToString(), stored.LastSyncResult.ToString())
|
||||
require.NotNil(t, stored.LastSyncAt)
|
||||
}
|
||||
|
||||
func TestSyncLogRepositoryUpdateResultClearsErrorMessage(t *testing.T) {
|
||||
db := setupDataSourceRepoTestDB(t)
|
||||
repo := NewSyncLogRepository(db)
|
||||
finishedAt := time.Now().UTC()
|
||||
result := types.JSON(`{"total":0}`)
|
||||
|
||||
log := &types.SyncLog{
|
||||
ID: "log-1",
|
||||
DataSourceID: "ds-1",
|
||||
TenantID: 1,
|
||||
Status: types.SyncLogStatusFailed,
|
||||
ErrorMessage: "previous failure",
|
||||
ItemsTotal: 1,
|
||||
ItemsFailed: 1,
|
||||
}
|
||||
require.NoError(t, repo.Create(context.Background(), log))
|
||||
|
||||
log.Status = types.SyncLogStatusSuccess
|
||||
log.ErrorMessage = ""
|
||||
log.FinishedAt = &finishedAt
|
||||
log.ItemsTotal = 0
|
||||
log.ItemsFailed = 0
|
||||
log.Result = result
|
||||
require.NoError(t, repo.UpdateResult(context.Background(), log))
|
||||
|
||||
var stored types.SyncLog
|
||||
require.NoError(t, db.First(&stored, "id = ?", log.ID).Error)
|
||||
assert.Equal(t, types.SyncLogStatusSuccess, stored.Status)
|
||||
assert.Empty(t, stored.ErrorMessage)
|
||||
assert.Zero(t, stored.ItemsTotal)
|
||||
assert.Zero(t, stored.ItemsFailed)
|
||||
assert.Equal(t, result.ToString(), stored.Result.ToString())
|
||||
require.NotNil(t, stored.FinishedAt)
|
||||
}
|
||||
@@ -666,15 +666,12 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync log with results
|
||||
syncLog.ItemsTotal = result.Total
|
||||
syncLog.ItemsCreated = result.Created
|
||||
syncLog.ItemsUpdated = result.Updated
|
||||
syncLog.ItemsDeleted = result.Deleted
|
||||
syncLog.ItemsSkipped = result.Skipped
|
||||
syncLog.ItemsFailed = result.Failed
|
||||
syncLog.Status = types.SyncLogStatusSuccess
|
||||
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
||||
resultJSON, _ := result.ToJSON()
|
||||
if err := allFetchedItemsFailedError(result); err != nil {
|
||||
logger.Errorf(ctx, "data source sync failed while processing fetched items: %v", err)
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
|
||||
return err
|
||||
}
|
||||
|
||||
// Update cursor for next incremental sync
|
||||
if nextCursor != nil {
|
||||
@@ -683,25 +680,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
|
||||
ds.LastSyncAt = timePtr(time.Now().UTC())
|
||||
if wasPaused {
|
||||
ds.Status = types.DataSourceStatusPaused
|
||||
} else {
|
||||
ds.Status = types.DataSourceStatusActive
|
||||
}
|
||||
ds.ErrorMessage = ""
|
||||
|
||||
// Store result
|
||||
resultJSON, _ := result.ToJSON()
|
||||
ds.LastSyncResult = resultJSON
|
||||
syncLog.Result = resultJSON
|
||||
|
||||
// Update database
|
||||
if err := s.dsRepo.Update(ctx, ds); err != nil {
|
||||
logger.Errorf(ctx, "failed to update data source: %v", err)
|
||||
}
|
||||
if err := s.syncLogRepo.Update(ctx, syncLog); err != nil {
|
||||
logger.Errorf(ctx, "failed to update sync log: %v", err)
|
||||
}
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusSuccess, "", wasPaused)
|
||||
|
||||
logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d",
|
||||
payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted)
|
||||
@@ -709,6 +688,69 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DataSourceService) updateSyncRunResult(
|
||||
ctx context.Context,
|
||||
ds *types.DataSource,
|
||||
syncLog *types.SyncLog,
|
||||
result *types.SyncResult,
|
||||
resultJSON types.JSON,
|
||||
status string,
|
||||
errorMessage string,
|
||||
wasPaused bool,
|
||||
) {
|
||||
syncLog.ItemsTotal = result.Total
|
||||
syncLog.ItemsCreated = result.Created
|
||||
syncLog.ItemsUpdated = result.Updated
|
||||
syncLog.ItemsDeleted = result.Deleted
|
||||
syncLog.ItemsSkipped = result.Skipped
|
||||
syncLog.ItemsFailed = result.Failed
|
||||
syncLog.Status = status
|
||||
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
||||
syncLog.ErrorMessage = errorMessage
|
||||
syncLog.Result = resultJSON
|
||||
if err := s.syncLogRepo.UpdateResult(ctx, syncLog); err != nil {
|
||||
logger.Errorf(ctx, "failed to update sync log: %v", err)
|
||||
}
|
||||
|
||||
if status == types.SyncLogStatusFailed {
|
||||
if !wasPaused {
|
||||
ds.Status = types.DataSourceStatusError
|
||||
}
|
||||
} else if wasPaused {
|
||||
ds.Status = types.DataSourceStatusPaused
|
||||
} else {
|
||||
ds.Status = types.DataSourceStatusActive
|
||||
}
|
||||
ds.ErrorMessage = errorMessage
|
||||
ds.LastSyncResult = resultJSON
|
||||
if err := s.dsRepo.UpdateSyncState(ctx, ds); err != nil {
|
||||
logger.Errorf(ctx, "failed to update data source: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func allFetchedItemsFailedError(result *types.SyncResult) error {
|
||||
if result == nil || result.Total == 0 {
|
||||
return nil
|
||||
}
|
||||
if result.Failed != result.Total || result.Created != 0 || result.Updated != 0 ||
|
||||
result.Deleted != 0 || result.Skipped != 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
detail := ""
|
||||
if len(result.Errors) > 0 {
|
||||
detail = result.Errors[0]
|
||||
const maxDetailLen = 500
|
||||
if len(detail) > maxDetailLen {
|
||||
detail = detail[:maxDetailLen] + "..."
|
||||
}
|
||||
}
|
||||
if detail == "" {
|
||||
return fmt.Errorf("all fetched items failed during sync (%d/%d)", result.Failed, result.Total)
|
||||
}
|
||||
return fmt.Errorf("all fetched items failed during sync (%d/%d): %s", result.Failed, result.Total, detail)
|
||||
}
|
||||
|
||||
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
|
||||
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
|
||||
connector, err := s.connectorRegistry.Get(connectorType)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAllFetchedItemsFailedError(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 2,
|
||||
Failed: 2,
|
||||
Errors: []string{"doc one: export failed"},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "all fetched items failed during sync (2/2)")
|
||||
assert.Contains(t, err.Error(), "doc one: export failed")
|
||||
}
|
||||
|
||||
func TestAllFetchedItemsFailedErrorIgnoresPartialFailure(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 3,
|
||||
Created: 1,
|
||||
Failed: 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAllFetchedItemsFailedErrorIgnoresSkippedItems(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 3,
|
||||
Skipped: 3,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 1,
|
||||
Failed: 1,
|
||||
Errors: []string{strings.Repeat("x", 600)},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.LessOrEqual(t, len(err.Error()), 560)
|
||||
assert.Contains(t, err.Error(), "...")
|
||||
}
|
||||
@@ -79,6 +79,9 @@ type DataSourceRepository interface {
|
||||
// Update updates an existing data source
|
||||
Update(ctx context.Context, ds *types.DataSource) error
|
||||
|
||||
// UpdateSyncState updates only fields produced by a sync run.
|
||||
UpdateSyncState(ctx context.Context, ds *types.DataSource) error
|
||||
|
||||
// Delete performs a soft delete
|
||||
Delete(ctx context.Context, id string) error
|
||||
|
||||
@@ -107,6 +110,9 @@ type SyncLogRepository interface {
|
||||
// Update updates an existing sync log entry
|
||||
Update(ctx context.Context, log *types.SyncLog) error
|
||||
|
||||
// UpdateResult updates only fields produced by a sync run.
|
||||
UpdateResult(ctx context.Context, log *types.SyncLog) error
|
||||
|
||||
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
|
||||
CancelPendingByDataSource(ctx context.Context, dsID string) error
|
||||
|
||||
|
||||
Reference in New Issue
Block a user