From 1f037414624c31cf499d831357910fb0a4c5b45d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 28 Apr 2026 18:08:20 +0800 Subject: [PATCH] fix(wiki): Lite ingest lock, failed-op requeue, sync task retry parity - Add per-KB sync.Map lock in Lite mode to mirror Redis SetNX concurrency. - Requeue failed wiki ops in Lite by enqueueing asynq tasks with delay/tracing. - Teach SyncTaskExecutor to honor ProcessIn delay and MaxRetry like Redis asynq. --- internal/application/service/wiki_ingest.go | 59 ++++++++++++++----- .../application/service/wiki_ingest_batch.go | 10 +++- internal/router/sync_task.go | 57 +++++++++++++++--- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 984bc438..736b8820 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strings" + "sync" "text/template" "time" @@ -139,6 +140,9 @@ type wikiIngestService struct { modelService interfaces.ModelService task interfaces.TaskEnqueuer redisClient *redis.Client // nil in Lite mode (no Redis) + // liteLocks provides per-KB mutual exclusion in Lite mode (no Redis). + // Keys are kbID strings; values are unused (presence = locked). + liteLocks sync.Map } // NewWikiIngestService creates a new wiki ingest service @@ -336,25 +340,52 @@ func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, co } } -// requeueFailedOps appends failed operations back to the pending list so they -// are retried in the next follow-up batch. Called after trimPendingList has -// already removed the consumed batch head. -func (s *wikiIngestService) requeueFailedOps(ctx context.Context, kbID string, ops []WikiPendingOp) { - if s.redisClient == nil { +// requeueFailedOps re-enqueues failed operations for retry. +// +// Redis mode: appends ops back to the pending list tail so the next follow-up +// batch picks them up. +// +// Lite mode (no Redis): enqueues a new asynq task per failed op with a short +// delay, since there is no shared pending list to append to. +func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) { + if s.redisClient != nil { + pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID + for _, op := range ops { + data, err := json.Marshal(op) + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err) + continue + } + if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err) + continue + } + logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle) + } return } - pendingKey := wikiPendingKeyPrefix + kbID + + // Lite mode: re-enqueue each failed op as a new asynq task. for _, op := range ops { - data, err := json.Marshal(op) - if err != nil { - logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err) + retryPayload := WikiIngestPayload{ + TenantID: payload.TenantID, + KnowledgeBaseID: payload.KnowledgeBaseID, + Language: op.Language, + LiteOps: []WikiPendingOp{op}, + } + langfuse.InjectTracing(ctx, &retryPayload) + payloadBytes, _ := json.Marshal(retryPayload) + t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, + asynq.Queue("low"), + asynq.MaxRetry(10), + asynq.Timeout(60*time.Minute), + asynq.ProcessIn(wikiIngestDelay), + ) + if _, err := s.task.Enqueue(t); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to requeue lite op %s: %v", op.KnowledgeID, err) continue } - if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err) - continue - } - logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle) + logger.Infof(ctx, "wiki ingest: re-queued failed lite op %s (%s) for retry", op.KnowledgeID, op.DocTitle) } } diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 638a0aa2..dee902c2 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -142,6 +142,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task }() } else { mode = "lite" + // In-process mutual exclusion: mirrors the Redis SetNX lock above. + if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { + exitStatus = "active_lock_conflict" + logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID) + return ErrWikiIngestConcurrent + } + lockAcquired = true + defer s.liteLocks.Delete(payload.KnowledgeBaseID) } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) @@ -419,7 +427,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // This must happen after trim: trim removes the consumed batch head, then // requeue appends the failed items to the tail for a future attempt. if len(failedOps) > 0 { - s.requeueFailedOps(ctx, payload.KnowledgeBaseID, failedOps) + s.requeueFailedOps(ctx, payload, failedOps) } logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected)) diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go index 3b32c57a..dacc9bd0 100644 --- a/internal/router/sync_task.go +++ b/internal/router/sync_task.go @@ -36,7 +36,8 @@ func (e *SyncTaskExecutor) RegisterHandler(pattern string, handler func(context. // Enqueue satisfies interfaces.TaskEnqueuer. // Instead of queuing to Redis, it dispatches the task to a goroutine. -func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { +// Supports ProcessIn (delay) and MaxRetry options for parity with asynq. +func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) { e.mu.RLock() handler, ok := e.handlers[task.Type()] e.mu.RUnlock() @@ -45,6 +46,28 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq. return nil, fmt.Errorf("sync task executor: no handler registered for type %q", task.Type()) } + var delay time.Duration + maxRetry := 25 // asynq default + maxRetrySet := false + for _, opt := range opts { + switch opt.Type() { + case asynq.ProcessInOpt: + if d, ok := opt.Value().(time.Duration); ok { + delay = d + } + case asynq.MaxRetryOpt: + if n, ok := opt.Value().(int); ok { + maxRetry = n + maxRetrySet = true + } + } + } + // Callers that explicitly pass MaxRetry(0) want no retries. + // Without the flag we can't distinguish "not set" from "set to 0". + if maxRetrySet && maxRetry < 0 { + maxRetry = 0 + } + taskID := uuid.New().String() info := &asynq.TaskInfo{ ID: taskID, @@ -53,16 +76,36 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq. } go func() { + if delay > 0 { + time.Sleep(delay) + } + ctx := context.Background() start := time.Now() logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID) - if err := handler(ctx, task); err != nil { - logger.Errorf(ctx, "[SyncTask] Task failed type=%s id=%s elapsed=%v err=%v", - task.Type(), taskID, time.Since(start), err) - } else { - logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v", - task.Type(), taskID, time.Since(start)) + + var lastErr error + for attempt := 0; attempt <= maxRetry; attempt++ { + if attempt > 0 { + backoff := time.Duration(attempt) * 5 * time.Second + if backoff > 30*time.Second { + backoff = 30 * time.Second + } + logger.Infof(ctx, "[SyncTask] Retrying task type=%s id=%s attempt=%d/%d backoff=%s", + task.Type(), taskID, attempt, maxRetry, backoff) + time.Sleep(backoff) + } + + lastErr = handler(ctx, task) + if lastErr == nil { + logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v", + task.Type(), taskID, time.Since(start)) + return + } } + + logger.Errorf(ctx, "[SyncTask] Task failed (exhausted retries) type=%s id=%s elapsed=%v err=%v", + task.Type(), taskID, time.Since(start), lastErr) }() return info, nil