From edbcfa80ab34f6312a4d2966540c87b0b78207d1 Mon Sep 17 00:00:00 2001 From: toy0116 Date: Wed, 6 May 2026 21:15:03 +0800 Subject: [PATCH] fix(wiki-ingest): prevent retry exhaustion on concurrent lock conflict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: when multiple wiki:ingest tasks queue up for the same KB, they all hit ErrWikiIngestConcurrent and retry every 15 s. With MaxRetry=10 (150 s window), any batch taking longer than ~2.5 min caused pending tasks to exhaust retries and land in archived. Confirmed via Redis inspection: archived wiki:ingest task had retry=10, retried=10, error="concurrent wiki task active". Two-part fix: 1. Early-exit at lock conflict when pending list is empty. If another batch is active and the pending list is already empty, there is nothing left to process — the concurrent task will handle everything. Return nil (success) immediately instead of burning through retry slots on a guaranteed no-op. Only retry when the pending list still has items, i.e., we have real work to do. 2. Increase MaxRetry 10 → 25 across all four enqueue sites. 25 × 15 s = ~6 min window; accommodates large KB batches and the 60 s orphan-lock expiry with generous headroom. Combined with the early-exit above, redundant tasks now consume 0 retries instead of 10, so the remaining budget is entirely available for genuine concurrency conflicts. No API or schema changes. Co-Authored-By: Claude Sonnet 4.6 --- internal/application/service/wiki_ingest.go | 6 +++--- internal/application/service/wiki_ingest_batch.go | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index ca78d2be..258b82f3 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -222,7 +222,7 @@ func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisC t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(10), // Increased from 3 to 10 to ensure it can outlast the 5-minute active lock TTL + asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), ) @@ -262,7 +262,7 @@ func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redis payloadBytes, _ := json.Marshal(ingestPayload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL + asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly ) @@ -379,7 +379,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn payloadBytes, _ := json.Marshal(retryPayload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(10), + asynq.MaxRetry(25), // match main enqueue budget asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), ) diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 5a39bc9f..0db76eaa 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -34,7 +34,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn payloadBytes, _ := json.Marshal(payload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL + asynq.MaxRetry(25), // match main enqueue budget; outlasts large KB batches asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then ) @@ -117,6 +117,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err) } else if !acquired { exitStatus = "active_lock_conflict" + // If the pending list is already empty, the active batch will process + // everything — no need to retry. Returning nil avoids burning through the + // retry budget on tasks that would be no-ops when they eventually acquire + // the lock. If there are still pending ops, retry so we don't miss them + // in case the active batch drained the list before we RPush'd. + pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID + if n, _ := s.redisClient.LLen(ctx, pendingKey).Result(); n == 0 { + exitStatus = "active_lock_conflict_empty" + logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending list empty — skipping", payload.KnowledgeBaseID) + return nil + } logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) return ErrWikiIngestConcurrent }