fix(wiki-ingest): prevent retry exhaustion on concurrent lock conflict

Root cause: when multiple wiki:ingest tasks queue up for the same KB,
they all hit ErrWikiIngestConcurrent and retry every 15 s. With
MaxRetry=10 (150 s window), any batch taking longer than ~2.5 min
caused pending tasks to exhaust retries and land in archived.

Confirmed via Redis inspection: archived wiki:ingest task had
retry=10, retried=10, error="concurrent wiki task active".

Two-part fix:

1. Early-exit at lock conflict when pending list is empty.
   If another batch is active and the pending list is already empty,
   there is nothing left to process — the concurrent task will handle
   everything. Return nil (success) immediately instead of burning
   through retry slots on a guaranteed no-op. Only retry when the
   pending list still has items, i.e., we have real work to do.

2. Increase MaxRetry 10 → 25 across all four enqueue sites.
   25 × 15 s = ~6 min window; accommodates large KB batches and the
   60 s orphan-lock expiry with generous headroom. Combined with the
   early-exit above, redundant tasks now consume 0 retries instead of
   10, so the remaining budget is entirely available for genuine
   concurrency conflicts.

No API or schema changes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
toy0116
2026-05-06 21:15:03 +08:00
committed by lyingbug
parent 16d31cb87b
commit edbcfa80ab
2 changed files with 15 additions and 4 deletions
+3 -3
View File
@@ -222,7 +222,7 @@ func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisC
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10), // Increased from 3 to 10 to ensure it can outlast the 5-minute active lock TTL
asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiIngestDelay),
)
@@ -262,7 +262,7 @@ func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redis
payloadBytes, _ := json.Marshal(ingestPayload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly
)
@@ -379,7 +379,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
payloadBytes, _ := json.Marshal(retryPayload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10),
asynq.MaxRetry(25), // match main enqueue budget
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiIngestDelay),
)
@@ -34,7 +34,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
payloadBytes, _ := json.Marshal(payload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
asynq.MaxRetry(25), // match main enqueue budget; outlasts large KB batches
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then
)
@@ -117,6 +117,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err)
} else if !acquired {
exitStatus = "active_lock_conflict"
// If the pending list is already empty, the active batch will process
// everything — no need to retry. Returning nil avoids burning through the
// retry budget on tasks that would be no-ops when they eventually acquire
// the lock. If there are still pending ops, retry so we don't miss them
// in case the active batch drained the list before we RPush'd.
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
if n, _ := s.redisClient.LLen(ctx, pendingKey).Result(); n == 0 {
exitStatus = "active_lock_conflict_empty"
logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending list empty — skipping", payload.KnowledgeBaseID)
return nil
}
logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID)
return ErrWikiIngestConcurrent
}