fix(wiki-ingest): prevent retry exhaustion on concurrent lock conflict
Root cause: when multiple wiki:ingest tasks queue up for the same KB, they all hit ErrWikiIngestConcurrent and retry every 15 s. With MaxRetry=10 (150 s window), any batch taking longer than ~2.5 min caused pending tasks to exhaust retries and land in archived. Confirmed via Redis inspection: archived wiki:ingest task had retry=10, retried=10, error="concurrent wiki task active". Two-part fix: 1. Early-exit at lock conflict when pending list is empty. If another batch is active and the pending list is already empty, there is nothing left to process — the concurrent task will handle everything. Return nil (success) immediately instead of burning through retry slots on a guaranteed no-op. Only retry when the pending list still has items, i.e., we have real work to do. 2. Increase MaxRetry 10 → 25 across all four enqueue sites. 25 × 15 s = ~6 min window; accommodates large KB batches and the 60 s orphan-lock expiry with generous headroom. Combined with the early-exit above, redundant tasks now consume 0 retries instead of 10, so the remaining budget is entirely available for genuine concurrency conflicts. No API or schema changes. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -222,7 +222,7 @@ func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisC
|
||||
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue("low"),
|
||||
asynq.MaxRetry(10), // Increased from 3 to 10 to ensure it can outlast the 5-minute active lock TTL
|
||||
asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(wikiIngestDelay),
|
||||
)
|
||||
@@ -262,7 +262,7 @@ func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redis
|
||||
payloadBytes, _ := json.Marshal(ingestPayload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue("low"),
|
||||
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
|
||||
asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly
|
||||
)
|
||||
@@ -379,7 +379,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
|
||||
payloadBytes, _ := json.Marshal(retryPayload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue("low"),
|
||||
asynq.MaxRetry(10),
|
||||
asynq.MaxRetry(25), // match main enqueue budget
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(wikiIngestDelay),
|
||||
)
|
||||
|
||||
@@ -34,7 +34,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
|
||||
payloadBytes, _ := json.Marshal(payload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue("low"),
|
||||
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
|
||||
asynq.MaxRetry(25), // match main enqueue budget; outlasts large KB batches
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then
|
||||
)
|
||||
@@ -117,6 +117,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err)
|
||||
} else if !acquired {
|
||||
exitStatus = "active_lock_conflict"
|
||||
// If the pending list is already empty, the active batch will process
|
||||
// everything — no need to retry. Returning nil avoids burning through the
|
||||
// retry budget on tasks that would be no-ops when they eventually acquire
|
||||
// the lock. If there are still pending ops, retry so we don't miss them
|
||||
// in case the active batch drained the list before we RPush'd.
|
||||
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
|
||||
if n, _ := s.redisClient.LLen(ctx, pendingKey).Result(); n == 0 {
|
||||
exitStatus = "active_lock_conflict_empty"
|
||||
logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending list empty — skipping", payload.KnowledgeBaseID)
|
||||
return nil
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID)
|
||||
return ErrWikiIngestConcurrent
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user