From e66c42343b05a25b22eddaf27cfc3d19c7ccbd20 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 8 May 2026 21:57:51 +0800 Subject: [PATCH] fix(wiki-ingest): harden conflict retry and right-size retry budget Avoid treating Redis pending-length read failures as empty queues, reset stale fail-count keys on fresh ingest/drop paths, and reduce wiki ingest MaxRetry to a moderate shared constant to limit queue churn. --- internal/application/service/wiki_ingest.go | 18 +++++++++++++++--- .../application/service/wiki_ingest_batch.go | 10 ++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 2fd7cf39..2fbefde8 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -72,6 +72,11 @@ const ( // persistently-broken doc clog the queue indefinitely. wikiMaxFailRetries = 5 + // wikiIngestMaxRetry controls asynq retry budget for wiki:ingest tasks. + // Keep this moderate: lock conflicts already retry every 15s via + // asynqRetryDelayFunc, and follow-up/retract paths fire quickly. + wikiIngestMaxRetry = 10 + // wikiDeletedKeyPrefix is the Redis key prefix for "recently deleted // knowledge" tombstones. Key: wiki:deleted:{kbID}:{knowledgeID}. Written // by cleanupWikiOnKnowledgeDelete so that any wiki_ingest task still in @@ -229,6 +234,10 @@ func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisC // Push to Redis pending list (if Redis available) if redisClient != nil { pendingKey := wikiPendingKeyPrefix + kbID + // Reset stale fail counter for fresh user-triggered ingest enqueue. + // This gives a newly re-ingested document a full retry budget. + failKey := wikiFailCountKeyPrefix + kbID + ":" + knowledgeID + redisClient.Del(ctx, failKey) op := WikiPendingOp{ Op: WikiOpIngest, KnowledgeID: knowledgeID, @@ -251,7 +260,7 @@ func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisC t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches + asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), ) @@ -291,7 +300,7 @@ func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redis payloadBytes, _ := json.Marshal(ingestPayload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(25), // 25 × 15 s ≈ 6 min window; outlasts even large KB batches + asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly ) @@ -395,6 +404,9 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn s.redisClient.Expire(ctx, failKey, wikiPendingTTL) if count > wikiMaxFailRetries { logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) + // Drop-path cleanup: avoid leaking stale counters that can + // poison future manual re-ingest within TTL window. + s.redisClient.Del(ctx, failKey) continue } } @@ -425,7 +437,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn payloadBytes, _ := json.Marshal(retryPayload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(25), // match main enqueue budget + asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), ) diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 33ef7c28..828d51ef 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -34,7 +34,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn payloadBytes, _ := json.Marshal(payload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue("low"), - asynq.MaxRetry(25), // match main enqueue budget; outlasts large KB batches + asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then ) @@ -123,7 +123,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // the lock. If there are still pending ops, retry so we don't miss them // in case the active batch drained the list before we RPush'd. pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID - if n, _ := s.redisClient.LLen(ctx, pendingKey).Result(); n == 0 { + n, nErr := s.redisClient.LLen(ctx, pendingKey).Result() + if nErr != nil { + logger.Warnf(ctx, "wiki ingest: failed to read pending length during lock conflict for KB %s: %v", payload.KnowledgeBaseID, nErr) + logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) + return ErrWikiIngestConcurrent + } + if n == 0 { exitStatus = "active_lock_conflict_empty" logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending list empty — skipping", payload.KnowledgeBaseID) return nil