From bf128689232b03caefca0c7e37ad43b17851fece Mon Sep 17 00:00:00 2001 From: toy0116 Date: Thu, 7 May 2026 17:50:21 +0800 Subject: [PATCH] fix(wiki): cap requeueFailedOps retry count to prevent queue pile-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: requeueFailedOps unconditionally RPush'd every failed op back to wiki:pending, causing unbounded growth when a document consistently triggers LLM timeouts. Observed: 553 entries in wiki:pending:08134644 (one KB), mostly duplicates for ~5 unique documents that each timed out on every batch cycle. Fix: - Add wikiFailCountKeyPrefix ("wiki:failcount:") and wikiMaxFailRetries (5) constants in wiki_ingest.go - requeueFailedOps now atomically Incr(wiki:failcount:{kbID}:{knowledgeID}) before each RPush; skips re-queue (drops op permanently) if count > 5 - Expire the fail-count key at wikiPendingTTL (24h) on every update - wiki_ingest_batch.go: Del the fail-count key at ingestSucceeded++ so transient errors don't permanently burn through a document's retry budget Evidence of problem severity: 5 persistently-failing docs × ~110 requeue cycles each = 553 pending entries; had to use Lua atomic dedup to recover. Behaviour after this fix: - First 5 failures → normal retry (re-queued, follow-up batch scheduled) - 6th failure → logged as WARN "dropping op … after 6 failures (limit 5)", not re-queued; prevents the feedback loop - If document is fixed upstream and re-ingested via EnqueueWikiIngest, the op is a fresh entry with no existing fail-count key → full 5-retry budget Co-Authored-By: Claude Sonnet 4.6 --- internal/application/service/wiki_ingest.go | 34 ++++++++++++++++++- .../application/service/wiki_ingest_batch.go | 7 ++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 258b82f3..a55f587a 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -57,6 +57,21 @@ const ( // and are picked up by the follow-up task. wikiMaxDocsPerBatch = 5 + // wikiFailCountKeyPrefix is the Redis key prefix for per-document failure + // counters. Key: wiki:failcount:{kbID}:{knowledgeID} → integer. + // Incremented each time requeueFailedOps retries an op; reset to 0 on + // successful ingest. Once the counter exceeds wikiMaxFailRetries the op is + // dropped instead of re-queued, preventing LLM-timeout storms from causing + // unbounded wiki:pending growth. + wikiFailCountKeyPrefix = "wiki:failcount:" + + // wikiMaxFailRetries is the maximum number of times a single document op + // may be re-queued via requeueFailedOps before it is permanently dropped. + // 5 retries ≈ five full batch cycles (each with a ~30 s delay), giving + // transient LLM errors a fair chance to recover without letting a + // persistently-broken doc clog the queue indefinitely. + wikiMaxFailRetries = 5 + // wikiDeletedKeyPrefix is the Redis key prefix for "recently deleted // knowledge" tombstones. Key: wiki:deleted:{kbID}:{knowledgeID}. Written // by cleanupWikiOnKnowledgeDelete so that any wiki_ingest task still in @@ -353,6 +368,23 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn if s.redisClient != nil { pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID for _, op := range ops { + // Increment failure counter; drop the op permanently once it + // exceeds wikiMaxFailRetries to prevent unbounded queue growth + // caused by persistent LLM timeouts or extraction errors. + failKey := wikiFailCountKeyPrefix + payload.KnowledgeBaseID + ":" + op.KnowledgeID + count, err := s.redisClient.Incr(ctx, failKey).Result() + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s: %v", op.KnowledgeID, err) + // Fall through and requeue anyway — better to retry than silently drop + } else { + // Refresh TTL on every update so the key doesn't outlast its usefulness + s.redisClient.Expire(ctx, failKey, wikiPendingTTL) + if count > wikiMaxFailRetries { + logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) + continue + } + } + data, err := json.Marshal(op) if err != nil { logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err) @@ -362,7 +394,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err) continue } - logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle) + logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) } return } diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 0db76eaa..7036d3f6 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -345,6 +345,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task slugUpdates[u.Slug] = append(slugUpdates[u.Slug], u) } mapMu.Unlock() + + // Reset the per-document failure counter on success so transient + // errors don't permanently exhaust the retry budget. + if s.redisClient != nil { + failKey := wikiFailCountKeyPrefix + payload.KnowledgeBaseID + ":" + op.KnowledgeID + s.redisClient.Del(mapCtx, failKey) + } } return nil })