From 60f7f35e40f393a19d107b1cd0896f82a4f55d37 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Jul 2026 16:07:03 +0800 Subject: [PATCH] fix(wiki): recover stranded claims and prevent finalize double-run - claim/peekPendingList now surface DB errors so a transient load failure retries via asynq instead of acking as a false "no pending ops" success that strands the queue. - no_pending_ops now arms a coalesced stale-claim recheck (TaskID keyed, delayed past wikiClaimStaleAfter) so rows held by a crashed batch's fresh claim are reclaimed automatically instead of sitting unclaimable and never re-triggered. - ProcessWikiFinalize fails CLOSED on a SetNX error (returns an error to retry) instead of running unlocked, avoiding two finalize runs double-rebuilding the index page. --- internal/application/service/wiki_ingest.go | 71 ++++++++++++++++--- .../application/service/wiki_ingest_batch.go | 30 +++++++- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index c5ae1e0c..c7ca9a10 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -626,19 +626,23 @@ func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIn // matches the legacy "LTrim peekedCount entries" semantics, where // duplicates collapsed by the consumer were also drained from the // list once their canonical sibling had been processed. -func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) { +func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) { if s.pendingRepo == nil { - return nil, nil + return nil, nil, nil } if limit <= 0 { limit = wikiMaxDocsPerBatch } rows, err := s.pendingRepo.PeekBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit) if err != nil { - logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err) - return nil, nil + // Surface the error so the caller can distinguish a transient DB + // failure from a genuinely empty queue: the former must trigger an + // asynq retry, whereas returning "no rows" here would ack the task + // as a false success and strand the pending list. + return nil, nil, err } - return s.decodePendingRows(ctx, rows) + ops, peekedIDs = s.decodePendingRows(ctx, rows) + return ops, peekedIDs, nil } // claimPendingList is the standard-mode (Redis) counterpart of @@ -649,9 +653,9 @@ func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, li // a crashed worker) are recovered. Dedup / peekedIDs semantics match // peekPendingList; the returned peekedIDs are the claimed rows that the // caller must DeleteByIDs on success or ReleaseByIDs to retry. -func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) { +func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) { if s.pendingRepo == nil { - return nil, nil + return nil, nil, nil } if limit <= 0 { limit = wikiMaxDocsPerBatch @@ -659,10 +663,13 @@ func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, l rows, err := s.pendingRepo.ClaimBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit, time.Now().Add(-wikiClaimStaleAfter)) if err != nil { - logger.Warnf(ctx, "wiki ingest: failed to claim pending list: %v", err) - return nil, nil + // A claim failure is transient (DB blip). Propagate it so the batch + // returns an error and asynq retries, instead of acking the trigger + // as a false "no pending ops" success and stranding the queue. + return nil, nil, err } - return s.decodePendingRows(ctx, rows) + ops, peekedIDs = s.decodePendingRows(ctx, rows) + return ops, peekedIDs, nil } // withSlugLock serializes read-modify-write on one shared wiki page across @@ -804,6 +811,50 @@ func (s *wikiIngestService) scheduleCappedRetry(ctx context.Context, payload Wik } } +// scheduleStaleClaimRecheck arms a single, far-future safety-net trigger for a +// KB that still has pending rows but yielded nothing to claim (every eligible +// row is held by a FRESH claim). Normally a running batch drains those rows and +// chains its own fast follow-up on completion; this net exists only for the +// case where the claim holder CRASHED, leaving claimed_at stamped so no worker +// can re-claim until wikiClaimStaleAfter elapses — and where nothing else would +// ever re-trigger the KB afterwards. +// +// The delay is set past the stale threshold so that when the net fires the +// abandoned claims are guaranteed eligible again. asynq.TaskID coalesces all +// rechecks for one KB into a single pending net (no thundering herd from +// concurrent no-op batches). If PendingCount is already zero the KB has fully +// drained and no net is needed. Returns true if a net is (or already was) +// scheduled. +func (s *wikiIngestService) scheduleStaleClaimRecheck(ctx context.Context, payload WikiIngestPayload) bool { + if s.pendingRepo == nil { + return false + } + count, err := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) + if err != nil || count == 0 { + return false + } + + logger.Infof(ctx, "wiki ingest: %d rows for KB %s held by fresh claims, arming stale-claim recheck", count, payload.KnowledgeBaseID) + + langfuse.InjectTracing(ctx, &payload) + b, _ := json.Marshal(payload) + t := asynq.NewTask(types.TypeWikiIngest, b, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(60*time.Minute), + asynq.ProcessIn(wikiClaimStaleAfter+wikiFollowUpDelay), + asynq.TaskID("wiki-ingest-recheck-"+payload.KnowledgeBaseID), + ) + if _, err := s.task.Enqueue(t); err != nil { + if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) { + return true // a recheck is already armed for this KB — coalesced + } + logger.Warnf(ctx, "wiki ingest: stale-claim recheck enqueue failed: %v", err) + return false + } + return true +} + // decodePendingRows converts raw task_pending_ops rows into WikiPendingOps, // applying the last-write-wins dedup by knowledge_id. peekedIDs carries the // db ids of EVERY row (including dedup-collapsed ones) so the caller can diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 349d5b96..e0a6014a 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -329,15 +329,33 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // batches); Lite mode peeks under its in-process lock. var pendingOps []WikiPendingOp var peekedIDs []int64 + var loadErr error if s.redisClient != nil { - pendingOps, peekedIDs = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize) + pendingOps, peekedIDs, loadErr = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize) } else { - pendingOps, peekedIDs = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) + pendingOps, peekedIDs, loadErr = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) + } + if loadErr != nil { + // Transient failure loading the pending list. Return an error so + // asynq retries this trigger with backoff — acking here would strand + // the queue until an unrelated upload happened to re-trigger it. + exitStatus = "load_pending_failed" + logger.Warnf(ctx, "wiki ingest: failed to load pending list for KB %s: %v", payload.KnowledgeBaseID, loadErr) + return fmt.Errorf("wiki ingest: load pending list: %w", loadErr) } pendingOpsCount = len(pendingOps) if len(pendingOps) == 0 { exitStatus = "no_pending_ops" logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) + // We claimed nothing, but rows may still exist held by a FRESH claim + // (a concurrent batch that is still running, or one that crashed + // mid-flight and left claimed_at stamped). This no-op return does not + // chain a follow-up, so without a safety net a crashed batch's rows + // would sit unclaimable until wikiClaimStaleAfter AND never get + // re-triggered afterwards — stranding the KB indefinitely. Schedule a + // coalesced recheck timed past the stale threshold so those rows are + // reclaimed automatically once eligible. + followUpScheduled = s.scheduleStaleClaimRecheck(ctx, payload) return nil } @@ -875,7 +893,13 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta key := "wiki:finalize:active:" + payload.KnowledgeBaseID acquired, err := s.redisClient.SetNX(ctx, key, "1", wikiFinalizeLockTTL).Result() if err != nil { - logger.Warnf(ctx, "wiki finalize: SetNX failed: %v", err) + // Fail CLOSED: proceeding unlocked would let two finalize runs + // drain the same PeekBatch rows and double-rebuild the index page + // (GetPage→modify→UpdatePage lost update). Return an error so + // asynq retries the (coalesced) finalize once Redis recovers, + // instead of racing a concurrent finalize. + logger.Warnf(ctx, "wiki finalize: SetNX failed for KB %s: %v (retrying)", payload.KnowledgeBaseID, err) + return fmt.Errorf("wiki finalize: acquire lock: %w", err) } else if !acquired { // Another finalize is running; it will drain the lane and // reschedule if rows remain. Safe to no-op.