fix(wiki): recover stranded claims and prevent finalize double-run

- claim/peekPendingList now surface DB errors so a transient load
  failure retries via asynq instead of acking as a false "no pending
  ops" success that strands the queue.
- no_pending_ops now arms a coalesced stale-claim recheck (TaskID
  keyed, delayed past wikiClaimStaleAfter) so rows held by a crashed
  batch's fresh claim are reclaimed automatically instead of sitting
  unclaimable and never re-triggered.
- ProcessWikiFinalize fails CLOSED on a SetNX error (returns an error
  to retry) instead of running unlocked, avoiding two finalize runs
  double-rebuilding the index page.
This commit is contained in:
wizardchen
2026-07-09 16:07:03 +08:00
committed by lyingbug
parent c8a4fd5ba3
commit 60f7f35e40
2 changed files with 88 additions and 13 deletions
+61 -10
View File
@@ -626,19 +626,23 @@ func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIn
// matches the legacy "LTrim peekedCount entries" semantics, where
// duplicates collapsed by the consumer were also drained from the
// list once their canonical sibling had been processed.
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) {
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) {
if s.pendingRepo == nil {
return nil, nil
return nil, nil, nil
}
if limit <= 0 {
limit = wikiMaxDocsPerBatch
}
rows, err := s.pendingRepo.PeekBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit)
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err)
return nil, nil
// Surface the error so the caller can distinguish a transient DB
// failure from a genuinely empty queue: the former must trigger an
// asynq retry, whereas returning "no rows" here would ack the task
// as a false success and strand the pending list.
return nil, nil, err
}
return s.decodePendingRows(ctx, rows)
ops, peekedIDs = s.decodePendingRows(ctx, rows)
return ops, peekedIDs, nil
}
// claimPendingList is the standard-mode (Redis) counterpart of
@@ -649,9 +653,9 @@ func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, li
// a crashed worker) are recovered. Dedup / peekedIDs semantics match
// peekPendingList; the returned peekedIDs are the claimed rows that the
// caller must DeleteByIDs on success or ReleaseByIDs to retry.
func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) {
func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) {
if s.pendingRepo == nil {
return nil, nil
return nil, nil, nil
}
if limit <= 0 {
limit = wikiMaxDocsPerBatch
@@ -659,10 +663,13 @@ func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, l
rows, err := s.pendingRepo.ClaimBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit,
time.Now().Add(-wikiClaimStaleAfter))
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to claim pending list: %v", err)
return nil, nil
// A claim failure is transient (DB blip). Propagate it so the batch
// returns an error and asynq retries, instead of acking the trigger
// as a false "no pending ops" success and stranding the queue.
return nil, nil, err
}
return s.decodePendingRows(ctx, rows)
ops, peekedIDs = s.decodePendingRows(ctx, rows)
return ops, peekedIDs, nil
}
// withSlugLock serializes read-modify-write on one shared wiki page across
@@ -804,6 +811,50 @@ func (s *wikiIngestService) scheduleCappedRetry(ctx context.Context, payload Wik
}
}
// scheduleStaleClaimRecheck arms a single, far-future safety-net trigger for a
// KB that still has pending rows but yielded nothing to claim (every eligible
// row is held by a FRESH claim). Normally a running batch drains those rows and
// chains its own fast follow-up on completion; this net exists only for the
// case where the claim holder CRASHED, leaving claimed_at stamped so no worker
// can re-claim until wikiClaimStaleAfter elapses — and where nothing else would
// ever re-trigger the KB afterwards.
//
// The delay is set past the stale threshold so that when the net fires the
// abandoned claims are guaranteed eligible again. asynq.TaskID coalesces all
// rechecks for one KB into a single pending net (no thundering herd from
// concurrent no-op batches). If PendingCount is already zero the KB has fully
// drained and no net is needed. Returns true if a net is (or already was)
// scheduled.
func (s *wikiIngestService) scheduleStaleClaimRecheck(ctx context.Context, payload WikiIngestPayload) bool {
if s.pendingRepo == nil {
return false
}
count, err := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID)
if err != nil || count == 0 {
return false
}
logger.Infof(ctx, "wiki ingest: %d rows for KB %s held by fresh claims, arming stale-claim recheck", count, payload.KnowledgeBaseID)
langfuse.InjectTracing(ctx, &payload)
b, _ := json.Marshal(payload)
t := asynq.NewTask(types.TypeWikiIngest, b,
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiClaimStaleAfter+wikiFollowUpDelay),
asynq.TaskID("wiki-ingest-recheck-"+payload.KnowledgeBaseID),
)
if _, err := s.task.Enqueue(t); err != nil {
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
return true // a recheck is already armed for this KB — coalesced
}
logger.Warnf(ctx, "wiki ingest: stale-claim recheck enqueue failed: %v", err)
return false
}
return true
}
// decodePendingRows converts raw task_pending_ops rows into WikiPendingOps,
// applying the last-write-wins dedup by knowledge_id. peekedIDs carries the
// db ids of EVERY row (including dedup-collapsed ones) so the caller can
@@ -329,15 +329,33 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// batches); Lite mode peeks under its in-process lock.
var pendingOps []WikiPendingOp
var peekedIDs []int64
var loadErr error
if s.redisClient != nil {
pendingOps, peekedIDs = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize)
pendingOps, peekedIDs, loadErr = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize)
} else {
pendingOps, peekedIDs = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize)
pendingOps, peekedIDs, loadErr = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize)
}
if loadErr != nil {
// Transient failure loading the pending list. Return an error so
// asynq retries this trigger with backoff — acking here would strand
// the queue until an unrelated upload happened to re-trigger it.
exitStatus = "load_pending_failed"
logger.Warnf(ctx, "wiki ingest: failed to load pending list for KB %s: %v", payload.KnowledgeBaseID, loadErr)
return fmt.Errorf("wiki ingest: load pending list: %w", loadErr)
}
pendingOpsCount = len(pendingOps)
if len(pendingOps) == 0 {
exitStatus = "no_pending_ops"
logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID)
// We claimed nothing, but rows may still exist held by a FRESH claim
// (a concurrent batch that is still running, or one that crashed
// mid-flight and left claimed_at stamped). This no-op return does not
// chain a follow-up, so without a safety net a crashed batch's rows
// would sit unclaimable until wikiClaimStaleAfter AND never get
// re-triggered afterwards — stranding the KB indefinitely. Schedule a
// coalesced recheck timed past the stale threshold so those rows are
// reclaimed automatically once eligible.
followUpScheduled = s.scheduleStaleClaimRecheck(ctx, payload)
return nil
}
@@ -875,7 +893,13 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
key := "wiki:finalize:active:" + payload.KnowledgeBaseID
acquired, err := s.redisClient.SetNX(ctx, key, "1", wikiFinalizeLockTTL).Result()
if err != nil {
logger.Warnf(ctx, "wiki finalize: SetNX failed: %v", err)
// Fail CLOSED: proceeding unlocked would let two finalize runs
// drain the same PeekBatch rows and double-rebuild the index page
// (GetPage→modify→UpdatePage lost update). Return an error so
// asynq retries the (coalesced) finalize once Redis recovers,
// instead of racing a concurrent finalize.
logger.Warnf(ctx, "wiki finalize: SetNX failed for KB %s: %v (retrying)", payload.KnowledgeBaseID, err)
return fmt.Errorf("wiki finalize: acquire lock: %w", err)
} else if !acquired {
// Another finalize is running; it will drain the lane and
// reschedule if rows remain. Safe to no-op.