fix(wiki): recover stranded claims and prevent finalize double-run
- claim/peekPendingList now surface DB errors so a transient load failure retries via asynq instead of acking as a false "no pending ops" success that strands the queue. - no_pending_ops now arms a coalesced stale-claim recheck (TaskID keyed, delayed past wikiClaimStaleAfter) so rows held by a crashed batch's fresh claim are reclaimed automatically instead of sitting unclaimable and never re-triggered. - ProcessWikiFinalize fails CLOSED on a SetNX error (returns an error to retry) instead of running unlocked, avoiding two finalize runs double-rebuilding the index page.
This commit is contained in:
@@ -626,19 +626,23 @@ func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIn
|
||||
// matches the legacy "LTrim peekedCount entries" semantics, where
|
||||
// duplicates collapsed by the consumer were also drained from the
|
||||
// list once their canonical sibling had been processed.
|
||||
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) {
|
||||
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) {
|
||||
if s.pendingRepo == nil {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = wikiMaxDocsPerBatch
|
||||
}
|
||||
rows, err := s.pendingRepo.PeekBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err)
|
||||
return nil, nil
|
||||
// Surface the error so the caller can distinguish a transient DB
|
||||
// failure from a genuinely empty queue: the former must trigger an
|
||||
// asynq retry, whereas returning "no rows" here would ack the task
|
||||
// as a false success and strand the pending list.
|
||||
return nil, nil, err
|
||||
}
|
||||
return s.decodePendingRows(ctx, rows)
|
||||
ops, peekedIDs = s.decodePendingRows(ctx, rows)
|
||||
return ops, peekedIDs, nil
|
||||
}
|
||||
|
||||
// claimPendingList is the standard-mode (Redis) counterpart of
|
||||
@@ -649,9 +653,9 @@ func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, li
|
||||
// a crashed worker) are recovered. Dedup / peekedIDs semantics match
|
||||
// peekPendingList; the returned peekedIDs are the claimed rows that the
|
||||
// caller must DeleteByIDs on success or ReleaseByIDs to retry.
|
||||
func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) {
|
||||
func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64, err error) {
|
||||
if s.pendingRepo == nil {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = wikiMaxDocsPerBatch
|
||||
@@ -659,10 +663,13 @@ func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, l
|
||||
rows, err := s.pendingRepo.ClaimBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit,
|
||||
time.Now().Add(-wikiClaimStaleAfter))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to claim pending list: %v", err)
|
||||
return nil, nil
|
||||
// A claim failure is transient (DB blip). Propagate it so the batch
|
||||
// returns an error and asynq retries, instead of acking the trigger
|
||||
// as a false "no pending ops" success and stranding the queue.
|
||||
return nil, nil, err
|
||||
}
|
||||
return s.decodePendingRows(ctx, rows)
|
||||
ops, peekedIDs = s.decodePendingRows(ctx, rows)
|
||||
return ops, peekedIDs, nil
|
||||
}
|
||||
|
||||
// withSlugLock serializes read-modify-write on one shared wiki page across
|
||||
@@ -804,6 +811,50 @@ func (s *wikiIngestService) scheduleCappedRetry(ctx context.Context, payload Wik
|
||||
}
|
||||
}
|
||||
|
||||
// scheduleStaleClaimRecheck arms a single, far-future safety-net trigger for a
|
||||
// KB that still has pending rows but yielded nothing to claim (every eligible
|
||||
// row is held by a FRESH claim). Normally a running batch drains those rows and
|
||||
// chains its own fast follow-up on completion; this net exists only for the
|
||||
// case where the claim holder CRASHED, leaving claimed_at stamped so no worker
|
||||
// can re-claim until wikiClaimStaleAfter elapses — and where nothing else would
|
||||
// ever re-trigger the KB afterwards.
|
||||
//
|
||||
// The delay is set past the stale threshold so that when the net fires the
|
||||
// abandoned claims are guaranteed eligible again. asynq.TaskID coalesces all
|
||||
// rechecks for one KB into a single pending net (no thundering herd from
|
||||
// concurrent no-op batches). If PendingCount is already zero the KB has fully
|
||||
// drained and no net is needed. Returns true if a net is (or already was)
|
||||
// scheduled.
|
||||
func (s *wikiIngestService) scheduleStaleClaimRecheck(ctx context.Context, payload WikiIngestPayload) bool {
|
||||
if s.pendingRepo == nil {
|
||||
return false
|
||||
}
|
||||
count, err := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID)
|
||||
if err != nil || count == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: %d rows for KB %s held by fresh claims, arming stale-claim recheck", count, payload.KnowledgeBaseID)
|
||||
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
b, _ := json.Marshal(payload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, b,
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(wikiClaimStaleAfter+wikiFollowUpDelay),
|
||||
asynq.TaskID("wiki-ingest-recheck-"+payload.KnowledgeBaseID),
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
|
||||
return true // a recheck is already armed for this KB — coalesced
|
||||
}
|
||||
logger.Warnf(ctx, "wiki ingest: stale-claim recheck enqueue failed: %v", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// decodePendingRows converts raw task_pending_ops rows into WikiPendingOps,
|
||||
// applying the last-write-wins dedup by knowledge_id. peekedIDs carries the
|
||||
// db ids of EVERY row (including dedup-collapsed ones) so the caller can
|
||||
|
||||
@@ -329,15 +329,33 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// batches); Lite mode peeks under its in-process lock.
|
||||
var pendingOps []WikiPendingOp
|
||||
var peekedIDs []int64
|
||||
var loadErr error
|
||||
if s.redisClient != nil {
|
||||
pendingOps, peekedIDs = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize)
|
||||
pendingOps, peekedIDs, loadErr = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize)
|
||||
} else {
|
||||
pendingOps, peekedIDs = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize)
|
||||
pendingOps, peekedIDs, loadErr = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize)
|
||||
}
|
||||
if loadErr != nil {
|
||||
// Transient failure loading the pending list. Return an error so
|
||||
// asynq retries this trigger with backoff — acking here would strand
|
||||
// the queue until an unrelated upload happened to re-trigger it.
|
||||
exitStatus = "load_pending_failed"
|
||||
logger.Warnf(ctx, "wiki ingest: failed to load pending list for KB %s: %v", payload.KnowledgeBaseID, loadErr)
|
||||
return fmt.Errorf("wiki ingest: load pending list: %w", loadErr)
|
||||
}
|
||||
pendingOpsCount = len(pendingOps)
|
||||
if len(pendingOps) == 0 {
|
||||
exitStatus = "no_pending_ops"
|
||||
logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID)
|
||||
// We claimed nothing, but rows may still exist held by a FRESH claim
|
||||
// (a concurrent batch that is still running, or one that crashed
|
||||
// mid-flight and left claimed_at stamped). This no-op return does not
|
||||
// chain a follow-up, so without a safety net a crashed batch's rows
|
||||
// would sit unclaimable until wikiClaimStaleAfter AND never get
|
||||
// re-triggered afterwards — stranding the KB indefinitely. Schedule a
|
||||
// coalesced recheck timed past the stale threshold so those rows are
|
||||
// reclaimed automatically once eligible.
|
||||
followUpScheduled = s.scheduleStaleClaimRecheck(ctx, payload)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -875,7 +893,13 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
key := "wiki:finalize:active:" + payload.KnowledgeBaseID
|
||||
acquired, err := s.redisClient.SetNX(ctx, key, "1", wikiFinalizeLockTTL).Result()
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: SetNX failed: %v", err)
|
||||
// Fail CLOSED: proceeding unlocked would let two finalize runs
|
||||
// drain the same PeekBatch rows and double-rebuild the index page
|
||||
// (GetPage→modify→UpdatePage lost update). Return an error so
|
||||
// asynq retries the (coalesced) finalize once Redis recovers,
|
||||
// instead of racing a concurrent finalize.
|
||||
logger.Warnf(ctx, "wiki finalize: SetNX failed for KB %s: %v (retrying)", payload.KnowledgeBaseID, err)
|
||||
return fmt.Errorf("wiki finalize: acquire lock: %w", err)
|
||||
} else if !acquired {
|
||||
// Another finalize is running; it will drain the lane and
|
||||
// reschedule if rows remain. Safe to no-op.
|
||||
|
||||
Reference in New Issue
Block a user