fix(wiki): prevent silent work loss and cross-batch doc races

Two correctness fixes on the Phase 3 concurrent claim-based ingest path:

- Slug-lock timeout / reduce failure no longer silently drops work. When a
  per-slug lock can't be acquired within wikiSlugLockWait, or reduce returns
  an error, the page's read-modify-write never lands. Previously the owning
  document was still counted as successful and its pending row was deleted at
  trim time, permanently losing the contribution (finalize only rebuilds the
  index / cross-links, it does not re-run reduce). Such documents are now
  folded into failedOps: their rows stay queued, their finalizing slot is
  held, and requeueFailedOps' fail_count budget retries them (dead-lettering
  a permanently hot/broken slug).

- ClaimBatch now claims by dedup_key affinity so a document is never split
  across concurrent batches. limit counts distinct dedup_keys (documents),
  and ALL eligible rows sharing a key are claimed together. On Postgres the
  per-key anchor row is locked with FOR UPDATE SKIP LOCKED, handing
  concurrent claimers disjoint key sets. This restores in-document ordering
  (e.g. ingest-then-retract collapses correctly) that the per-batch dedup
  could not guarantee once the exclusive per-KB lock was removed, preventing
  a stale ingest from resurrecting a retracted document.

Adds repository tests for same-key affinity and in-flight sibling rows.
This commit is contained in:
wizardchen
2026-07-09 15:25:05 +08:00
committed by lyingbug
parent 70e2e85fa6
commit c8a4fd5ba3
4 changed files with 208 additions and 29 deletions
@@ -512,6 +512,27 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// slugs out of the wiki log feed so users don't see clickable entries
// pointing at missing pages.
failedAdditionSlugs := make(map[string]struct{})
// unappliedSlugKIDs collects the knowledge_ids that contributed to a
// slug whose update never landed — either because we could NOT acquire
// the per-slug lock within wikiSlugLockWait, or because reduce returned
// an error. In both cases the page keeps its prior content, so the
// owning document(s) must be re-queued rather than trimmed — otherwise
// the row is deleted and the contribution is silently lost forever
// (finalize only rebuilds the index / cross-links, it does not re-run
// reduce). requeueFailedOps' fail_count budget bounds the retries and
// dead-letters a document whose slug fails/stays hot permanently.
unappliedSlugKIDs := make(map[string]struct{})
// collectUnapplied records every knowledge_id backing a slug we failed
// to apply. Caller holds no lock; it takes reduceMu itself.
collectUnapplied := func(updates []SlugUpdate) {
reduceMu.Lock()
for _, u := range updates {
if u.KnowledgeID != "" {
unappliedSlugKIDs[u.KnowledgeID] = struct{}{}
}
}
reduceMu.Unlock()
}
// Build the kid → wikiSpan lookup before kicking off reduce. Each
// per-slug reduce attaches a postprocess.wiki.page[slug] subspan
@@ -546,14 +567,22 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
return nil
}
if !acquired {
// Contended slug we couldn't get in time. Best-effort skip,
// matching reduce-failure semantics: the page keeps its prior
// content and the finalize/dead-link passes reconcile later.
// Contended slug we couldn't get in time. The page keeps its
// prior content, so the documents that fed this slug are NOT
// done: record their knowledge_ids so the trim phase re-queues
// them (via the failed-op retry budget) for a later,
// hopefully-uncontended batch instead of deleting their rows.
logger.Warnf(reduceCtx, "wiki ingest: slug %s busy > %s, deferring update", slug, wikiSlugLockWait)
collectUnapplied(updates)
return nil
}
if reduceErr != nil {
// The page's read-modify-write failed, so this slug's update
// never landed. Re-queue the contributing documents (same
// fail_count budget as a map failure) so a later batch retries
// instead of silently dropping the row at trim time.
logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, reduceErr)
collectUnapplied(updates)
if isLikelyRateLimitError(reduceErr) {
reduceMu.Lock()
rateLimited = true
@@ -712,7 +741,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// the WikiSpan nil-check below so a doc that had no attempt to
// attach a span to still drains its counter slot. The matching +1
// is seeded by KnowledgePostProcess.SetFinalizing.
s.finalizeWikiSubtask(ctx, r.KnowledgeID)
//
// EXCEPT docs with an unapplied slug (contended lock or reduce
// error): those are re-queued below, so keep their finalizing slot
// held — the retry (or the dead-letter drain in requeueFailedOps)
// releases it once the op reaches a real terminal state.
if _, unapplied := unappliedSlugKIDs[r.KnowledgeID]; !unapplied {
s.finalizeWikiSubtask(ctx, r.KnowledgeID)
}
if r.WikiSpan == nil {
continue
}
@@ -748,6 +784,29 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// mapOneDocument (the failedOps path returns before reaching
// docResults). Nothing extra to do here for them.
// Fold documents with an unapplied slug (contended lock or reduce error)
// into failedOps so they are neither trimmed nor promoted to completed:
// requeueFailedOps then runs them through the same fail_count budget
// (retry now, dead-letter once the slug stays permanently hot/broken) as
// a map-phase failure. A doc already counted as a map failure is skipped
// to avoid a double fail_count bump.
if len(unappliedSlugKIDs) > 0 {
failedKIDs := make(map[string]struct{}, len(failedOps))
for _, op := range failedOps {
failedKIDs[op.KnowledgeID] = struct{}{}
}
for _, op := range pendingOps {
if _, unapplied := unappliedSlugKIDs[op.KnowledgeID]; !unapplied {
continue
}
if _, already := failedKIDs[op.KnowledgeID]; already {
continue
}
failedOps = append(failedOps, op)
failedKIDs[op.KnowledgeID] = struct{}{}
}
}
// Build the trim set: rows that should be removed from
// task_pending_ops. We start from the full peekedIDs (every row we
// pulled, even ones de-duplicated by knowledge_id) and subtract