diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index 1d8848ff..328bd533 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -81,8 +81,17 @@ func (r *taskPendingOpsRepository) PeekBatch( // otherwise each batch's per-batch last-write-wins dedup can't collapse the // pair and the two ops race (a stale ingest could resurrect a retracted doc). // +// To uphold that invariant even for a sibling enqueued AFTER a batch already +// claimed the key (e.g. a retract arriving while the ingest is still in +// flight), a dedup_key that has ANY fresh claim (claimed_at >= staleBefore) is +// skipped ENTIRELY — not just its already-claimed rows. The late sibling waits +// for the holder to finish (which deletes the claimed rows, freeing the key) or +// for the claim to go stale. This serializes same-document ops across +// concurrent batches instead of letting them race on wall-clock completion. +// // Eligibility = unclaimed (claimed_at IS NULL) OR stale claim -// (claimed_at < staleBefore). The whole thing runs in one transaction: +// (claimed_at < staleBefore), AND the key has no fresh claim. The whole thing +// runs in one transaction: // // - Postgres: we lock the ANCHOR row (earliest eligible id) of each // candidate dedup_key with FOR UPDATE SKIP LOCKED. Because the anchor @@ -113,11 +122,14 @@ func (r *taskPendingOpsRepository) ClaimBatch( var claimed []*types.TaskPendingOp err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // 1. Pick up to `limit` distinct dedup_keys to claim, oldest first. + // Keys with a fresh claim are excluded WHOLESALE so a late sibling + // of an in-flight document never gets claimed on its own. var keys []string if tx.Dialector.Name() == "postgres" { // Lock the anchor (earliest eligible) row of each key with SKIP // LOCKED so concurrent claimers get disjoint KEY sets, then map - // the locked anchors back to their dedup_keys. + // the locked anchors back to their dedup_keys. The NOT IN subquery + // drops any key that still has a fresh (non-stale) claim. const anchorSQL = ` SELECT dedup_key FROM task_pending_ops WHERE id IN ( @@ -126,19 +138,32 @@ WHERE id IN ( FROM task_pending_ops WHERE task_type = ? AND scope = ? AND scope_id = ? AND (claimed_at IS NULL OR claimed_at < ?) + AND dedup_key NOT IN ( + SELECT dedup_key FROM task_pending_ops + WHERE task_type = ? AND scope = ? AND scope_id = ? + AND claimed_at IS NOT NULL AND claimed_at >= ? + ) ) anchors WHERE anchors.rn = 1 ) ORDER BY id ASC LIMIT ? FOR UPDATE SKIP LOCKED` - if err := tx.Raw(anchorSQL, taskType, scope, scopeID, staleBefore, limit). + if err := tx.Raw(anchorSQL, + taskType, scope, scopeID, staleBefore, + taskType, scope, scopeID, staleBefore, + limit). Scan(&keys).Error; err != nil { return err } } else { + freshKeys := tx.Model(&types.TaskPendingOp{}). + Select("dedup_key"). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore) if err := tx.Model(&types.TaskPendingOp{}). Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore). + Where("dedup_key NOT IN (?)", freshKeys). Group("dedup_key"). Order("MIN(id) ASC"). Limit(limit). diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index 9e633ad2..a3aece6c 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -347,10 +347,13 @@ func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) { assert.Equal(t, "k3", second[0].DedupKey) } -// TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed verifies a -// row enqueued for an already-claimed (in-flight) document is left for a -// later batch rather than being handed out alongside the fresh claim. -func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) { +// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a +// row enqueued for an already-claimed (in-flight) document is NOT claimed on +// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late +// sibling waits for the holder to finish (rows deleted → key freed) or for the +// claim to go stale, keeping same-document ops serialized across concurrent +// batches. +func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) { db := setupTaskQueueTestDB(t) repo := NewTaskPendingOpsRepository(db) ctx := context.Background() @@ -361,16 +364,51 @@ func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) { first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale) require.NoError(t, err) require.Len(t, first, 1) + ingestID := first[0].ID // A retract for the same, still-in-flight document arrives. require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil))) - // The next claim gets ONLY the new (unclaimed) retract row — the - // still-claimed ingest row is not re-handed-out. + // The retract must NOT be claimed while the ingest claim is fresh — + // the whole k1 key is blocked. second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale) require.NoError(t, err) - require.Len(t, second, 1) - assert.Equal(t, "retract", second[0].Op) + require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh") + + // Once the holder finishes (its claimed rows are deleted), the key is + // free and the retract becomes claimable. + require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID})) + third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale) + require.NoError(t, err) + require.Len(t, third, 1) + assert.Equal(t, "retract", third[0].Op) +} + +// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the +// other release path: if the holder CRASHES (claim never cleared), the whole +// key — original row + late sibling — becomes claimable together once the +// claim goes stale, so the pair is folded back into one batch. +func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil))) + first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour)) + require.NoError(t, err) + require.Len(t, first, 1) + + // Late retract arrives; holder then crashes (claim left stamped). + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil))) + + // A future stale threshold makes the crashed claim eligible again; both + // k1 rows are reclaimed together (never split). + got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour)) + require.NoError(t, err) + require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together") + for _, r := range got { + assert.Equal(t, "k1", r.DedupKey) + } } // TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index e0a6014a..c5bbf35a 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -361,6 +361,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) + // Crash/abort safety net (standard/claim mode only). If this batch exits + // abnormally — panic, ctx timeout, or an early error return — BEFORE it + // settles its claimed rows (trim + requeueFailedOps), release the claims + // so the next trigger can re-claim within seconds instead of waiting out + // wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true + // once the rows reach their terminal state, making this a no-op. Uses a + // background context because ctx may already be cancelled on the timeout + // path. Lite mode peeks without claiming, so there is nothing to release. + claimsSettled := false + if s.redisClient != nil && len(peekedIDs) > 0 { + defer func() { + if claimsSettled { + return + } + if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err) + return + } + logger.Warnf(ctx, "wiki ingest: released %d claimed rows on abnormal exit for KB %s (re-claimable immediately)", len(peekedIDs), payload.KnowledgeBaseID) + }() + } + // Resolve extraction granularity once per batch. Historical rows with // empty/unknown values fall back to Standard via Normalize(). Failures // to load the KB (unlikely since we're already acting on it) also @@ -852,6 +874,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task if len(failedOps) > 0 { s.requeueFailedOps(ctx, payload, failedOps) } + // All claimed rows have now reached a terminal state (deleted on success, + // released for retry, or dead-lettered), so disarm the abnormal-exit + // release net. + claimsSettled = true logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected)) diff --git a/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql b/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql index 81e7548b..d9cdbd52 100644 --- a/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql +++ b/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql @@ -39,12 +39,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm; -- consumer (e.g. wiki ingest batch handler) uses PeekBatch to pull the head -- of the list, processes the ops, then DeleteByIDs the consumed rows. -- --- claimed_at is intentionally added but unused in this revision: the wiki --- ingest pipeline guarantees per-KB serialization via its own active-batch --- lock (Redis SetNX wiki:active:), so PeekBatch doesn't need --- SELECT ... FOR UPDATE. Future task types that don't carry an external --- lock can flip claimed_at = NOW() inside a row-level lock and rely on the --- column to detect orphaned claims after a worker crash. +-- claimed_at backs the concurrent-claim workflow (ClaimBatch): standard-mode +-- wiki ingest dropped the exclusive per-KB lock in favour of multiple batches +-- claiming DISJOINT dedup_keys via SELECT ... FOR UPDATE SKIP LOCKED, stamping +-- claimed_at = NOW() on the rows they take. A claim older than the consumer's +-- stale threshold (a crashed/abandoned worker) is recoverable by the next +-- claimer; a fresh claim blocks its whole dedup_key so same-document ops never +-- split across concurrent batches. Lite mode (no Redis) still serializes per +-- KB in-process and leaves claimed_at NULL. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS task_pending_ops ( id BIGSERIAL PRIMARY KEY, @@ -65,7 +67,7 @@ COMMENT ON COLUMN task_pending_ops.task_type IS 'Free-form task identifier, e.g. COMMENT ON COLUMN task_pending_ops.scope IS 'Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". Read together with scope_id.'; COMMENT ON COLUMN task_pending_ops.dedup_key IS 'Optional service-defined key used by the consumer to de-duplicate equivalent ops within a single batch peek. Empty means no de-dup.'; COMMENT ON COLUMN task_pending_ops.fail_count IS 'In-batch retry counter: the consumer increments it via IncrFailCount and dead-letters once it exceeds a service-defined cap.'; -COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Reserved for future locking workflows; consumers in this revision rely on external mutual exclusion and ignore the column.'; +COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Concurrent-claim marker: set to NOW() when a consumer claims the row (SELECT ... FOR UPDATE SKIP LOCKED). NULL = unclaimed; a value older than the consumer stale threshold is a crashed/abandoned claim and is recoverable. A fresh claim blocks its whole dedup_key so same-document ops never split across concurrent batches.'; -- Cover the PeekBatch query: the consumer scans rows for one -- (task_type, scope, scope_id) tuple ordered by id ASC.