refactor(task_queue): enhance claim handling and test coverage

- Renamed TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed to TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim for clarity on behavior.
- Added new test TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale to verify claim behavior when a holder crashes.
- Updated ClaimBatch logic to ensure that fresh claims block late siblings entirely, maintaining serialization of same-document operations.
- Improved comments in task_queue.go to clarify the handling of concurrent claims and the implications of the claimed_at field.
- Added crash safety mechanism in ProcessWikiIngest to release claims on abnormal exits, ensuring timely re-claims of pending operations.
This commit is contained in:
wizardchen
2026-07-09 16:23:50 +08:00
committed by lyingbug
parent 60f7f35e40
commit 76a8092483
4 changed files with 109 additions and 18 deletions
+28 -3
View File
@@ -81,8 +81,17 @@ func (r *taskPendingOpsRepository) PeekBatch(
// otherwise each batch's per-batch last-write-wins dedup can't collapse the
// pair and the two ops race (a stale ingest could resurrect a retracted doc).
//
// To uphold that invariant even for a sibling enqueued AFTER a batch already
// claimed the key (e.g. a retract arriving while the ingest is still in
// flight), a dedup_key that has ANY fresh claim (claimed_at >= staleBefore) is
// skipped ENTIRELY — not just its already-claimed rows. The late sibling waits
// for the holder to finish (which deletes the claimed rows, freeing the key) or
// for the claim to go stale. This serializes same-document ops across
// concurrent batches instead of letting them race on wall-clock completion.
//
// Eligibility = unclaimed (claimed_at IS NULL) OR stale claim
// (claimed_at < staleBefore). The whole thing runs in one transaction:
// (claimed_at < staleBefore), AND the key has no fresh claim. The whole thing
// runs in one transaction:
//
// - Postgres: we lock the ANCHOR row (earliest eligible id) of each
// candidate dedup_key with FOR UPDATE SKIP LOCKED. Because the anchor
@@ -113,11 +122,14 @@ func (r *taskPendingOpsRepository) ClaimBatch(
var claimed []*types.TaskPendingOp
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// 1. Pick up to `limit` distinct dedup_keys to claim, oldest first.
// Keys with a fresh claim are excluded WHOLESALE so a late sibling
// of an in-flight document never gets claimed on its own.
var keys []string
if tx.Dialector.Name() == "postgres" {
// Lock the anchor (earliest eligible) row of each key with SKIP
// LOCKED so concurrent claimers get disjoint KEY sets, then map
// the locked anchors back to their dedup_keys.
// the locked anchors back to their dedup_keys. The NOT IN subquery
// drops any key that still has a fresh (non-stale) claim.
const anchorSQL = `
SELECT dedup_key FROM task_pending_ops
WHERE id IN (
@@ -126,19 +138,32 @@ WHERE id IN (
FROM task_pending_ops
WHERE task_type = ? AND scope = ? AND scope_id = ?
AND (claimed_at IS NULL OR claimed_at < ?)
AND dedup_key NOT IN (
SELECT dedup_key FROM task_pending_ops
WHERE task_type = ? AND scope = ? AND scope_id = ?
AND claimed_at IS NOT NULL AND claimed_at >= ?
)
) anchors WHERE anchors.rn = 1
)
ORDER BY id ASC
LIMIT ?
FOR UPDATE SKIP LOCKED`
if err := tx.Raw(anchorSQL, taskType, scope, scopeID, staleBefore, limit).
if err := tx.Raw(anchorSQL,
taskType, scope, scopeID, staleBefore,
taskType, scope, scopeID, staleBefore,
limit).
Scan(&keys).Error; err != nil {
return err
}
} else {
freshKeys := tx.Model(&types.TaskPendingOp{}).
Select("dedup_key").
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
if err := tx.Model(&types.TaskPendingOp{}).
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore).
Where("dedup_key NOT IN (?)", freshKeys).
Group("dedup_key").
Order("MIN(id) ASC").
Limit(limit).
@@ -347,10 +347,13 @@ func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) {
assert.Equal(t, "k3", second[0].DedupKey)
}
// TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed verifies a
// row enqueued for an already-claimed (in-flight) document is left for a
// later batch rather than being handed out alongside the fresh claim.
func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a
// row enqueued for an already-claimed (in-flight) document is NOT claimed on
// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late
// sibling waits for the holder to finish (rows deleted → key freed) or for the
// claim to go stale, keeping same-document ops serialized across concurrent
// batches.
func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
ctx := context.Background()
@@ -361,16 +364,51 @@ func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, first, 1)
ingestID := first[0].ID
// A retract for the same, still-in-flight document arrives.
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
// The next claim gets ONLY the new (unclaimed) retract row — the
// still-claimed ingest row is not re-handed-out.
// The retract must NOT be claimed while the ingest claim is fresh —
// the whole k1 key is blocked.
second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, second, 1)
assert.Equal(t, "retract", second[0].Op)
require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh")
// Once the holder finishes (its claimed rows are deleted), the key is
// free and the retract becomes claimable.
require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID}))
third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, third, 1)
assert.Equal(t, "retract", third[0].Op)
}
// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the
// other release path: if the holder CRASHES (claim never cleared), the whole
// key — original row + late sibling — becomes claimable together once the
// claim goes stale, so the pair is folded back into one batch.
func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
ctx := context.Background()
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour))
require.NoError(t, err)
require.Len(t, first, 1)
// Late retract arrives; holder then crashes (claim left stamped).
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
// A future stale threshold makes the crashed claim eligible again; both
// k1 rows are reclaimed together (never split).
got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour))
require.NoError(t, err)
require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together")
for _, r := range got {
assert.Equal(t, "k1", r.DedupKey)
}
}
// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than
@@ -361,6 +361,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
// Crash/abort safety net (standard/claim mode only). If this batch exits
// abnormally — panic, ctx timeout, or an early error return — BEFORE it
// settles its claimed rows (trim + requeueFailedOps), release the claims
// so the next trigger can re-claim within seconds instead of waiting out
// wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true
// once the rows reach their terminal state, making this a no-op. Uses a
// background context because ctx may already be cancelled on the timeout
// path. Lite mode peeks without claiming, so there is nothing to release.
claimsSettled := false
if s.redisClient != nil && len(peekedIDs) > 0 {
defer func() {
if claimsSettled {
return
}
if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err)
return
}
logger.Warnf(ctx, "wiki ingest: released %d claimed rows on abnormal exit for KB %s (re-claimable immediately)", len(peekedIDs), payload.KnowledgeBaseID)
}()
}
// Resolve extraction granularity once per batch. Historical rows with
// empty/unknown values fall back to Standard via Normalize(). Failures
// to load the KB (unlikely since we're already acting on it) also
@@ -852,6 +874,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
if len(failedOps) > 0 {
s.requeueFailedOps(ctx, payload, failedOps)
}
// All claimed rows have now reached a terminal state (deleted on success,
// released for retry, or dead-lettered), so disarm the abnormal-exit
// release net.
claimsSettled = true
logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
@@ -39,12 +39,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm;
-- consumer (e.g. wiki ingest batch handler) uses PeekBatch to pull the head
-- of the list, processes the ops, then DeleteByIDs the consumed rows.
--
-- claimed_at is intentionally added but unused in this revision: the wiki
-- ingest pipeline guarantees per-KB serialization via its own active-batch
-- lock (Redis SetNX wiki:active:<kbID>), so PeekBatch doesn't need
-- SELECT ... FOR UPDATE. Future task types that don't carry an external
-- lock can flip claimed_at = NOW() inside a row-level lock and rely on the
-- column to detect orphaned claims after a worker crash.
-- claimed_at backs the concurrent-claim workflow (ClaimBatch): standard-mode
-- wiki ingest dropped the exclusive per-KB lock in favour of multiple batches
-- claiming DISJOINT dedup_keys via SELECT ... FOR UPDATE SKIP LOCKED, stamping
-- claimed_at = NOW() on the rows they take. A claim older than the consumer's
-- stale threshold (a crashed/abandoned worker) is recoverable by the next
-- claimer; a fresh claim blocks its whole dedup_key so same-document ops never
-- split across concurrent batches. Lite mode (no Redis) still serializes per
-- KB in-process and leaves claimed_at NULL.
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS task_pending_ops (
id BIGSERIAL PRIMARY KEY,
@@ -65,7 +67,7 @@ COMMENT ON COLUMN task_pending_ops.task_type IS 'Free-form task identifier, e.g.
COMMENT ON COLUMN task_pending_ops.scope IS 'Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". Read together with scope_id.';
COMMENT ON COLUMN task_pending_ops.dedup_key IS 'Optional service-defined key used by the consumer to de-duplicate equivalent ops within a single batch peek. Empty means no de-dup.';
COMMENT ON COLUMN task_pending_ops.fail_count IS 'In-batch retry counter: the consumer increments it via IncrFailCount and dead-letters once it exceeds a service-defined cap.';
COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Reserved for future locking workflows; consumers in this revision rely on external mutual exclusion and ignore the column.';
COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Concurrent-claim marker: set to NOW() when a consumer claims the row (SELECT ... FOR UPDATE SKIP LOCKED). NULL = unclaimed; a value older than the consumer stale threshold is a crashed/abandoned claim and is recoverable. A fresh claim blocks its whole dedup_key so same-document ops never split across concurrent batches.';
-- Cover the PeekBatch query: the consumer scans rows for one
-- (task_type, scope, scope_id) tuple ordered by id ASC.