refactor(task_queue): enhance claim handling and test coverage
- Renamed TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed to TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim for clarity on behavior. - Added new test TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale to verify claim behavior when a holder crashes. - Updated ClaimBatch logic to ensure that fresh claims block late siblings entirely, maintaining serialization of same-document operations. - Improved comments in task_queue.go to clarify the handling of concurrent claims and the implications of the claimed_at field. - Added crash safety mechanism in ProcessWikiIngest to release claims on abnormal exits, ensuring timely re-claims of pending operations.
This commit is contained in:
@@ -81,8 +81,17 @@ func (r *taskPendingOpsRepository) PeekBatch(
|
||||
// otherwise each batch's per-batch last-write-wins dedup can't collapse the
|
||||
// pair and the two ops race (a stale ingest could resurrect a retracted doc).
|
||||
//
|
||||
// To uphold that invariant even for a sibling enqueued AFTER a batch already
|
||||
// claimed the key (e.g. a retract arriving while the ingest is still in
|
||||
// flight), a dedup_key that has ANY fresh claim (claimed_at >= staleBefore) is
|
||||
// skipped ENTIRELY — not just its already-claimed rows. The late sibling waits
|
||||
// for the holder to finish (which deletes the claimed rows, freeing the key) or
|
||||
// for the claim to go stale. This serializes same-document ops across
|
||||
// concurrent batches instead of letting them race on wall-clock completion.
|
||||
//
|
||||
// Eligibility = unclaimed (claimed_at IS NULL) OR stale claim
|
||||
// (claimed_at < staleBefore). The whole thing runs in one transaction:
|
||||
// (claimed_at < staleBefore), AND the key has no fresh claim. The whole thing
|
||||
// runs in one transaction:
|
||||
//
|
||||
// - Postgres: we lock the ANCHOR row (earliest eligible id) of each
|
||||
// candidate dedup_key with FOR UPDATE SKIP LOCKED. Because the anchor
|
||||
@@ -113,11 +122,14 @@ func (r *taskPendingOpsRepository) ClaimBatch(
|
||||
var claimed []*types.TaskPendingOp
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// 1. Pick up to `limit` distinct dedup_keys to claim, oldest first.
|
||||
// Keys with a fresh claim are excluded WHOLESALE so a late sibling
|
||||
// of an in-flight document never gets claimed on its own.
|
||||
var keys []string
|
||||
if tx.Dialector.Name() == "postgres" {
|
||||
// Lock the anchor (earliest eligible) row of each key with SKIP
|
||||
// LOCKED so concurrent claimers get disjoint KEY sets, then map
|
||||
// the locked anchors back to their dedup_keys.
|
||||
// the locked anchors back to their dedup_keys. The NOT IN subquery
|
||||
// drops any key that still has a fresh (non-stale) claim.
|
||||
const anchorSQL = `
|
||||
SELECT dedup_key FROM task_pending_ops
|
||||
WHERE id IN (
|
||||
@@ -126,19 +138,32 @@ WHERE id IN (
|
||||
FROM task_pending_ops
|
||||
WHERE task_type = ? AND scope = ? AND scope_id = ?
|
||||
AND (claimed_at IS NULL OR claimed_at < ?)
|
||||
AND dedup_key NOT IN (
|
||||
SELECT dedup_key FROM task_pending_ops
|
||||
WHERE task_type = ? AND scope = ? AND scope_id = ?
|
||||
AND claimed_at IS NOT NULL AND claimed_at >= ?
|
||||
)
|
||||
) anchors WHERE anchors.rn = 1
|
||||
)
|
||||
ORDER BY id ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE SKIP LOCKED`
|
||||
if err := tx.Raw(anchorSQL, taskType, scope, scopeID, staleBefore, limit).
|
||||
if err := tx.Raw(anchorSQL,
|
||||
taskType, scope, scopeID, staleBefore,
|
||||
taskType, scope, scopeID, staleBefore,
|
||||
limit).
|
||||
Scan(&keys).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
freshKeys := tx.Model(&types.TaskPendingOp{}).
|
||||
Select("dedup_key").
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
|
||||
if err := tx.Model(&types.TaskPendingOp{}).
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore).
|
||||
Where("dedup_key NOT IN (?)", freshKeys).
|
||||
Group("dedup_key").
|
||||
Order("MIN(id) ASC").
|
||||
Limit(limit).
|
||||
|
||||
@@ -347,10 +347,13 @@ func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) {
|
||||
assert.Equal(t, "k3", second[0].DedupKey)
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed verifies a
|
||||
// row enqueued for an already-claimed (in-flight) document is left for a
|
||||
// later batch rather than being handed out alongside the fresh claim.
|
||||
func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
|
||||
// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a
|
||||
// row enqueued for an already-claimed (in-flight) document is NOT claimed on
|
||||
// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late
|
||||
// sibling waits for the holder to finish (rows deleted → key freed) or for the
|
||||
// claim to go stale, keeping same-document ops serialized across concurrent
|
||||
// batches.
|
||||
func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
ctx := context.Background()
|
||||
@@ -361,16 +364,51 @@ func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
|
||||
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
ingestID := first[0].ID
|
||||
|
||||
// A retract for the same, still-in-flight document arrives.
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
|
||||
|
||||
// The next claim gets ONLY the new (unclaimed) retract row — the
|
||||
// still-claimed ingest row is not re-handed-out.
|
||||
// The retract must NOT be claimed while the ingest claim is fresh —
|
||||
// the whole k1 key is blocked.
|
||||
second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, second, 1)
|
||||
assert.Equal(t, "retract", second[0].Op)
|
||||
require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh")
|
||||
|
||||
// Once the holder finishes (its claimed rows are deleted), the key is
|
||||
// free and the retract becomes claimable.
|
||||
require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID}))
|
||||
third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, third, 1)
|
||||
assert.Equal(t, "retract", third[0].Op)
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the
|
||||
// other release path: if the holder CRASHES (claim never cleared), the whole
|
||||
// key — original row + late sibling — becomes claimable together once the
|
||||
// claim goes stale, so the pair is folded back into one batch.
|
||||
func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
|
||||
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
|
||||
// Late retract arrives; holder then crashes (claim left stamped).
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
|
||||
|
||||
// A future stale threshold makes the crashed claim eligible again; both
|
||||
// k1 rows are reclaimed together (never split).
|
||||
got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together")
|
||||
for _, r := range got {
|
||||
assert.Equal(t, "k1", r.DedupKey)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than
|
||||
|
||||
@@ -361,6 +361,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
|
||||
|
||||
// Crash/abort safety net (standard/claim mode only). If this batch exits
|
||||
// abnormally — panic, ctx timeout, or an early error return — BEFORE it
|
||||
// settles its claimed rows (trim + requeueFailedOps), release the claims
|
||||
// so the next trigger can re-claim within seconds instead of waiting out
|
||||
// wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true
|
||||
// once the rows reach their terminal state, making this a no-op. Uses a
|
||||
// background context because ctx may already be cancelled on the timeout
|
||||
// path. Lite mode peeks without claiming, so there is nothing to release.
|
||||
claimsSettled := false
|
||||
if s.redisClient != nil && len(peekedIDs) > 0 {
|
||||
defer func() {
|
||||
if claimsSettled {
|
||||
return
|
||||
}
|
||||
if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
logger.Warnf(ctx, "wiki ingest: released %d claimed rows on abnormal exit for KB %s (re-claimable immediately)", len(peekedIDs), payload.KnowledgeBaseID)
|
||||
}()
|
||||
}
|
||||
|
||||
// Resolve extraction granularity once per batch. Historical rows with
|
||||
// empty/unknown values fall back to Standard via Normalize(). Failures
|
||||
// to load the KB (unlikely since we're already acting on it) also
|
||||
@@ -852,6 +874,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
if len(failedOps) > 0 {
|
||||
s.requeueFailedOps(ctx, payload, failedOps)
|
||||
}
|
||||
// All claimed rows have now reached a terminal state (deleted on success,
|
||||
// released for retry, or dead-lettered), so disarm the abnormal-exit
|
||||
// release net.
|
||||
claimsSettled = true
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
|
||||
|
||||
|
||||
@@ -39,12 +39,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
-- consumer (e.g. wiki ingest batch handler) uses PeekBatch to pull the head
|
||||
-- of the list, processes the ops, then DeleteByIDs the consumed rows.
|
||||
--
|
||||
-- claimed_at is intentionally added but unused in this revision: the wiki
|
||||
-- ingest pipeline guarantees per-KB serialization via its own active-batch
|
||||
-- lock (Redis SetNX wiki:active:<kbID>), so PeekBatch doesn't need
|
||||
-- SELECT ... FOR UPDATE. Future task types that don't carry an external
|
||||
-- lock can flip claimed_at = NOW() inside a row-level lock and rely on the
|
||||
-- column to detect orphaned claims after a worker crash.
|
||||
-- claimed_at backs the concurrent-claim workflow (ClaimBatch): standard-mode
|
||||
-- wiki ingest dropped the exclusive per-KB lock in favour of multiple batches
|
||||
-- claiming DISJOINT dedup_keys via SELECT ... FOR UPDATE SKIP LOCKED, stamping
|
||||
-- claimed_at = NOW() on the rows they take. A claim older than the consumer's
|
||||
-- stale threshold (a crashed/abandoned worker) is recoverable by the next
|
||||
-- claimer; a fresh claim blocks its whole dedup_key so same-document ops never
|
||||
-- split across concurrent batches. Lite mode (no Redis) still serializes per
|
||||
-- KB in-process and leaves claimed_at NULL.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS task_pending_ops (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
@@ -65,7 +67,7 @@ COMMENT ON COLUMN task_pending_ops.task_type IS 'Free-form task identifier, e.g.
|
||||
COMMENT ON COLUMN task_pending_ops.scope IS 'Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". Read together with scope_id.';
|
||||
COMMENT ON COLUMN task_pending_ops.dedup_key IS 'Optional service-defined key used by the consumer to de-duplicate equivalent ops within a single batch peek. Empty means no de-dup.';
|
||||
COMMENT ON COLUMN task_pending_ops.fail_count IS 'In-batch retry counter: the consumer increments it via IncrFailCount and dead-letters once it exceeds a service-defined cap.';
|
||||
COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Reserved for future locking workflows; consumers in this revision rely on external mutual exclusion and ignore the column.';
|
||||
COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Concurrent-claim marker: set to NOW() when a consumer claims the row (SELECT ... FOR UPDATE SKIP LOCKED). NULL = unclaimed; a value older than the consumer stale threshold is a crashed/abandoned claim and is recoverable. A fresh claim blocks its whole dedup_key so same-document ops never split across concurrent batches.';
|
||||
|
||||
-- Cover the PeekBatch query: the consumer scans rows for one
|
||||
-- (task_type, scope, scope_id) tuple ordered by id ASC.
|
||||
|
||||
Reference in New Issue
Block a user