refactor(task_queue): enhance claim handling and test coverage
- Renamed TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed to TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim for clarity on behavior. - Added new test TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale to verify claim behavior when a holder crashes. - Updated ClaimBatch logic to ensure that fresh claims block late siblings entirely, maintaining serialization of same-document operations. - Improved comments in task_queue.go to clarify the handling of concurrent claims and the implications of the claimed_at field. - Added crash safety mechanism in ProcessWikiIngest to release claims on abnormal exits, ensuring timely re-claims of pending operations.
This commit is contained in:
@@ -81,8 +81,17 @@ func (r *taskPendingOpsRepository) PeekBatch(
|
||||
// otherwise each batch's per-batch last-write-wins dedup can't collapse the
|
||||
// pair and the two ops race (a stale ingest could resurrect a retracted doc).
|
||||
//
|
||||
// To uphold that invariant even for a sibling enqueued AFTER a batch already
|
||||
// claimed the key (e.g. a retract arriving while the ingest is still in
|
||||
// flight), a dedup_key that has ANY fresh claim (claimed_at >= staleBefore) is
|
||||
// skipped ENTIRELY — not just its already-claimed rows. The late sibling waits
|
||||
// for the holder to finish (which deletes the claimed rows, freeing the key) or
|
||||
// for the claim to go stale. This serializes same-document ops across
|
||||
// concurrent batches instead of letting them race on wall-clock completion.
|
||||
//
|
||||
// Eligibility = unclaimed (claimed_at IS NULL) OR stale claim
|
||||
// (claimed_at < staleBefore). The whole thing runs in one transaction:
|
||||
// (claimed_at < staleBefore), AND the key has no fresh claim. The whole thing
|
||||
// runs in one transaction:
|
||||
//
|
||||
// - Postgres: we lock the ANCHOR row (earliest eligible id) of each
|
||||
// candidate dedup_key with FOR UPDATE SKIP LOCKED. Because the anchor
|
||||
@@ -113,11 +122,14 @@ func (r *taskPendingOpsRepository) ClaimBatch(
|
||||
var claimed []*types.TaskPendingOp
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// 1. Pick up to `limit` distinct dedup_keys to claim, oldest first.
|
||||
// Keys with a fresh claim are excluded WHOLESALE so a late sibling
|
||||
// of an in-flight document never gets claimed on its own.
|
||||
var keys []string
|
||||
if tx.Dialector.Name() == "postgres" {
|
||||
// Lock the anchor (earliest eligible) row of each key with SKIP
|
||||
// LOCKED so concurrent claimers get disjoint KEY sets, then map
|
||||
// the locked anchors back to their dedup_keys.
|
||||
// the locked anchors back to their dedup_keys. The NOT IN subquery
|
||||
// drops any key that still has a fresh (non-stale) claim.
|
||||
const anchorSQL = `
|
||||
SELECT dedup_key FROM task_pending_ops
|
||||
WHERE id IN (
|
||||
@@ -126,19 +138,32 @@ WHERE id IN (
|
||||
FROM task_pending_ops
|
||||
WHERE task_type = ? AND scope = ? AND scope_id = ?
|
||||
AND (claimed_at IS NULL OR claimed_at < ?)
|
||||
AND dedup_key NOT IN (
|
||||
SELECT dedup_key FROM task_pending_ops
|
||||
WHERE task_type = ? AND scope = ? AND scope_id = ?
|
||||
AND claimed_at IS NOT NULL AND claimed_at >= ?
|
||||
)
|
||||
) anchors WHERE anchors.rn = 1
|
||||
)
|
||||
ORDER BY id ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE SKIP LOCKED`
|
||||
if err := tx.Raw(anchorSQL, taskType, scope, scopeID, staleBefore, limit).
|
||||
if err := tx.Raw(anchorSQL,
|
||||
taskType, scope, scopeID, staleBefore,
|
||||
taskType, scope, scopeID, staleBefore,
|
||||
limit).
|
||||
Scan(&keys).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
freshKeys := tx.Model(&types.TaskPendingOp{}).
|
||||
Select("dedup_key").
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
|
||||
if err := tx.Model(&types.TaskPendingOp{}).
|
||||
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
|
||||
Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore).
|
||||
Where("dedup_key NOT IN (?)", freshKeys).
|
||||
Group("dedup_key").
|
||||
Order("MIN(id) ASC").
|
||||
Limit(limit).
|
||||
|
||||
@@ -347,10 +347,13 @@ func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) {
|
||||
assert.Equal(t, "k3", second[0].DedupKey)
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed verifies a
|
||||
// row enqueued for an already-claimed (in-flight) document is left for a
|
||||
// later batch rather than being handed out alongside the fresh claim.
|
||||
func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
|
||||
// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a
|
||||
// row enqueued for an already-claimed (in-flight) document is NOT claimed on
|
||||
// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late
|
||||
// sibling waits for the holder to finish (rows deleted → key freed) or for the
|
||||
// claim to go stale, keeping same-document ops serialized across concurrent
|
||||
// batches.
|
||||
func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
ctx := context.Background()
|
||||
@@ -361,16 +364,51 @@ func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
|
||||
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
ingestID := first[0].ID
|
||||
|
||||
// A retract for the same, still-in-flight document arrives.
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
|
||||
|
||||
// The next claim gets ONLY the new (unclaimed) retract row — the
|
||||
// still-claimed ingest row is not re-handed-out.
|
||||
// The retract must NOT be claimed while the ingest claim is fresh —
|
||||
// the whole k1 key is blocked.
|
||||
second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, second, 1)
|
||||
assert.Equal(t, "retract", second[0].Op)
|
||||
require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh")
|
||||
|
||||
// Once the holder finishes (its claimed rows are deleted), the key is
|
||||
// free and the retract becomes claimable.
|
||||
require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID}))
|
||||
third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, third, 1)
|
||||
assert.Equal(t, "retract", third[0].Op)
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the
|
||||
// other release path: if the holder CRASHES (claim never cleared), the whole
|
||||
// key — original row + late sibling — becomes claimable together once the
|
||||
// claim goes stale, so the pair is folded back into one batch.
|
||||
func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
|
||||
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
|
||||
// Late retract arrives; holder then crashes (claim left stamped).
|
||||
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
|
||||
|
||||
// A future stale threshold makes the crashed claim eligible again; both
|
||||
// k1 rows are reclaimed together (never split).
|
||||
got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together")
|
||||
for _, r := range got {
|
||||
assert.Equal(t, "k1", r.DedupKey)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than
|
||||
|
||||
@@ -361,6 +361,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
|
||||
|
||||
// Crash/abort safety net (standard/claim mode only). If this batch exits
|
||||
// abnormally — panic, ctx timeout, or an early error return — BEFORE it
|
||||
// settles its claimed rows (trim + requeueFailedOps), release the claims
|
||||
// so the next trigger can re-claim within seconds instead of waiting out
|
||||
// wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true
|
||||
// once the rows reach their terminal state, making this a no-op. Uses a
|
||||
// background context because ctx may already be cancelled on the timeout
|
||||
// path. Lite mode peeks without claiming, so there is nothing to release.
|
||||
claimsSettled := false
|
||||
if s.redisClient != nil && len(peekedIDs) > 0 {
|
||||
defer func() {
|
||||
if claimsSettled {
|
||||
return
|
||||
}
|
||||
if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
logger.Warnf(ctx, "wiki ingest: released %d claimed rows on abnormal exit for KB %s (re-claimable immediately)", len(peekedIDs), payload.KnowledgeBaseID)
|
||||
}()
|
||||
}
|
||||
|
||||
// Resolve extraction granularity once per batch. Historical rows with
|
||||
// empty/unknown values fall back to Standard via Normalize(). Failures
|
||||
// to load the KB (unlikely since we're already acting on it) also
|
||||
@@ -852,6 +874,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
if len(failedOps) > 0 {
|
||||
s.requeueFailedOps(ctx, payload, failedOps)
|
||||
}
|
||||
// All claimed rows have now reached a terminal state (deleted on success,
|
||||
// released for retry, or dead-lettered), so disarm the abnormal-exit
|
||||
// release net.
|
||||
claimsSettled = true
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user