refactor(task_queue): enhance claim handling and test coverage

- Renamed TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed to TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim for clarity on behavior.
- Added new test TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale to verify claim behavior when a holder crashes.
- Updated ClaimBatch logic to ensure that fresh claims block late siblings entirely, maintaining serialization of same-document operations.
- Improved comments in task_queue.go to clarify the handling of concurrent claims and the implications of the claimed_at field.
- Added crash safety mechanism in ProcessWikiIngest to release claims on abnormal exits, ensuring timely re-claims of pending operations.
This commit is contained in:
wizardchen
2026-07-09 16:23:50 +08:00
committed by lyingbug
parent 60f7f35e40
commit 76a8092483
4 changed files with 109 additions and 18 deletions
@@ -347,10 +347,13 @@ func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) {
assert.Equal(t, "k3", second[0].DedupKey)
}
// TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed verifies a
// row enqueued for an already-claimed (in-flight) document is left for a
// later batch rather than being handed out alongside the fresh claim.
func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a
// row enqueued for an already-claimed (in-flight) document is NOT claimed on
// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late
// sibling waits for the holder to finish (rows deleted → key freed) or for the
// claim to go stale, keeping same-document ops serialized across concurrent
// batches.
func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
ctx := context.Background()
@@ -361,16 +364,51 @@ func TestTaskPendingOps_ClaimBatch_NewSiblingRowNotDoubleClaimed(t *testing.T) {
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, first, 1)
ingestID := first[0].ID
// A retract for the same, still-in-flight document arrives.
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
// The next claim gets ONLY the new (unclaimed) retract row — the
// still-claimed ingest row is not re-handed-out.
// The retract must NOT be claimed while the ingest claim is fresh —
// the whole k1 key is blocked.
second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, second, 1)
assert.Equal(t, "retract", second[0].Op)
require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh")
// Once the holder finishes (its claimed rows are deleted), the key is
// free and the retract becomes claimable.
require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID}))
third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
require.NoError(t, err)
require.Len(t, third, 1)
assert.Equal(t, "retract", third[0].Op)
}
// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the
// other release path: if the holder CRASHES (claim never cleared), the whole
// key — original row + late sibling — becomes claimable together once the
// claim goes stale, so the pair is folded back into one batch.
func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
ctx := context.Background()
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour))
require.NoError(t, err)
require.Len(t, first, 1)
// Late retract arrives; holder then crashes (claim left stamped).
require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
// A future stale threshold makes the crashed claim eligible again; both
// k1 rows are reclaimed together (never split).
got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour))
require.NoError(t, err)
require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together")
for _, r := range got {
assert.Equal(t, "k1", r.DedupKey)
}
}
// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than