fix(orchestration): test poll/validate schedules carry the retry round
jobs.idempotency_key is globally unique, but test-poll:<pub>:<seq> and test-validate:<pub>:<seq> were reused verbatim by every manual retry. The first round's completed rows made ON CONFLICT DO NOTHING silently swallow the next round's schedule SQL: a retried publication re-uploaded, polled once, and then froze in pending/uploaded with no successor job and no error. Keys and job ids are now derived from (publication, retry_count, sequence) inside the schedule SQL so every retry round owns fresh orchestration rows.
This commit is contained in:
@@ -230,13 +230,20 @@ WITH updated AS (
|
||||
(ingest_state = 'uploading' AND knowledge_id = '' AND poll_sequence = 0)
|
||||
OR (ingest_state = 'uploaded' AND knowledge_id = $2 AND poll_sequence = 1)
|
||||
)
|
||||
RETURNING poll_sequence
|
||||
RETURNING poll_sequence, retry_count
|
||||
), enqueued AS (
|
||||
-- Orchestration identity is (publication, round, sequence): retry_count is
|
||||
-- bumped by every manual retry, so poll/validate keys can never collide
|
||||
-- with the consumed first round's completed rows (jobs.idempotency_key is
|
||||
-- globally unique and ON CONFLICT DO NOTHING would silently swallow the
|
||||
-- next round's schedules).
|
||||
INSERT INTO jobs (
|
||||
id, type, payload, state, priority, available_at,
|
||||
max_attempts, idempotency_key, created_at, updated_at
|
||||
)
|
||||
SELECT $4, 'test.poll', $5::jsonb, 'pending', $6, $7, $8, $9, $3, $3
|
||||
SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#1') from 1 for 32),
|
||||
'test.poll', $4::jsonb, 'pending', $5, $6, $7,
|
||||
'test-poll:' || $1 || ':' || updated.retry_count || ':1', $3, $3
|
||||
FROM updated
|
||||
ON CONFLICT (idempotency_key) DO NOTHING
|
||||
)
|
||||
@@ -263,13 +270,15 @@ WITH updated AS (
|
||||
AND ingest_state = 'uploaded'
|
||||
AND knowledge_id <> ''
|
||||
AND processing_state IN ('pending', 'processing', 'finalizing')
|
||||
RETURNING poll_sequence
|
||||
RETURNING poll_sequence, retry_count
|
||||
), enqueued AS (
|
||||
INSERT INTO jobs (
|
||||
id, type, payload, state, priority, available_at,
|
||||
max_attempts, idempotency_key, created_at, updated_at
|
||||
)
|
||||
SELECT $6, 'test.poll', $7::jsonb, 'pending', $8, $9, $10, $11, $5, $5
|
||||
SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32),
|
||||
'test.poll', $6::jsonb, 'pending', $7, $8, $9,
|
||||
'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $5, $5
|
||||
FROM updated
|
||||
ON CONFLICT (idempotency_key) DO NOTHING
|
||||
)
|
||||
@@ -284,8 +293,8 @@ WITH updated AS (
|
||||
last_remote_updated_at = $3,
|
||||
last_observed_at = $4,
|
||||
completed_at = $4,
|
||||
parser_generation = $11,
|
||||
index_generation = $12,
|
||||
parser_generation = $7,
|
||||
index_generation = $8,
|
||||
last_error = '',
|
||||
updated_at = $4
|
||||
WHERE id = $1
|
||||
@@ -293,13 +302,15 @@ WITH updated AS (
|
||||
AND ingest_state = 'uploaded'
|
||||
AND knowledge_id <> ''
|
||||
AND processing_state IN ('pending', 'processing', 'finalizing')
|
||||
RETURNING poll_sequence
|
||||
RETURNING poll_sequence, retry_count
|
||||
), enqueued AS (
|
||||
INSERT INTO jobs (
|
||||
id, type, payload, state, priority, available_at,
|
||||
max_attempts, idempotency_key, created_at, updated_at
|
||||
)
|
||||
SELECT $5, 'test.validate', $6::jsonb, 'pending', $7, $8, $9, $10, $4, $4
|
||||
SELECT 'job-' || substring(md5($1 || '#test.validate#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32),
|
||||
'test.validate', $5::jsonb, 'pending', $6, $9, $10,
|
||||
'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $4, $4
|
||||
FROM updated
|
||||
ON CONFLICT (idempotency_key) DO NOTHING
|
||||
)
|
||||
@@ -582,8 +593,6 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID,
|
||||
if err != nil {
|
||||
return 0, errors.New("encode first test poll job")
|
||||
}
|
||||
idempotencyKey := "test-poll:" + publicationID + ":1"
|
||||
jobID := orchestrationJobID("test.poll", publicationID, "1")
|
||||
var sequence int
|
||||
err = s.db.QueryRow(
|
||||
ctx,
|
||||
@@ -591,12 +600,10 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID,
|
||||
publicationID,
|
||||
knowledgeID,
|
||||
now,
|
||||
jobID,
|
||||
payload,
|
||||
0,
|
||||
nextAt,
|
||||
5,
|
||||
idempotencyKey,
|
||||
).Scan(&sequence)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, ErrPublicationStateConflict
|
||||
@@ -627,8 +634,6 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll(
|
||||
if err != nil {
|
||||
return 0, errors.New("encode next test poll job")
|
||||
}
|
||||
idempotencyKey := fmt.Sprintf("test-poll:%s:%d", publicationID, nextSequence)
|
||||
jobID := orchestrationJobID("test.poll", publicationID, fmt.Sprint(nextSequence))
|
||||
var storedSequence int
|
||||
err = s.db.QueryRow(
|
||||
ctx,
|
||||
@@ -638,12 +643,10 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll(
|
||||
string(status),
|
||||
remoteUpdatedAt,
|
||||
observedAt,
|
||||
jobID,
|
||||
payload,
|
||||
0,
|
||||
nextAt,
|
||||
5,
|
||||
idempotencyKey,
|
||||
).Scan(&storedSequence)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, ErrPublicationStateConflict
|
||||
@@ -681,8 +684,6 @@ func (s *Store) RecordCompletedAndScheduleValidation(
|
||||
if err != nil {
|
||||
return errors.New("encode test validation job")
|
||||
}
|
||||
idempotencyKey := fmt.Sprintf("test-validate:%s:%d", publicationID, sequence)
|
||||
jobID := orchestrationJobID("test.validate", publicationID, fmt.Sprint(sequence))
|
||||
var storedSequence int
|
||||
err = s.db.QueryRow(
|
||||
ctx,
|
||||
@@ -691,14 +692,12 @@ func (s *Store) RecordCompletedAndScheduleValidation(
|
||||
sequence,
|
||||
remoteUpdatedAt,
|
||||
observedAt,
|
||||
jobID,
|
||||
payload,
|
||||
0,
|
||||
generation,
|
||||
generation,
|
||||
observedAt,
|
||||
5,
|
||||
idempotencyKey,
|
||||
generation,
|
||||
generation,
|
||||
).Scan(&storedSequence)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrPublicationStateConflict
|
||||
|
||||
@@ -64,14 +64,14 @@ func TestRecordUploadAndSchedulePollIsAtomicAndIdempotent(t *testing.T) {
|
||||
if updateAt < 0 || insertAt < 0 || updateAt >= insertAt {
|
||||
t.Fatalf("atomic SQL missing ordered update/insert: %q", recordUploadAndSchedulePollSQL)
|
||||
}
|
||||
if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 9 {
|
||||
if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 7 {
|
||||
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
|
||||
}
|
||||
if db.queryRowArgs[8] != "test-poll:publication-1:1" {
|
||||
t.Fatalf("idempotency key = %#v", db.queryRowArgs[8])
|
||||
if !strings.Contains(recordUploadAndSchedulePollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':1'") {
|
||||
t.Fatalf("poll schedule key must carry the retry round: %q", recordUploadAndSchedulePollSQL)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(db.queryRowArgs[4].([]byte), &payload); err != nil {
|
||||
if err := json.Unmarshal(db.queryRowArgs[3].([]byte), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload) != 2 || payload["publication_id"] != "publication-1" || payload["sequence"] != float64(1) {
|
||||
@@ -102,11 +102,11 @@ func TestRecordRemoteStatusSchedulesNextSequenceAtomically(t *testing.T) {
|
||||
if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "poll_sequence = $2") || !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "INSERT INTO jobs") {
|
||||
t.Fatalf("SQL lacks sequence guard or atomic enqueue: %q", recordRemoteStatusAndScheduleNextPollSQL)
|
||||
}
|
||||
if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 11 {
|
||||
if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 9 {
|
||||
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
|
||||
}
|
||||
if db.queryRowArgs[10] != "test-poll:publication-1:2" {
|
||||
t.Fatalf("idempotency key = %#v", db.queryRowArgs[10])
|
||||
if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") {
|
||||
t.Fatalf("next-poll schedule key must carry the retry round: %q", recordRemoteStatusAndScheduleNextPollSQL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,17 +123,17 @@ func TestRecordCompletedSchedulesValidationAtomically(t *testing.T) {
|
||||
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "processing_state = 'completed'") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test.validate'") {
|
||||
t.Fatalf("SQL lacks completed update or validation enqueue: %q", recordCompletedAndScheduleValidationSQL)
|
||||
}
|
||||
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $11") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $12") {
|
||||
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $7") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $8") {
|
||||
t.Fatalf("SQL lacks release generation stamping: %q", recordCompletedAndScheduleValidationSQL)
|
||||
}
|
||||
if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 12 {
|
||||
if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 10 {
|
||||
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
|
||||
}
|
||||
if db.queryRowArgs[9] != "test-validate:publication-1:3" {
|
||||
t.Fatalf("idempotency key = %#v", db.queryRowArgs[9])
|
||||
if db.queryRowArgs[6] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[7] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" {
|
||||
t.Fatalf("generation args = %#v", db.queryRowArgs[6:])
|
||||
}
|
||||
if db.queryRowArgs[10] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[11] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" {
|
||||
t.Fatalf("generation args = %#v", db.queryRowArgs[10:])
|
||||
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") {
|
||||
t.Fatalf("validate schedule key must carry the retry round: %q", recordCompletedAndScheduleValidationSQL)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user