From d24e402ef6687080ee078610969854fbd863fd7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 14 Sep 2026 00:13:03 +0800 Subject: [PATCH] fix(orchestration): test poll/validate schedules carry the retry round jobs.idempotency_key is globally unique, but test-poll:: and test-validate:: were reused verbatim by every manual retry. The first round's completed rows made ON CONFLICT DO NOTHING silently swallow the next round's schedule SQL: a retried publication re-uploaded, polled once, and then froze in pending/uploaded with no successor job and no error. Keys and job ids are now derived from (publication, retry_count, sequence) inside the schedule SQL so every retry round owns fresh orchestration rows. --- .../platform/postgres/test_publications.go | 43 +++++++++---------- .../postgres/test_publications_test.go | 26 +++++------ 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/internal/platform/postgres/test_publications.go b/internal/platform/postgres/test_publications.go index de8212d..0bcd44c 100644 --- a/internal/platform/postgres/test_publications.go +++ b/internal/platform/postgres/test_publications.go @@ -230,13 +230,20 @@ WITH updated AS ( (ingest_state = 'uploading' AND knowledge_id = '' AND poll_sequence = 0) OR (ingest_state = 'uploaded' AND knowledge_id = $2 AND poll_sequence = 1) ) - RETURNING poll_sequence + RETURNING poll_sequence, retry_count ), enqueued AS ( + -- Orchestration identity is (publication, round, sequence): retry_count is + -- bumped by every manual retry, so poll/validate keys can never collide + -- with the consumed first round's completed rows (jobs.idempotency_key is + -- globally unique and ON CONFLICT DO NOTHING would silently swallow the + -- next round's schedules). INSERT INTO jobs ( id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at ) - SELECT $4, 'test.poll', $5::jsonb, 'pending', $6, $7, $8, $9, $3, $3 + SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#1') from 1 for 32), + 'test.poll', $4::jsonb, 'pending', $5, $6, $7, + 'test-poll:' || $1 || ':' || updated.retry_count || ':1', $3, $3 FROM updated ON CONFLICT (idempotency_key) DO NOTHING ) @@ -263,13 +270,15 @@ WITH updated AS ( AND ingest_state = 'uploaded' AND knowledge_id <> '' AND processing_state IN ('pending', 'processing', 'finalizing') - RETURNING poll_sequence + RETURNING poll_sequence, retry_count ), enqueued AS ( INSERT INTO jobs ( id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at ) - SELECT $6, 'test.poll', $7::jsonb, 'pending', $8, $9, $10, $11, $5, $5 + SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32), + 'test.poll', $6::jsonb, 'pending', $7, $8, $9, + 'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $5, $5 FROM updated ON CONFLICT (idempotency_key) DO NOTHING ) @@ -284,8 +293,8 @@ WITH updated AS ( last_remote_updated_at = $3, last_observed_at = $4, completed_at = $4, - parser_generation = $11, - index_generation = $12, + parser_generation = $7, + index_generation = $8, last_error = '', updated_at = $4 WHERE id = $1 @@ -293,13 +302,15 @@ WITH updated AS ( AND ingest_state = 'uploaded' AND knowledge_id <> '' AND processing_state IN ('pending', 'processing', 'finalizing') - RETURNING poll_sequence + RETURNING poll_sequence, retry_count ), enqueued AS ( INSERT INTO jobs ( id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at ) - SELECT $5, 'test.validate', $6::jsonb, 'pending', $7, $8, $9, $10, $4, $4 + SELECT 'job-' || substring(md5($1 || '#test.validate#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32), + 'test.validate', $5::jsonb, 'pending', $6, $9, $10, + 'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $4, $4 FROM updated ON CONFLICT (idempotency_key) DO NOTHING ) @@ -582,8 +593,6 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID, if err != nil { return 0, errors.New("encode first test poll job") } - idempotencyKey := "test-poll:" + publicationID + ":1" - jobID := orchestrationJobID("test.poll", publicationID, "1") var sequence int err = s.db.QueryRow( ctx, @@ -591,12 +600,10 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID, publicationID, knowledgeID, now, - jobID, payload, 0, nextAt, 5, - idempotencyKey, ).Scan(&sequence) if errors.Is(err, pgx.ErrNoRows) { return 0, ErrPublicationStateConflict @@ -627,8 +634,6 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll( if err != nil { return 0, errors.New("encode next test poll job") } - idempotencyKey := fmt.Sprintf("test-poll:%s:%d", publicationID, nextSequence) - jobID := orchestrationJobID("test.poll", publicationID, fmt.Sprint(nextSequence)) var storedSequence int err = s.db.QueryRow( ctx, @@ -638,12 +643,10 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll( string(status), remoteUpdatedAt, observedAt, - jobID, payload, 0, nextAt, 5, - idempotencyKey, ).Scan(&storedSequence) if errors.Is(err, pgx.ErrNoRows) { return 0, ErrPublicationStateConflict @@ -681,8 +684,6 @@ func (s *Store) RecordCompletedAndScheduleValidation( if err != nil { return errors.New("encode test validation job") } - idempotencyKey := fmt.Sprintf("test-validate:%s:%d", publicationID, sequence) - jobID := orchestrationJobID("test.validate", publicationID, fmt.Sprint(sequence)) var storedSequence int err = s.db.QueryRow( ctx, @@ -691,14 +692,12 @@ func (s *Store) RecordCompletedAndScheduleValidation( sequence, remoteUpdatedAt, observedAt, - jobID, payload, 0, + generation, + generation, observedAt, 5, - idempotencyKey, - generation, - generation, ).Scan(&storedSequence) if errors.Is(err, pgx.ErrNoRows) { return ErrPublicationStateConflict diff --git a/internal/platform/postgres/test_publications_test.go b/internal/platform/postgres/test_publications_test.go index 16782ac..255dfe7 100644 --- a/internal/platform/postgres/test_publications_test.go +++ b/internal/platform/postgres/test_publications_test.go @@ -64,14 +64,14 @@ func TestRecordUploadAndSchedulePollIsAtomicAndIdempotent(t *testing.T) { if updateAt < 0 || insertAt < 0 || updateAt >= insertAt { t.Fatalf("atomic SQL missing ordered update/insert: %q", recordUploadAndSchedulePollSQL) } - if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 9 { + if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 7 { t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) } - if db.queryRowArgs[8] != "test-poll:publication-1:1" { - t.Fatalf("idempotency key = %#v", db.queryRowArgs[8]) + if !strings.Contains(recordUploadAndSchedulePollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':1'") { + t.Fatalf("poll schedule key must carry the retry round: %q", recordUploadAndSchedulePollSQL) } var payload map[string]any - if err := json.Unmarshal(db.queryRowArgs[4].([]byte), &payload); err != nil { + if err := json.Unmarshal(db.queryRowArgs[3].([]byte), &payload); err != nil { t.Fatal(err) } if len(payload) != 2 || payload["publication_id"] != "publication-1" || payload["sequence"] != float64(1) { @@ -102,11 +102,11 @@ func TestRecordRemoteStatusSchedulesNextSequenceAtomically(t *testing.T) { if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "poll_sequence = $2") || !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "INSERT INTO jobs") { t.Fatalf("SQL lacks sequence guard or atomic enqueue: %q", recordRemoteStatusAndScheduleNextPollSQL) } - if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 11 { + if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 9 { t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) } - if db.queryRowArgs[10] != "test-poll:publication-1:2" { - t.Fatalf("idempotency key = %#v", db.queryRowArgs[10]) + if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") { + t.Fatalf("next-poll schedule key must carry the retry round: %q", recordRemoteStatusAndScheduleNextPollSQL) } } @@ -123,17 +123,17 @@ func TestRecordCompletedSchedulesValidationAtomically(t *testing.T) { if !strings.Contains(recordCompletedAndScheduleValidationSQL, "processing_state = 'completed'") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test.validate'") { t.Fatalf("SQL lacks completed update or validation enqueue: %q", recordCompletedAndScheduleValidationSQL) } - if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $11") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $12") { + if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $7") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $8") { t.Fatalf("SQL lacks release generation stamping: %q", recordCompletedAndScheduleValidationSQL) } - if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 12 { + if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 10 { t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) } - if db.queryRowArgs[9] != "test-validate:publication-1:3" { - t.Fatalf("idempotency key = %#v", db.queryRowArgs[9]) + if db.queryRowArgs[6] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[7] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" { + t.Fatalf("generation args = %#v", db.queryRowArgs[6:]) } - if db.queryRowArgs[10] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[11] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" { - t.Fatalf("generation args = %#v", db.queryRowArgs[10:]) + if !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") { + t.Fatalf("validate schedule key must carry the retry round: %q", recordCompletedAndScheduleValidationSQL) } }