fix(orchestration): test poll/validate schedules carry the retry round

jobs.idempotency_key is globally unique, but test-poll:<pub>:<seq> and
test-validate:<pub>:<seq> were reused verbatim by every manual retry. The
first round's completed rows made ON CONFLICT DO NOTHING silently swallow
the next round's schedule SQL: a retried publication re-uploaded, polled
once, and then froze in pending/uploaded with no successor job and no
error. Keys and job ids are now derived from (publication, retry_count,
sequence) inside the schedule SQL so every retry round owns fresh
orchestration rows.
This commit is contained in:
姚章浩
2026-09-14 00:13:03 +08:00
parent c2d093bc90
commit d24e402ef6
2 changed files with 34 additions and 35 deletions
+21 -22
View File
@@ -230,13 +230,20 @@ WITH updated AS (
(ingest_state = 'uploading' AND knowledge_id = '' AND poll_sequence = 0)
OR (ingest_state = 'uploaded' AND knowledge_id = $2 AND poll_sequence = 1)
)
RETURNING poll_sequence
RETURNING poll_sequence, retry_count
), enqueued AS (
-- Orchestration identity is (publication, round, sequence): retry_count is
-- bumped by every manual retry, so poll/validate keys can never collide
-- with the consumed first round's completed rows (jobs.idempotency_key is
-- globally unique and ON CONFLICT DO NOTHING would silently swallow the
-- next round's schedules).
INSERT INTO jobs (
id, type, payload, state, priority, available_at,
max_attempts, idempotency_key, created_at, updated_at
)
SELECT $4, 'test.poll', $5::jsonb, 'pending', $6, $7, $8, $9, $3, $3
SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#1') from 1 for 32),
'test.poll', $4::jsonb, 'pending', $5, $6, $7,
'test-poll:' || $1 || ':' || updated.retry_count || ':1', $3, $3
FROM updated
ON CONFLICT (idempotency_key) DO NOTHING
)
@@ -263,13 +270,15 @@ WITH updated AS (
AND ingest_state = 'uploaded'
AND knowledge_id <> ''
AND processing_state IN ('pending', 'processing', 'finalizing')
RETURNING poll_sequence
RETURNING poll_sequence, retry_count
), enqueued AS (
INSERT INTO jobs (
id, type, payload, state, priority, available_at,
max_attempts, idempotency_key, created_at, updated_at
)
SELECT $6, 'test.poll', $7::jsonb, 'pending', $8, $9, $10, $11, $5, $5
SELECT 'job-' || substring(md5($1 || '#test.poll#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32),
'test.poll', $6::jsonb, 'pending', $7, $8, $9,
'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $5, $5
FROM updated
ON CONFLICT (idempotency_key) DO NOTHING
)
@@ -284,8 +293,8 @@ WITH updated AS (
last_remote_updated_at = $3,
last_observed_at = $4,
completed_at = $4,
parser_generation = $11,
index_generation = $12,
parser_generation = $7,
index_generation = $8,
last_error = '',
updated_at = $4
WHERE id = $1
@@ -293,13 +302,15 @@ WITH updated AS (
AND ingest_state = 'uploaded'
AND knowledge_id <> ''
AND processing_state IN ('pending', 'processing', 'finalizing')
RETURNING poll_sequence
RETURNING poll_sequence, retry_count
), enqueued AS (
INSERT INTO jobs (
id, type, payload, state, priority, available_at,
max_attempts, idempotency_key, created_at, updated_at
)
SELECT $5, 'test.validate', $6::jsonb, 'pending', $7, $8, $9, $10, $4, $4
SELECT 'job-' || substring(md5($1 || '#test.validate#' || updated.retry_count || '#' || updated.poll_sequence) from 1 for 32),
'test.validate', $5::jsonb, 'pending', $6, $9, $10,
'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence, $4, $4
FROM updated
ON CONFLICT (idempotency_key) DO NOTHING
)
@@ -582,8 +593,6 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID,
if err != nil {
return 0, errors.New("encode first test poll job")
}
idempotencyKey := "test-poll:" + publicationID + ":1"
jobID := orchestrationJobID("test.poll", publicationID, "1")
var sequence int
err = s.db.QueryRow(
ctx,
@@ -591,12 +600,10 @@ func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID,
publicationID,
knowledgeID,
now,
jobID,
payload,
0,
nextAt,
5,
idempotencyKey,
).Scan(&sequence)
if errors.Is(err, pgx.ErrNoRows) {
return 0, ErrPublicationStateConflict
@@ -627,8 +634,6 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll(
if err != nil {
return 0, errors.New("encode next test poll job")
}
idempotencyKey := fmt.Sprintf("test-poll:%s:%d", publicationID, nextSequence)
jobID := orchestrationJobID("test.poll", publicationID, fmt.Sprint(nextSequence))
var storedSequence int
err = s.db.QueryRow(
ctx,
@@ -638,12 +643,10 @@ func (s *Store) RecordRemoteStatusAndScheduleNextPoll(
string(status),
remoteUpdatedAt,
observedAt,
jobID,
payload,
0,
nextAt,
5,
idempotencyKey,
).Scan(&storedSequence)
if errors.Is(err, pgx.ErrNoRows) {
return 0, ErrPublicationStateConflict
@@ -681,8 +684,6 @@ func (s *Store) RecordCompletedAndScheduleValidation(
if err != nil {
return errors.New("encode test validation job")
}
idempotencyKey := fmt.Sprintf("test-validate:%s:%d", publicationID, sequence)
jobID := orchestrationJobID("test.validate", publicationID, fmt.Sprint(sequence))
var storedSequence int
err = s.db.QueryRow(
ctx,
@@ -691,14 +692,12 @@ func (s *Store) RecordCompletedAndScheduleValidation(
sequence,
remoteUpdatedAt,
observedAt,
jobID,
payload,
0,
generation,
generation,
observedAt,
5,
idempotencyKey,
generation,
generation,
).Scan(&storedSequence)
if errors.Is(err, pgx.ErrNoRows) {
return ErrPublicationStateConflict
@@ -64,14 +64,14 @@ func TestRecordUploadAndSchedulePollIsAtomicAndIdempotent(t *testing.T) {
if updateAt < 0 || insertAt < 0 || updateAt >= insertAt {
t.Fatalf("atomic SQL missing ordered update/insert: %q", recordUploadAndSchedulePollSQL)
}
if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 9 {
if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 7 {
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
}
if db.queryRowArgs[8] != "test-poll:publication-1:1" {
t.Fatalf("idempotency key = %#v", db.queryRowArgs[8])
if !strings.Contains(recordUploadAndSchedulePollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':1'") {
t.Fatalf("poll schedule key must carry the retry round: %q", recordUploadAndSchedulePollSQL)
}
var payload map[string]any
if err := json.Unmarshal(db.queryRowArgs[4].([]byte), &payload); err != nil {
if err := json.Unmarshal(db.queryRowArgs[3].([]byte), &payload); err != nil {
t.Fatal(err)
}
if len(payload) != 2 || payload["publication_id"] != "publication-1" || payload["sequence"] != float64(1) {
@@ -102,11 +102,11 @@ func TestRecordRemoteStatusSchedulesNextSequenceAtomically(t *testing.T) {
if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "poll_sequence = $2") || !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "INSERT INTO jobs") {
t.Fatalf("SQL lacks sequence guard or atomic enqueue: %q", recordRemoteStatusAndScheduleNextPollSQL)
}
if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 11 {
if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 9 {
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
}
if db.queryRowArgs[10] != "test-poll:publication-1:2" {
t.Fatalf("idempotency key = %#v", db.queryRowArgs[10])
if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "'test-poll:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") {
t.Fatalf("next-poll schedule key must carry the retry round: %q", recordRemoteStatusAndScheduleNextPollSQL)
}
}
@@ -123,17 +123,17 @@ func TestRecordCompletedSchedulesValidationAtomically(t *testing.T) {
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "processing_state = 'completed'") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test.validate'") {
t.Fatalf("SQL lacks completed update or validation enqueue: %q", recordCompletedAndScheduleValidationSQL)
}
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $11") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $12") {
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $7") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $8") {
t.Fatalf("SQL lacks release generation stamping: %q", recordCompletedAndScheduleValidationSQL)
}
if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 12 {
if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 10 {
t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs)
}
if db.queryRowArgs[9] != "test-validate:publication-1:3" {
t.Fatalf("idempotency key = %#v", db.queryRowArgs[9])
if db.queryRowArgs[6] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[7] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" {
t.Fatalf("generation args = %#v", db.queryRowArgs[6:])
}
if db.queryRowArgs[10] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[11] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" {
t.Fatalf("generation args = %#v", db.queryRowArgs[10:])
if !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test-validate:' || $1 || ':' || updated.retry_count || ':' || updated.poll_sequence") {
t.Fatalf("validate schedule key must carry the retry round: %q", recordCompletedAndScheduleValidationSQL)
}
}