package postgres import ( "context" "encoding/json" "os" "testing" "time" "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" ) // TestP6DeadLetterAndServerSideBatchRetry freezes docs plan P6: an ordinary // failure that exhausts its budget lands in the DLQ projection without // touching the original audit row, a permanent failure lands immediately, and // the server-side batch retry returns mixed outcomes atomically with replay. func TestP6DeadLetterAndServerSideBatchRetry(t *testing.T) { databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) if databaseURL == "" { t.Skip("TEST_DATABASE_URL is not set") } ctx := context.Background() pool, err := pgxpool.New(ctx, databaseURL) if err != nil { t.Fatalf("connect: %v", err) } defer pool.Close() if err := ApplyMigrations(ctx, pool); err != nil { t.Fatalf("migrate: %v", err) } _, seedErr := pool.Exec(ctx, ` INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p6','alist','P6 来源',true); INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p6',repeat('a',64),10,'verified/p6.pdf','verified'); INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin) VALUES('item-p6','conn-p6','ext-p6',repeat('d',64),'P6.pdf','document','r1','internal','manual'); INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES ('siv-1','item-p6','r1','co-p6','verified'), ('siv-2','item-p6','r2','co-p6','verified'), ('siv-3','item-p6','r3','co-p6','verified'), ('siv-4','item-p6','r4','co-p6','verified'); INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,updated_at) VALUES ('tp-1','siv-1','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), ('tp-2','siv-2','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), ('tp-3','siv-3','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), ('tp-5','siv-4','kb-1','uploaded','processing','unpublished','internal','2026-09-12T01:00:00Z');`) if seedErr != nil { t.Fatalf("seed: %v", seedErr) } store := NewStore(pool) base := time.Date(2026, time.September, 12, 1, 0, 0, 0, time.UTC) // 1) Ordinary failure exhausting the budget projects into the DLQ. uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-1"}) if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-1','test.upload',$1,'running',5,5,'k-dlq-1','w1','cs-p6','siv-1','kb-1','upload',$2)`, uploadPayload, base); err != nil { t.Fatalf("seed job: %v", err) } if err := store.Fail(ctx, "job-dlq-1", "w1", "remote parse failed", "validation_failed", base, base.Add(time.Minute)); err != nil { t.Fatalf("exhausting fail: %v", err) } var jobState, errorCode, failureState string var attempt int if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState, &errorCode); err != nil { t.Fatal(err) } if jobState != "failed" || errorCode != "validation_failed" { t.Fatalf("exhausted job = (%s,%s)", jobState, errorCode) } var failedPublicationID string if err := pool.QueryRow(ctx, `SELECT state,attempt,error_code,publication_id FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &attempt, &errorCode, &failedPublicationID); err != nil { t.Fatalf("dlq projection missing: %v", err) } if failureState != "open" || attempt != 5 || errorCode != "validation_failed" || failedPublicationID != "tp-1" { t.Fatalf("dlq row = (%s,%d,%s,%s)", failureState, attempt, errorCode, failedPublicationID) } // 2) Permanent failure parks immediately and projects. removePayload, _ := json.Marshal(map[string]string{"publication_id": "tp-3"}) if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-3','test.remove',$1,'running',1,10,'k-dlq-3','w1','cs-p6','siv-3','','remove',$2)`, removePayload, base); err != nil { t.Fatalf("seed remove job: %v", err) } var _ jobs.TerminalFailer = store if err := store.FailTerminal(ctx, "job-dlq-3", "w1", "remote rejected", "remote_permanent", base); err != nil { t.Fatalf("terminal fail: %v", err) } if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-3'`).Scan(&jobState, &errorCode); err != nil { t.Fatal(err) } if jobState != "failed" || errorCode != "remote_permanent" { t.Fatalf("terminal job = (%s,%s)", jobState, errorCode) } // 3) DLQ query lists both open projections. page, err := store.ListJobFailures(ctx, governance.JobFailureFilter{Limit: 50}) if err != nil { t.Fatalf("list failures: %v", err) } if page.Total != 2 || len(page.Items) != 2 { t.Fatalf("open failures = %d/%d, want 2", len(page.Items), page.Total) } // 4) Server-side batch retry: tp-1 + tp-2 started, unknown id not_retryable. result, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"}, OperatorRef: "姚工", Reason: "解析服务恢复", Now: base.Add(2 * time.Minute), }, "op-retry-batch-1") if err != nil || replayed { t.Fatalf("batch retry: %v replayed=%v", err, replayed) } if result.Total != 4 || result.Started != 2 || result.NotRetryable != 1 || result.AlreadyRunning != 1 { t.Fatalf("batch result = %+v", result) } if result.Outcomes[0].PublicationID != "tp-1" || result.Outcomes[0].Result != governance.RetryStarted { t.Fatalf("tp-1 outcome = %+v", result.Outcomes[0]) } if result.Outcomes[1].PublicationID != "tp-2" || result.Outcomes[1].Result != governance.RetryStarted { t.Fatalf("tp-2 outcome = %+v", result.Outcomes[1]) } if result.Outcomes[2].PublicationID != "tp-4-missing" || result.Outcomes[2].Result != governance.RetryNotRetryable || result.Outcomes[2].Detail != "publication not found" { t.Fatalf("missing outcome = %+v", result.Outcomes[2]) } if result.Outcomes[3].PublicationID != "tp-5" || result.Outcomes[3].Result != governance.RetryAlreadyRunning { t.Fatalf("tp-5 outcome = %+v", result.Outcomes[3]) } // tp-1's open DLQ row flipped to requeued with operator and reason. var actor, reason string if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &actor, &reason); err != nil { t.Fatal(err) } if failureState != "requeued" || actor != "姚工" || reason != "解析服务恢复" { t.Fatalf("tp-1 failure after retry = (%s,%s,%s)", failureState, actor, reason) } // tp-3's failure stays open (not part of this batch). if err := pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE job_id='job-dlq-3'`).Scan(&failureState); err != nil || failureState != "open" { t.Fatalf("tp-3 failure state = %s err=%v", failureState, err) } // Audit rows recorded atomically for every selection. var auditCount int if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM job_retry_operations WHERE batch_key='op-retry-batch-1'`).Scan(&auditCount); err != nil || auditCount != 4 { t.Fatalf("retry audit rows = %d err=%v", auditCount, err) } // Fresh retry jobs exist and the original exhausted job stays untouched. var newJobs int if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND state='pending' AND idempotency_key LIKE 'test-retry:tp-1:%'`).Scan(&newJobs); err != nil || newJobs != 1 { t.Fatalf("tp-1 retry jobs = %d err=%v", newJobs, err) } if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState); err != nil || jobState != "failed" { t.Fatalf("original job state = %q err=%v, want failed (untouched)", jobState, err) } // 5) Replaying the same operation id returns the identical stored result. replay, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"}, OperatorRef: "姚工", Reason: "解析服务恢复", Now: base.Add(9 * time.Minute), }, "op-retry-batch-1") if err != nil || !replayed { t.Fatalf("replay: %v replayed=%v", err, replayed) } if replay.Started != result.Started || replay.Total != result.Total { t.Fatalf("replay result = %+v, want %+v", replay, result) } // 6) Same key with a different body conflicts. if _, _, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ PublicationIDs: []string{"tp-2"}, OperatorRef: "姚工", Reason: "解析服务恢复", Now: base.Add(10 * time.Minute), }, "op-retry-batch-1"); err == nil { t.Fatal("expected operation request conflict for same key different body") } }