knowledge-governance/internal/platform/postgres/p6_dlq_integration_test.go

package postgres

import (
	"context"
	"encoding/json"
	"os"
	"testing"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"
	"knowledge-governance/internal/governance"
	"knowledge-governance/internal/jobs"
)

// TestP6DeadLetterAndServerSideBatchRetry freezes docs plan P6: an ordinary
// failure that exhausts its budget lands in the DLQ projection without
// touching the original audit row, a permanent failure lands immediately, and
// the server-side batch retry returns mixed outcomes atomically with replay.
func TestP6DeadLetterAndServerSideBatchRetry(t *testing.T) {
	databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
	if databaseURL == "" {
		t.Skip("TEST_DATABASE_URL is not set")
	}
	ctx := context.Background()
	pool, err := pgxpool.New(ctx, databaseURL)
	if err != nil {
		t.Fatalf("connect: %v", err)
	}
	defer pool.Close()
	if err := ApplyMigrations(ctx, pool); err != nil {
		t.Fatalf("migrate: %v", err)
	}
	_, seedErr := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p6','alist','P6 来源',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p6',repeat('a',64),10,'verified/p6.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
VALUES('item-p6','conn-p6','ext-p6',repeat('d',64),'P6.pdf','document','r1','internal','manual');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES
 ('siv-1','item-p6','r1','co-p6','verified'),
 ('siv-2','item-p6','r2','co-p6','verified'),
 ('siv-3','item-p6','r3','co-p6','verified'),
 ('siv-4','item-p6','r4','co-p6','verified');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,updated_at) VALUES
 ('tp-1','siv-1','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
 ('tp-2','siv-2','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
 ('tp-3','siv-3','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
 ('tp-5','siv-4','kb-1','uploaded','processing','unpublished','internal','2026-09-12T01:00:00Z');`)
	if seedErr != nil {
		t.Fatalf("seed: %v", seedErr)
	}
	store := NewStore(pool)
	base := time.Date(2026, time.September, 12, 1, 0, 0, 0, time.UTC)

	// 1) Ordinary failure exhausting the budget projects into the DLQ.
	uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-1"})
	if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-1','test.upload',$1,'running',5,5,'k-dlq-1','w1','cs-p6','siv-1','kb-1','upload',$2)`, uploadPayload, base); err != nil {
		t.Fatalf("seed job: %v", err)
	}
	if err := store.Fail(ctx, "job-dlq-1", "w1", "remote parse failed", "validation_failed", base, base.Add(time.Minute)); err != nil {
		t.Fatalf("exhausting fail: %v", err)
	}
	var jobState, errorCode, failureState string
	var attempt int
	if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState, &errorCode); err != nil {
		t.Fatal(err)
	}
	if jobState != "failed" || errorCode != "validation_failed" {
		t.Fatalf("exhausted job = (%s,%s)", jobState, errorCode)
	}
	var failedPublicationID string
	if err := pool.QueryRow(ctx, `SELECT state,attempt,error_code,publication_id FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &attempt, &errorCode, &failedPublicationID); err != nil {
		t.Fatalf("dlq projection missing: %v", err)
	}
	if failureState != "open" || attempt != 5 || errorCode != "validation_failed" || failedPublicationID != "tp-1" {
		t.Fatalf("dlq row = (%s,%d,%s,%s)", failureState, attempt, errorCode, failedPublicationID)
	}

	// 2) Permanent failure parks immediately and projects.
	removePayload, _ := json.Marshal(map[string]string{"publication_id": "tp-3"})
	if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-3','test.remove',$1,'running',1,10,'k-dlq-3','w1','cs-p6','siv-3','','remove',$2)`, removePayload, base); err != nil {
		t.Fatalf("seed remove job: %v", err)
	}
	var _ jobs.TerminalFailer = store
	if err := store.FailTerminal(ctx, "job-dlq-3", "w1", "remote rejected", "remote_permanent", base); err != nil {
		t.Fatalf("terminal fail: %v", err)
	}
	if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-3'`).Scan(&jobState, &errorCode); err != nil {
		t.Fatal(err)
	}
	if jobState != "failed" || errorCode != "remote_permanent" {
		t.Fatalf("terminal job = (%s,%s)", jobState, errorCode)
	}

	// 3) DLQ query lists both open projections.
	page, err := store.ListJobFailures(ctx, governance.JobFailureFilter{Limit: 50})
	if err != nil {
		t.Fatalf("list failures: %v", err)
	}
	if page.Total != 2 || len(page.Items) != 2 {
		t.Fatalf("open failures = %d/%d, want 2", len(page.Items), page.Total)
	}

	// 4) Server-side batch retry: tp-1 + tp-2 started, unknown id not_retryable.
	result, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
		PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
		OperatorRef:    "姚工",
		Reason:         "解析服务恢复",
		Now:            base.Add(2 * time.Minute),
	}, "op-retry-batch-1")
	if err != nil || replayed {
		t.Fatalf("batch retry: %v replayed=%v", err, replayed)
	}
	if result.Total != 4 || result.Started != 2 || result.NotRetryable != 1 || result.AlreadyRunning != 1 {
		t.Fatalf("batch result = %+v", result)
	}
	if result.Outcomes[0].PublicationID != "tp-1" || result.Outcomes[0].Result != governance.RetryStarted {
		t.Fatalf("tp-1 outcome = %+v", result.Outcomes[0])
	}
	if result.Outcomes[1].PublicationID != "tp-2" || result.Outcomes[1].Result != governance.RetryStarted {
		t.Fatalf("tp-2 outcome = %+v", result.Outcomes[1])
	}
	if result.Outcomes[2].PublicationID != "tp-4-missing" || result.Outcomes[2].Result != governance.RetryNotRetryable || result.Outcomes[2].Detail != "publication not found" {
		t.Fatalf("missing outcome = %+v", result.Outcomes[2])
	}
	if result.Outcomes[3].PublicationID != "tp-5" || result.Outcomes[3].Result != governance.RetryAlreadyRunning {
		t.Fatalf("tp-5 outcome = %+v", result.Outcomes[3])
	}

	// tp-1's open DLQ row flipped to requeued with operator and reason.
	var actor, reason string
	if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &actor, &reason); err != nil {
		t.Fatal(err)
	}
	if failureState != "requeued" || actor != "姚工" || reason != "解析服务恢复" {
		t.Fatalf("tp-1 failure after retry = (%s,%s,%s)", failureState, actor, reason)
	}
	// tp-3's failure stays open (not part of this batch).
	if err := pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE job_id='job-dlq-3'`).Scan(&failureState); err != nil || failureState != "open" {
		t.Fatalf("tp-3 failure state = %s err=%v", failureState, err)
	}

	// Audit rows recorded atomically for every selection.
	var auditCount int
	if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM job_retry_operations WHERE batch_key='op-retry-batch-1'`).Scan(&auditCount); err != nil || auditCount != 4 {
		t.Fatalf("retry audit rows = %d err=%v", auditCount, err)
	}
	// Fresh retry jobs exist and the original exhausted job stays untouched.
	var newJobs int
	if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND state='pending' AND idempotency_key LIKE 'test-retry:tp-1:%'`).Scan(&newJobs); err != nil || newJobs != 1 {
		t.Fatalf("tp-1 retry jobs = %d err=%v", newJobs, err)
	}
	if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState); err != nil || jobState != "failed" {
		t.Fatalf("original job state = %q err=%v, want failed (untouched)", jobState, err)
	}

	// 5) Replaying the same operation id returns the identical stored result.
	replay, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
		PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
		OperatorRef:    "姚工",
		Reason:         "解析服务恢复",
		Now:            base.Add(9 * time.Minute),
	}, "op-retry-batch-1")
	if err != nil || !replayed {
		t.Fatalf("replay: %v replayed=%v", err, replayed)
	}
	if replay.Started != result.Started || replay.Total != result.Total {
		t.Fatalf("replay result = %+v, want %+v", replay, result)
	}

	// 6) Same key with a different body conflicts.
	if _, _, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
		PublicationIDs: []string{"tp-2"},
		OperatorRef:    "姚工",
		Reason:         "解析服务恢复",
		Now:            base.Add(10 * time.Minute),
	}, "op-retry-batch-1"); err == nil {
		t.Fatal("expected operation request conflict for same key different body")
	}
}