package postgres import ( "context" "encoding/json" "errors" "fmt" "github.com/jackc/pgx/v5" "knowledge-governance/internal/governance" ) // ErrJobFailureNotRequeueable marks a dead letter that cannot be re-enqueued: // not an artifact fetch, already resolved, or missing its source coordinates. var ErrJobFailureNotRequeueable = errors.New("job failure is not requeueable") const loadRequeueableArtifactFailureSQL = ` SELECT jf.state, jf.job_type, COALESCE(j.payload->>'source_connection_id',''), COALESCE(j.payload->>'node_id',''), COALESCE(j.payload->>'version_id',''), COALESCE(NULLIF(intent.name,''),'') FROM job_failures jf LEFT JOIN jobs j ON j.id = jf.job_id LEFT JOIN catalog_import_intents intent ON intent.source_connection_id = j.payload->>'source_connection_id' AND intent.node_id = j.payload->>'node_id' WHERE jf.id = $1 FOR UPDATE OF jf ` const countArtifactFetchRoundsSQL = ` SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch' AND payload->>'version_id' = $1 ` const requeueArtifactFetchJobSQL = ` INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at) VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING ` const resetIntentFetchStateSQL = ` UPDATE catalog_import_intents SET fetch_state='pending', last_error='', updated_at=$3 WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching') ` const resolveRequeuedFailureSQL = ` UPDATE job_failures SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4 WHERE id = $1 AND state = 'open' ` // RequeueArtifactFetchWithOperation re-enqueues one terminal 源头取件 dead // letter under a new round-suffixed idempotency key, resets the intent's fetch // state and closes the dead letter — all in one operation envelope. func (s *Store) RequeueArtifactFetchWithOperation(ctx context.Context, request governance.RequeueArtifactFetchRequest, operationID string) (governance.RequeueArtifactFetchResult, bool, error) { if operationID == "" || len(operationID) > 128 { return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid operation id") } if request.FailureID == "" || len(request.FailureID) > 64 { return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid failure id") } if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 500 || request.Now.IsZero() { return governance.RequeueArtifactFetchResult{}, false, errors.New("actor_ref and a non-empty reason are required") } fingerprint, err := operationFingerprint(struct { FailureID string `json:"failure_id"` OperatorRef string `json:"operator_ref"` Reason string `json:"reason"` }{request.FailureID, request.OperatorRef, request.Reason}) if err != nil { return governance.RequeueArtifactFetchResult{}, false, err } raw, replayed, err := s.executeOperation(ctx, "artifact.fetch.requeue", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { result, err := requeueArtifactFetchTx(ctx, tx, request) if err != nil { return nil, err } return json.Marshal(result) }) if err != nil { return governance.RequeueArtifactFetchResult{}, false, err } var result governance.RequeueArtifactFetchResult if err := json.Unmarshal(raw, &result); err != nil { return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid stored requeue result") } return result, replayed, nil } func requeueArtifactFetchTx(ctx context.Context, tx pgx.Tx, request governance.RequeueArtifactFetchRequest) (governance.RequeueArtifactFetchResult, error) { var state, jobType, connectionID, nodeID, versionID, docName string err := tx.QueryRow(ctx, loadRequeueableArtifactFailureSQL, request.FailureID). Scan(&state, &jobType, &connectionID, &nodeID, &versionID, &docName) if errors.Is(err, pgx.ErrNoRows) { return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable } if err != nil { return governance.RequeueArtifactFetchResult{}, fmt.Errorf("load requeueable failure: %w", err) } if jobType != "artifact.fetch" || state != "open" || connectionID == "" || nodeID == "" || versionID == "" { return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable } var rounds int if err := tx.QueryRow(ctx, countArtifactFetchRoundsSQL, versionID).Scan(&rounds); err != nil { return governance.RequeueArtifactFetchResult{}, fmt.Errorf("count artifact fetch rounds: %w", err) } round := rounds + 1 jobID, err := newOpaqueID("job-") if err != nil { return governance.RequeueArtifactFetchResult{}, err } payload, err := json.Marshal(map[string]string{ "source_connection_id": connectionID, "node_id": nodeID, "version_id": versionID, }) if err != nil { return governance.RequeueArtifactFetchResult{}, errors.New("encode artifact fetch job") } tag, err := tx.Exec(ctx, requeueArtifactFetchJobSQL, jobID, string(payload), request.Now, fmt.Sprintf("artifact-fetch:%s:r%d", versionID, round)) if err != nil { return governance.RequeueArtifactFetchResult{}, fmt.Errorf("requeue artifact fetch: %w", err) } result := governance.RequeueArtifactFetchResult{ FailureID: request.FailureID, JobID: jobID, VersionID: versionID, Round: round, DocName: docName, } if tag.RowsAffected() == 0 { // The round key already exists (concurrent requeue): report it instead of // resolving the dead letter twice. result.AlreadyQueued = true return result, nil } if _, err := tx.Exec(ctx, resetIntentFetchStateSQL, connectionID, nodeID, request.Now); err != nil { return governance.RequeueArtifactFetchResult{}, fmt.Errorf("reset intent fetch state: %w", err) } if _, err := tx.Exec(ctx, resolveRequeuedFailureSQL, request.FailureID, request.Now, request.OperatorRef, request.Reason); err != nil { return governance.RequeueArtifactFetchResult{}, fmt.Errorf("resolve job failure: %w", err) } return result, nil } const listJobFailuresSQL = ` SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code, jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id, jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at, COALESCE(siv.source_item_id,''), COALESCE(NULLIF(item.title,''), NULLIF(ci.name,''), ''), COALESCE(NULLIF(jf.last_error,''), j.last_error, '') FROM job_failures jf LEFT JOIN test_publications p ON p.id = jf.publication_id LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id LEFT JOIN source_items item ON item.id = siv.source_item_id LEFT JOIN jobs j ON j.id = jf.job_id LEFT JOIN LATERAL ( SELECT intent.name FROM catalog_import_intents intent WHERE intent.source_connection_id = j.payload->>'source_connection_id' AND intent.node_id = j.payload->>'node_id' LIMIT 1 ) ci ON TRUE WHERE ($1 = '' OR jf.state = $1) ORDER BY jf.created_at DESC, jf.id DESC LIMIT $2 OFFSET $3 ` const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)` // ListJobFailures is the DLQ view (P6): projections of terminal attempts. The // original jobs and publication audit rows are never moved or deleted. func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) { state := filter.State if state == "" { state = "open" } if state != "open" && state != "requeued" && state != "abandoned" { return governance.JobFailurePage{}, errors.New("invalid failure state filter") } limit, offset := filter.Limit, filter.Offset if limit <= 0 { limit = 50 } if limit > 100 { return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100") } if offset < 0 { offset = 0 } var total int if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil { return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err) } rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset) if err != nil { return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err) } defer rows.Close() items := make([]governance.JobFailure, 0, limit) for rows.Next() { var item governance.JobFailure if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode, &item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID, &item.State, &item.CreatedAt, &item.SourceItemID, &item.DocName, &item.LastError); err != nil { return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err) } items = append(items, item) } if err := rows.Err(); err != nil { return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err) } return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil } const failurePublicationStateSQL = ` SELECT p.processing_state, p.ingest_state, EXISTS ( SELECT 1 FROM source_item_versions siv JOIN content_objects co ON co.id = siv.content_object_id WHERE siv.id = p.source_item_version_id AND co.content_state = 'verified' AND co.local_spool_ref <> '' ) FROM test_publications p WHERE p.id = $1 ` const resolvePublicationFailuresSQL = ` UPDATE job_failures SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4 WHERE publication_id = $1 AND state = 'open' ` // abandonPublicationFailuresSQL closes open dead letters the same way retry // does, but records the operator's "stop trying" decision instead. const abandonPublicationFailuresSQL = ` UPDATE job_failures SET state = 'abandoned', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4 WHERE publication_id = $1 AND state = 'open' ` // cancelPublicationSQL moves a failed/stalled publication out of the retry // loop. Rows already cancelled are left alone (idempotent replay). const cancelPublicationSQL = ` UPDATE test_publications SET processing_state = 'cancelled', updated_at = $2 WHERE id = $1 AND superseded_at IS NULL AND (processing_state IN ('failed', 'stalled') OR (ingest_state = 'failed' AND processing_state = 'pending')) ` const cancelQueuedUploadJobsSQL = ` UPDATE jobs SET state = 'cancelled', lease_owner = NULL, lease_until = NULL, last_error = 'cancelled by operator', last_error_code = '', updated_at = $2 WHERE type = 'test.upload' AND payload->>'publication_id' = $1 AND state IN ('pending', 'retry') ` // CancelPublicationsBatchWithOperation moves 1..100 failing publications to // 'cancelled' in one operation envelope: per-item outcome, dead letters // abandoned, queued upload jobs dropped, one audit row per item. func (s *Store) CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) { if operationID == "" || len(operationID) > 128 { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id") } if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items") } if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() { return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required") } seen := make(map[string]bool, len(request.PublicationIDs)) ids := make([]string, 0, len(request.PublicationIDs)) for _, id := range request.PublicationIDs { if id == "" || len(id) > 64 || seen[id] { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry") } seen[id] = true ids = append(ids, id) } fingerprint, err := operationFingerprint(struct { PublicationIDs []string `json:"publication_ids"` OperatorRef string `json:"operator_ref"` Reason string `json:"reason"` Kind string `json:"kind"` }{ids, request.OperatorRef, request.Reason, "cancel"}) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } raw, replayed, err := s.executeOperation(ctx, "test.publication.cancel_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { result, err := cancelPublicationsBatchTx(ctx, tx, request, ids, operationID) if err != nil { return nil, err } return json.Marshal(result) }) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } var result governance.RetryPublicationsBatchResult if err := json.Unmarshal(raw, &result); err != nil { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored cancel batch result") } return result, replayed, nil } func cancelPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.CancelPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) { sorted := append([]string(nil), ids...) for i := 1; i < len(sorted); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))} for _, id := range sorted { var processingState, ingestState string err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, new(bool)) outcome := governance.RetryOutcome{PublicationID: id} switch { case errors.Is(err, pgx.ErrNoRows): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication not found" case err != nil: return result, fmt.Errorf("classify publication cancel: %w", err) case processingState == "processing" || processingState == "finalizing": outcome.Result = governance.RetryAlreadyRunning case processingState == "cancelled": // Idempotent replay of a previous cancel: count it as done. outcome.Result = governance.CancelCancelled case processingState == "completed": outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication already completed" case processingState == "planned" || (processingState == "pending" && ingestState != "failed"): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication is not in a failed state" default: tag, err := tx.Exec(ctx, cancelPublicationSQL, id, request.Now) if err != nil { return result, fmt.Errorf("cancel publication %s: %w", id, err) } if tag.RowsAffected() == 0 { outcome.Result = governance.RetryAlreadyRunning } else { if _, err := tx.Exec(ctx, cancelQueuedUploadJobsSQL, id, request.Now); err != nil { return result, fmt.Errorf("cancel queued upload jobs: %w", err) } if _, err := tx.Exec(ctx, abandonPublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil { return result, fmt.Errorf("abandon job failures: %w", err) } outcome.Result = governance.CancelCancelled } } if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil { return result, fmt.Errorf("record cancel audit: %w", err) } switch outcome.Result { case governance.CancelCancelled: result.Started++ case governance.RetryAlreadyRunning: result.AlreadyRunning++ default: result.NotRetryable++ } result.Outcomes = append(result.Outcomes, outcome) } result.Total = len(sorted) return result, nil } const recordRetryAuditSQL = ` INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (batch_key, publication_id) DO NOTHING ` // RetryPublicationsBatchWithOperation performs the whole selection server-side // (P6): one operation envelope, one transaction, per-item outcome, atomic audit. func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) { if operationID == "" || len(operationID) > 128 { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id") } if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items") } if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() { return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required") } seen := make(map[string]bool, len(request.PublicationIDs)) ids := make([]string, 0, len(request.PublicationIDs)) for _, id := range request.PublicationIDs { if id == "" || len(id) > 64 || seen[id] { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry") } seen[id] = true ids = append(ids, id) } fingerprint, err := operationFingerprint(struct { PublicationIDs []string `json:"publication_ids"` OperatorRef string `json:"operator_ref"` Reason string `json:"reason"` }{ids, request.OperatorRef, request.Reason}) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID) if err != nil { return nil, err } return json.Marshal(result) }) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } var result governance.RetryPublicationsBatchResult if err := json.Unmarshal(raw, &result); err != nil { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result") } return result, replayed, nil } func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) { // Stable lock order prevents cross-batch deadlocks. sorted := append([]string(nil), ids...) for i := 1; i < len(sorted); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))} store := NewStore(tx) for _, id := range sorted { var processingState, ingestState string var spoolAvailable bool err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable) outcome := governance.RetryOutcome{PublicationID: id} switch { case errors.Is(err, pgx.ErrNoRows): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication not found" case err != nil: return result, fmt.Errorf("classify publication retry: %w", err) case processingState == "processing" || processingState == "finalizing": outcome.Result = governance.RetryAlreadyRunning case processingState == "pending" && ingestState != "failed": outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication is already queued" case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication is not in a failed state" case !spoolAvailable: outcome.Result = governance.RetryNotRetryable outcome.Detail = "no verified spool artifact" default: if err := store.RetryPublication(ctx, id, request.Now); err != nil { return result, fmt.Errorf("retry publication %s: %w", id, err) } if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil { return result, fmt.Errorf("resolve job failures: %w", err) } outcome.Result = governance.RetryStarted } if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil { return result, fmt.Errorf("record retry audit: %w", err) } switch outcome.Result { case governance.RetryStarted: result.Started++ case governance.RetryAlreadyRunning: result.AlreadyRunning++ default: result.NotRetryable++ } result.Outcomes = append(result.Outcomes, outcome) } result.Total = len(sorted) return result, nil }