package postgres import ( "context" "encoding/json" "errors" "fmt" "github.com/jackc/pgx/v5" "knowledge-governance/internal/governance" ) const listJobFailuresSQL = ` SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code, jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id, jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at, COALESCE(siv.source_item_id,'') FROM job_failures jf LEFT JOIN test_publications p ON p.id = jf.publication_id LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id WHERE ($1 = '' OR jf.state = $1) ORDER BY jf.created_at DESC, jf.id DESC LIMIT $2 OFFSET $3 ` const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)` // ListJobFailures is the DLQ view (P6): projections of terminal attempts. The // original jobs and publication audit rows are never moved or deleted. func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) { state := filter.State if state == "" { state = "open" } if state != "open" && state != "requeued" && state != "abandoned" { return governance.JobFailurePage{}, errors.New("invalid failure state filter") } limit, offset := filter.Limit, filter.Offset if limit <= 0 { limit = 50 } if limit > 100 { return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100") } if offset < 0 { offset = 0 } var total int if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil { return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err) } rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset) if err != nil { return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err) } defer rows.Close() items := make([]governance.JobFailure, 0, limit) for rows.Next() { var item governance.JobFailure if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode, &item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID, &item.State, &item.CreatedAt, &item.SourceItemID); err != nil { return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err) } items = append(items, item) } if err := rows.Err(); err != nil { return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err) } return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil } const failurePublicationStateSQL = ` SELECT p.processing_state, p.ingest_state, EXISTS ( SELECT 1 FROM source_item_versions siv JOIN content_objects co ON co.id = siv.content_object_id WHERE siv.id = p.source_item_version_id AND co.content_state = 'verified' AND co.local_spool_ref <> '' ) FROM test_publications p WHERE p.id = $1 ` const resolvePublicationFailuresSQL = ` UPDATE job_failures SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4 WHERE publication_id = $1 AND state = 'open' ` const recordRetryAuditSQL = ` INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (batch_key, publication_id) DO NOTHING ` // RetryPublicationsBatchWithOperation performs the whole selection server-side // (P6): one operation envelope, one transaction, per-item outcome, atomic audit. func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) { if operationID == "" || len(operationID) > 128 { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id") } if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items") } if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() { return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required") } seen := make(map[string]bool, len(request.PublicationIDs)) ids := make([]string, 0, len(request.PublicationIDs)) for _, id := range request.PublicationIDs { if id == "" || len(id) > 64 || seen[id] { return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry") } seen[id] = true ids = append(ids, id) } fingerprint, err := operationFingerprint(struct { PublicationIDs []string `json:"publication_ids"` OperatorRef string `json:"operator_ref"` Reason string `json:"reason"` }{ids, request.OperatorRef, request.Reason}) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID) if err != nil { return nil, err } return json.Marshal(result) }) if err != nil { return governance.RetryPublicationsBatchResult{}, false, err } var result governance.RetryPublicationsBatchResult if err := json.Unmarshal(raw, &result); err != nil { return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result") } return result, replayed, nil } func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) { // Stable lock order prevents cross-batch deadlocks. sorted := append([]string(nil), ids...) for i := 1; i < len(sorted); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))} store := NewStore(tx) for _, id := range sorted { var processingState, ingestState string var spoolAvailable bool err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable) outcome := governance.RetryOutcome{PublicationID: id} switch { case errors.Is(err, pgx.ErrNoRows): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication not found" case err != nil: return result, fmt.Errorf("classify publication retry: %w", err) case processingState == "processing" || processingState == "finalizing": outcome.Result = governance.RetryAlreadyRunning case processingState == "pending" && ingestState != "failed": outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication is already queued" case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"): outcome.Result = governance.RetryNotRetryable outcome.Detail = "publication is not in a failed state" case !spoolAvailable: outcome.Result = governance.RetryNotRetryable outcome.Detail = "no verified spool artifact" default: if err := store.RetryPublication(ctx, id, request.Now); err != nil { return result, fmt.Errorf("retry publication %s: %w", id, err) } if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil { return result, fmt.Errorf("resolve job failures: %w", err) } outcome.Result = governance.RetryStarted } if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil { return result, fmt.Errorf("record retry audit: %w", err) } switch outcome.Result { case governance.RetryStarted: result.Started++ case governance.RetryAlreadyRunning: result.AlreadyRunning++ default: result.NotRetryable++ } result.Outcomes = append(result.Outcomes, outcome) } result.Total = len(sorted) return result, nil }