Files
knowledge-governance/internal/platform/postgres/job_failures.go
T

212 lines
8.6 KiB
Go

package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
const listJobFailuresSQL = `
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at,
COALESCE(siv.source_item_id,''),
COALESCE(NULLIF(item.title,''), NULLIF(ci.name,''), ''),
COALESCE(NULLIF(jf.last_error,''), j.last_error, '')
FROM job_failures jf
LEFT JOIN test_publications p ON p.id = jf.publication_id
LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id
LEFT JOIN source_items item ON item.id = siv.source_item_id
LEFT JOIN jobs j ON j.id = jf.job_id
LEFT JOIN LATERAL (
SELECT intent.name FROM catalog_import_intents intent
WHERE intent.source_connection_id = j.payload->>'source_connection_id'
AND intent.node_id = j.payload->>'node_id'
LIMIT 1
) ci ON TRUE
WHERE ($1 = '' OR jf.state = $1)
ORDER BY jf.created_at DESC, jf.id DESC
LIMIT $2 OFFSET $3
`
const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)`
// ListJobFailures is the DLQ view (P6): projections of terminal attempts. The
// original jobs and publication audit rows are never moved or deleted.
func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) {
state := filter.State
if state == "" {
state = "open"
}
if state != "open" && state != "requeued" && state != "abandoned" {
return governance.JobFailurePage{}, errors.New("invalid failure state filter")
}
limit, offset := filter.Limit, filter.Offset
if limit <= 0 {
limit = 50
}
if limit > 100 {
return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100")
}
if offset < 0 {
offset = 0
}
var total int
if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err)
}
rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset)
if err != nil {
return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err)
}
defer rows.Close()
items := make([]governance.JobFailure, 0, limit)
for rows.Next() {
var item governance.JobFailure
if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode,
&item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID,
&item.State, &item.CreatedAt, &item.SourceItemID, &item.DocName, &item.LastError); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err)
}
return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
const failurePublicationStateSQL = `
SELECT p.processing_state, p.ingest_state,
EXISTS (
SELECT 1 FROM source_item_versions siv
JOIN content_objects co ON co.id = siv.content_object_id
WHERE siv.id = p.source_item_version_id
AND co.content_state = 'verified'
AND co.local_spool_ref <> ''
)
FROM test_publications p
WHERE p.id = $1
`
const resolvePublicationFailuresSQL = `
UPDATE job_failures
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
WHERE publication_id = $1 AND state = 'open'
`
const recordRetryAuditSQL = `
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (batch_key, publication_id) DO NOTHING
`
// RetryPublicationsBatchWithOperation performs the whole selection server-side
// (P6): one operation envelope, one transaction, per-item outcome, atomic audit.
func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
if operationID == "" || len(operationID) > 128 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
}
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
}
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
}
seen := make(map[string]bool, len(request.PublicationIDs))
ids := make([]string, 0, len(request.PublicationIDs))
for _, id := range request.PublicationIDs {
if id == "" || len(id) > 64 || seen[id] {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
}
seen[id] = true
ids = append(ids, id)
}
fingerprint, err := operationFingerprint(struct {
PublicationIDs []string `json:"publication_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{ids, request.OperatorRef, request.Reason})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
var result governance.RetryPublicationsBatchResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result")
}
return result, replayed, nil
}
func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
// Stable lock order prevents cross-batch deadlocks.
sorted := append([]string(nil), ids...)
for i := 1; i < len(sorted); i++ {
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
}
}
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
store := NewStore(tx)
for _, id := range sorted {
var processingState, ingestState string
var spoolAvailable bool
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable)
outcome := governance.RetryOutcome{PublicationID: id}
switch {
case errors.Is(err, pgx.ErrNoRows):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication not found"
case err != nil:
return result, fmt.Errorf("classify publication retry: %w", err)
case processingState == "processing" || processingState == "finalizing":
outcome.Result = governance.RetryAlreadyRunning
case processingState == "pending" && ingestState != "failed":
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication is already queued"
case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication is not in a failed state"
case !spoolAvailable:
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "no verified spool artifact"
default:
if err := store.RetryPublication(ctx, id, request.Now); err != nil {
return result, fmt.Errorf("retry publication %s: %w", id, err)
}
if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
return result, fmt.Errorf("resolve job failures: %w", err)
}
outcome.Result = governance.RetryStarted
}
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
return result, fmt.Errorf("record retry audit: %w", err)
}
switch outcome.Result {
case governance.RetryStarted:
result.Started++
case governance.RetryAlreadyRunning:
result.AlreadyRunning++
default:
result.NotRetryable++
}
result.Outcomes = append(result.Outcomes, outcome)
}
result.Total = len(sorted)
return result, nil
}