hermes
cddc4c90d1
- platform/postgres: listTestPublicationsSQL 的 can_start/can_retry 在 LEFT JOIN
内容对象为空时算出 NULL，Go 侧 bool 扫描失败使整个列表接口 500，前端三个 lane
全空（表现为“待入库 0 / 全部任务 0”）。改为 COALESCE(..., false)。
- 新增 POST /api/v1/job-failures/{id}/requeue：取件任务的幂等键绑在版本上，
失败那次已占住它，重新标注同一文件不会再入队；现在按新轮次 (…:r<n>) 重排、
复位意图取件状态并把死信标记为 requeued（幂等信封 + 审计字段）。
- web: 死信区对 artifact.fetch 显示「重新入队」；系统设置新增「知识库绑定」分区，
挂上一直没被渲染的 RoutingBindingsSection，导入目标库可在页面上切换。
- 测试：真实 PostgreSQL 集成测试（NULL 折叠 + 重排轮次/幂等/拒绝）、前端死信按钮
与设置分区渲染测试。
479 lines
20 KiB
Go
479 lines
20 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
// ErrJobFailureNotRequeueable marks a dead letter that cannot be re-enqueued:
|
|
// not an artifact fetch, already resolved, or missing its source coordinates.
|
|
var ErrJobFailureNotRequeueable = errors.New("job failure is not requeueable")
|
|
|
|
const loadRequeueableArtifactFailureSQL = `
|
|
SELECT jf.state, jf.job_type,
|
|
COALESCE(j.payload->>'source_connection_id',''),
|
|
COALESCE(j.payload->>'node_id',''),
|
|
COALESCE(j.payload->>'version_id',''),
|
|
COALESCE(NULLIF(intent.name,''),'')
|
|
FROM job_failures jf
|
|
LEFT JOIN jobs j ON j.id = jf.job_id
|
|
LEFT JOIN catalog_import_intents intent
|
|
ON intent.source_connection_id = j.payload->>'source_connection_id'
|
|
AND intent.node_id = j.payload->>'node_id'
|
|
WHERE jf.id = $1
|
|
FOR UPDATE OF jf
|
|
`
|
|
|
|
const countArtifactFetchRoundsSQL = `
|
|
SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch' AND payload->>'version_id' = $1
|
|
`
|
|
|
|
const requeueArtifactFetchJobSQL = `
|
|
INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
|
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3)
|
|
ON CONFLICT (idempotency_key) DO NOTHING
|
|
`
|
|
|
|
const resetIntentFetchStateSQL = `
|
|
UPDATE catalog_import_intents SET fetch_state='pending', last_error='', updated_at=$3
|
|
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')
|
|
`
|
|
|
|
const resolveRequeuedFailureSQL = `
|
|
UPDATE job_failures
|
|
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
|
WHERE id = $1 AND state = 'open'
|
|
`
|
|
|
|
// RequeueArtifactFetchWithOperation re-enqueues one terminal 源头取件 dead
|
|
// letter under a new round-suffixed idempotency key, resets the intent's fetch
|
|
// state and closes the dead letter — all in one operation envelope.
|
|
func (s *Store) RequeueArtifactFetchWithOperation(ctx context.Context, request governance.RequeueArtifactFetchRequest, operationID string) (governance.RequeueArtifactFetchResult, bool, error) {
|
|
if operationID == "" || len(operationID) > 128 {
|
|
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid operation id")
|
|
}
|
|
if request.FailureID == "" || len(request.FailureID) > 64 {
|
|
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid failure id")
|
|
}
|
|
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 500 || request.Now.IsZero() {
|
|
return governance.RequeueArtifactFetchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
|
}
|
|
fingerprint, err := operationFingerprint(struct {
|
|
FailureID string `json:"failure_id"`
|
|
OperatorRef string `json:"operator_ref"`
|
|
Reason string `json:"reason"`
|
|
}{request.FailureID, request.OperatorRef, request.Reason})
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, false, err
|
|
}
|
|
raw, replayed, err := s.executeOperation(ctx, "artifact.fetch.requeue", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
|
result, err := requeueArtifactFetchTx(ctx, tx, request)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(result)
|
|
})
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, false, err
|
|
}
|
|
var result governance.RequeueArtifactFetchResult
|
|
if err := json.Unmarshal(raw, &result); err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid stored requeue result")
|
|
}
|
|
return result, replayed, nil
|
|
}
|
|
|
|
func requeueArtifactFetchTx(ctx context.Context, tx pgx.Tx, request governance.RequeueArtifactFetchRequest) (governance.RequeueArtifactFetchResult, error) {
|
|
var state, jobType, connectionID, nodeID, versionID, docName string
|
|
err := tx.QueryRow(ctx, loadRequeueableArtifactFailureSQL, request.FailureID).
|
|
Scan(&state, &jobType, &connectionID, &nodeID, &versionID, &docName)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
|
|
}
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("load requeueable failure: %w", err)
|
|
}
|
|
if jobType != "artifact.fetch" || state != "open" || connectionID == "" || nodeID == "" || versionID == "" {
|
|
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
|
|
}
|
|
var rounds int
|
|
if err := tx.QueryRow(ctx, countArtifactFetchRoundsSQL, versionID).Scan(&rounds); err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("count artifact fetch rounds: %w", err)
|
|
}
|
|
round := rounds + 1
|
|
jobID, err := newOpaqueID("job-")
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, err
|
|
}
|
|
payload, err := json.Marshal(map[string]string{
|
|
"source_connection_id": connectionID, "node_id": nodeID, "version_id": versionID,
|
|
})
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, errors.New("encode artifact fetch job")
|
|
}
|
|
tag, err := tx.Exec(ctx, requeueArtifactFetchJobSQL, jobID, string(payload), request.Now,
|
|
fmt.Sprintf("artifact-fetch:%s:r%d", versionID, round))
|
|
if err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("requeue artifact fetch: %w", err)
|
|
}
|
|
result := governance.RequeueArtifactFetchResult{
|
|
FailureID: request.FailureID, JobID: jobID, VersionID: versionID, Round: round, DocName: docName,
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
// The round key already exists (concurrent requeue): report it instead of
|
|
// resolving the dead letter twice.
|
|
result.AlreadyQueued = true
|
|
return result, nil
|
|
}
|
|
if _, err := tx.Exec(ctx, resetIntentFetchStateSQL, connectionID, nodeID, request.Now); err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("reset intent fetch state: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, resolveRequeuedFailureSQL, request.FailureID, request.Now, request.OperatorRef, request.Reason); err != nil {
|
|
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("resolve job failure: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
const listJobFailuresSQL = `
|
|
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
|
|
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
|
|
jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at,
|
|
COALESCE(siv.source_item_id,''),
|
|
COALESCE(NULLIF(item.title,''), NULLIF(ci.name,''), ''),
|
|
COALESCE(NULLIF(jf.last_error,''), j.last_error, '')
|
|
FROM job_failures jf
|
|
LEFT JOIN test_publications p ON p.id = jf.publication_id
|
|
LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id
|
|
LEFT JOIN source_items item ON item.id = siv.source_item_id
|
|
LEFT JOIN jobs j ON j.id = jf.job_id
|
|
LEFT JOIN LATERAL (
|
|
SELECT intent.name FROM catalog_import_intents intent
|
|
WHERE intent.source_connection_id = j.payload->>'source_connection_id'
|
|
AND intent.node_id = j.payload->>'node_id'
|
|
LIMIT 1
|
|
) ci ON TRUE
|
|
WHERE ($1 = '' OR jf.state = $1)
|
|
ORDER BY jf.created_at DESC, jf.id DESC
|
|
LIMIT $2 OFFSET $3
|
|
`
|
|
|
|
const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)`
|
|
|
|
// ListJobFailures is the DLQ view (P6): projections of terminal attempts. The
|
|
// original jobs and publication audit rows are never moved or deleted.
|
|
func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) {
|
|
state := filter.State
|
|
if state == "" {
|
|
state = "open"
|
|
}
|
|
if state != "open" && state != "requeued" && state != "abandoned" {
|
|
return governance.JobFailurePage{}, errors.New("invalid failure state filter")
|
|
}
|
|
limit, offset := filter.Limit, filter.Offset
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 100 {
|
|
return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100")
|
|
}
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
var total int
|
|
if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil {
|
|
return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err)
|
|
}
|
|
rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset)
|
|
if err != nil {
|
|
return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
items := make([]governance.JobFailure, 0, limit)
|
|
for rows.Next() {
|
|
var item governance.JobFailure
|
|
if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode,
|
|
&item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID,
|
|
&item.State, &item.CreatedAt, &item.SourceItemID, &item.DocName, &item.LastError); err != nil {
|
|
return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err)
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err)
|
|
}
|
|
return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
|
|
}
|
|
|
|
const failurePublicationStateSQL = `
|
|
SELECT p.processing_state, p.ingest_state,
|
|
EXISTS (
|
|
SELECT 1 FROM source_item_versions siv
|
|
JOIN content_objects co ON co.id = siv.content_object_id
|
|
WHERE siv.id = p.source_item_version_id
|
|
AND co.content_state = 'verified'
|
|
AND co.local_spool_ref <> ''
|
|
)
|
|
FROM test_publications p
|
|
WHERE p.id = $1
|
|
`
|
|
|
|
const resolvePublicationFailuresSQL = `
|
|
UPDATE job_failures
|
|
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
|
WHERE publication_id = $1 AND state = 'open'
|
|
`
|
|
|
|
// abandonPublicationFailuresSQL closes open dead letters the same way retry
|
|
// does, but records the operator's "stop trying" decision instead.
|
|
const abandonPublicationFailuresSQL = `
|
|
UPDATE job_failures
|
|
SET state = 'abandoned', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
|
WHERE publication_id = $1 AND state = 'open'
|
|
`
|
|
|
|
// cancelPublicationSQL moves a failed/stalled publication out of the retry
|
|
// loop. Rows already cancelled are left alone (idempotent replay).
|
|
const cancelPublicationSQL = `
|
|
UPDATE test_publications
|
|
SET processing_state = 'cancelled', updated_at = $2
|
|
WHERE id = $1
|
|
AND superseded_at IS NULL
|
|
AND (processing_state IN ('failed', 'stalled')
|
|
OR (ingest_state = 'failed' AND processing_state = 'pending'))
|
|
`
|
|
|
|
const cancelQueuedUploadJobsSQL = `
|
|
UPDATE jobs
|
|
SET state = 'cancelled', lease_owner = NULL, lease_until = NULL,
|
|
last_error = 'cancelled by operator', last_error_code = '', updated_at = $2
|
|
WHERE type = 'test.upload' AND payload->>'publication_id' = $1
|
|
AND state IN ('pending', 'retry')
|
|
`
|
|
|
|
// CancelPublicationsBatchWithOperation moves 1..100 failing publications to
|
|
// 'cancelled' in one operation envelope: per-item outcome, dead letters
|
|
// abandoned, queued upload jobs dropped, one audit row per item.
|
|
func (s *Store) CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
|
if operationID == "" || len(operationID) > 128 {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
|
|
}
|
|
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
|
|
}
|
|
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
|
}
|
|
seen := make(map[string]bool, len(request.PublicationIDs))
|
|
ids := make([]string, 0, len(request.PublicationIDs))
|
|
for _, id := range request.PublicationIDs {
|
|
if id == "" || len(id) > 64 || seen[id] {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
|
|
}
|
|
seen[id] = true
|
|
ids = append(ids, id)
|
|
}
|
|
fingerprint, err := operationFingerprint(struct {
|
|
PublicationIDs []string `json:"publication_ids"`
|
|
OperatorRef string `json:"operator_ref"`
|
|
Reason string `json:"reason"`
|
|
Kind string `json:"kind"`
|
|
}{ids, request.OperatorRef, request.Reason, "cancel"})
|
|
if err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
}
|
|
raw, replayed, err := s.executeOperation(ctx, "test.publication.cancel_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
|
result, err := cancelPublicationsBatchTx(ctx, tx, request, ids, operationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(result)
|
|
})
|
|
if err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
}
|
|
var result governance.RetryPublicationsBatchResult
|
|
if err := json.Unmarshal(raw, &result); err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored cancel batch result")
|
|
}
|
|
return result, replayed, nil
|
|
}
|
|
|
|
func cancelPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.CancelPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
|
|
sorted := append([]string(nil), ids...)
|
|
for i := 1; i < len(sorted); i++ {
|
|
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
|
|
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
|
}
|
|
}
|
|
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
|
|
for _, id := range sorted {
|
|
var processingState, ingestState string
|
|
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, new(bool))
|
|
outcome := governance.RetryOutcome{PublicationID: id}
|
|
switch {
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication not found"
|
|
case err != nil:
|
|
return result, fmt.Errorf("classify publication cancel: %w", err)
|
|
case processingState == "processing" || processingState == "finalizing":
|
|
outcome.Result = governance.RetryAlreadyRunning
|
|
case processingState == "cancelled":
|
|
// Idempotent replay of a previous cancel: count it as done.
|
|
outcome.Result = governance.CancelCancelled
|
|
case processingState == "completed":
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication already completed"
|
|
case processingState == "planned" || (processingState == "pending" && ingestState != "failed"):
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication is not in a failed state"
|
|
default:
|
|
tag, err := tx.Exec(ctx, cancelPublicationSQL, id, request.Now)
|
|
if err != nil {
|
|
return result, fmt.Errorf("cancel publication %s: %w", id, err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
outcome.Result = governance.RetryAlreadyRunning
|
|
} else {
|
|
if _, err := tx.Exec(ctx, cancelQueuedUploadJobsSQL, id, request.Now); err != nil {
|
|
return result, fmt.Errorf("cancel queued upload jobs: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, abandonPublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
|
|
return result, fmt.Errorf("abandon job failures: %w", err)
|
|
}
|
|
outcome.Result = governance.CancelCancelled
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
|
|
return result, fmt.Errorf("record cancel audit: %w", err)
|
|
}
|
|
switch outcome.Result {
|
|
case governance.CancelCancelled:
|
|
result.Started++
|
|
case governance.RetryAlreadyRunning:
|
|
result.AlreadyRunning++
|
|
default:
|
|
result.NotRetryable++
|
|
}
|
|
result.Outcomes = append(result.Outcomes, outcome)
|
|
}
|
|
result.Total = len(sorted)
|
|
return result, nil
|
|
}
|
|
|
|
const recordRetryAuditSQL = `
|
|
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
|
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
ON CONFLICT (batch_key, publication_id) DO NOTHING
|
|
`
|
|
|
|
// RetryPublicationsBatchWithOperation performs the whole selection server-side
|
|
// (P6): one operation envelope, one transaction, per-item outcome, atomic audit.
|
|
func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
|
if operationID == "" || len(operationID) > 128 {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
|
|
}
|
|
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
|
|
}
|
|
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
|
}
|
|
seen := make(map[string]bool, len(request.PublicationIDs))
|
|
ids := make([]string, 0, len(request.PublicationIDs))
|
|
for _, id := range request.PublicationIDs {
|
|
if id == "" || len(id) > 64 || seen[id] {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
|
|
}
|
|
seen[id] = true
|
|
ids = append(ids, id)
|
|
}
|
|
fingerprint, err := operationFingerprint(struct {
|
|
PublicationIDs []string `json:"publication_ids"`
|
|
OperatorRef string `json:"operator_ref"`
|
|
Reason string `json:"reason"`
|
|
}{ids, request.OperatorRef, request.Reason})
|
|
if err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
}
|
|
raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
|
result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(result)
|
|
})
|
|
if err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
}
|
|
var result governance.RetryPublicationsBatchResult
|
|
if err := json.Unmarshal(raw, &result); err != nil {
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result")
|
|
}
|
|
return result, replayed, nil
|
|
}
|
|
|
|
func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
|
|
// Stable lock order prevents cross-batch deadlocks.
|
|
sorted := append([]string(nil), ids...)
|
|
for i := 1; i < len(sorted); i++ {
|
|
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
|
|
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
|
}
|
|
}
|
|
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
|
|
store := NewStore(tx)
|
|
for _, id := range sorted {
|
|
var processingState, ingestState string
|
|
var spoolAvailable bool
|
|
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable)
|
|
outcome := governance.RetryOutcome{PublicationID: id}
|
|
switch {
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication not found"
|
|
case err != nil:
|
|
return result, fmt.Errorf("classify publication retry: %w", err)
|
|
case processingState == "processing" || processingState == "finalizing":
|
|
outcome.Result = governance.RetryAlreadyRunning
|
|
case processingState == "pending" && ingestState != "failed":
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication is already queued"
|
|
case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"):
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "publication is not in a failed state"
|
|
case !spoolAvailable:
|
|
outcome.Result = governance.RetryNotRetryable
|
|
outcome.Detail = "no verified spool artifact"
|
|
default:
|
|
if err := store.RetryPublication(ctx, id, request.Now); err != nil {
|
|
return result, fmt.Errorf("retry publication %s: %w", id, err)
|
|
}
|
|
if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
|
|
return result, fmt.Errorf("resolve job failures: %w", err)
|
|
}
|
|
outcome.Result = governance.RetryStarted
|
|
}
|
|
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
|
|
return result, fmt.Errorf("record retry audit: %w", err)
|
|
}
|
|
switch outcome.Result {
|
|
case governance.RetryStarted:
|
|
result.Started++
|
|
case governance.RetryAlreadyRunning:
|
|
result.AlreadyRunning++
|
|
default:
|
|
result.NotRetryable++
|
|
}
|
|
result.Outcomes = append(result.Outcomes, outcome)
|
|
}
|
|
result.Total = len(sorted)
|
|
return result, nil
|
|
}
|