|
|
|
@@ -100,6 +100,144 @@ SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_
|
|
|
|
|
WHERE publication_id = $1 AND state = 'open'
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
// abandonPublicationFailuresSQL closes open dead letters the same way retry
|
|
|
|
|
// does, but records the operator's "stop trying" decision instead.
|
|
|
|
|
const abandonPublicationFailuresSQL = `
|
|
|
|
|
UPDATE job_failures
|
|
|
|
|
SET state = 'abandoned', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
|
|
|
|
WHERE publication_id = $1 AND state = 'open'
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
// cancelPublicationSQL moves a failed/stalled publication out of the retry
|
|
|
|
|
// loop. Rows already cancelled are left alone (idempotent replay).
|
|
|
|
|
const cancelPublicationSQL = `
|
|
|
|
|
UPDATE test_publications
|
|
|
|
|
SET processing_state = 'cancelled', updated_at = $2
|
|
|
|
|
WHERE id = $1
|
|
|
|
|
AND superseded_at IS NULL
|
|
|
|
|
AND (processing_state IN ('failed', 'stalled')
|
|
|
|
|
OR (ingest_state = 'failed' AND processing_state = 'pending'))
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
const cancelQueuedUploadJobsSQL = `
|
|
|
|
|
UPDATE jobs
|
|
|
|
|
SET state = 'cancelled', lease_owner = NULL, lease_until = NULL,
|
|
|
|
|
last_error = 'cancelled by operator', last_error_code = '', updated_at = $2
|
|
|
|
|
WHERE type = 'test.upload' AND payload->>'publication_id' = $1
|
|
|
|
|
AND state IN ('pending', 'retry')
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
// CancelPublicationsBatchWithOperation moves 1..100 failing publications to
|
|
|
|
|
// 'cancelled' in one operation envelope: per-item outcome, dead letters
|
|
|
|
|
// abandoned, queued upload jobs dropped, one audit row per item.
|
|
|
|
|
func (s *Store) CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
|
|
|
|
if operationID == "" || len(operationID) > 128 {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
|
|
|
|
|
}
|
|
|
|
|
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
|
|
|
|
|
}
|
|
|
|
|
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
|
|
|
|
}
|
|
|
|
|
seen := make(map[string]bool, len(request.PublicationIDs))
|
|
|
|
|
ids := make([]string, 0, len(request.PublicationIDs))
|
|
|
|
|
for _, id := range request.PublicationIDs {
|
|
|
|
|
if id == "" || len(id) > 64 || seen[id] {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
|
|
|
|
|
}
|
|
|
|
|
seen[id] = true
|
|
|
|
|
ids = append(ids, id)
|
|
|
|
|
}
|
|
|
|
|
fingerprint, err := operationFingerprint(struct {
|
|
|
|
|
PublicationIDs []string `json:"publication_ids"`
|
|
|
|
|
OperatorRef string `json:"operator_ref"`
|
|
|
|
|
Reason string `json:"reason"`
|
|
|
|
|
Kind string `json:"kind"`
|
|
|
|
|
}{ids, request.OperatorRef, request.Reason, "cancel"})
|
|
|
|
|
if err != nil {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
|
|
|
}
|
|
|
|
|
raw, replayed, err := s.executeOperation(ctx, "test.publication.cancel_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
|
|
|
|
result, err := cancelPublicationsBatchTx(ctx, tx, request, ids, operationID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
return json.Marshal(result)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, err
|
|
|
|
|
}
|
|
|
|
|
var result governance.RetryPublicationsBatchResult
|
|
|
|
|
if err := json.Unmarshal(raw, &result); err != nil {
|
|
|
|
|
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored cancel batch result")
|
|
|
|
|
}
|
|
|
|
|
return result, replayed, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func cancelPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.CancelPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
|
|
|
|
|
sorted := append([]string(nil), ids...)
|
|
|
|
|
for i := 1; i < len(sorted); i++ {
|
|
|
|
|
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
|
|
|
|
|
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
|
|
|
|
|
for _, id := range sorted {
|
|
|
|
|
var processingState, ingestState string
|
|
|
|
|
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, new(bool))
|
|
|
|
|
outcome := governance.RetryOutcome{PublicationID: id}
|
|
|
|
|
switch {
|
|
|
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
|
|
|
outcome.Result = governance.RetryNotRetryable
|
|
|
|
|
outcome.Detail = "publication not found"
|
|
|
|
|
case err != nil:
|
|
|
|
|
return result, fmt.Errorf("classify publication cancel: %w", err)
|
|
|
|
|
case processingState == "processing" || processingState == "finalizing":
|
|
|
|
|
outcome.Result = governance.RetryAlreadyRunning
|
|
|
|
|
case processingState == "cancelled":
|
|
|
|
|
// Idempotent replay of a previous cancel: count it as done.
|
|
|
|
|
outcome.Result = governance.CancelCancelled
|
|
|
|
|
case processingState == "completed":
|
|
|
|
|
outcome.Result = governance.RetryNotRetryable
|
|
|
|
|
outcome.Detail = "publication already completed"
|
|
|
|
|
case processingState == "planned" || (processingState == "pending" && ingestState != "failed"):
|
|
|
|
|
outcome.Result = governance.RetryNotRetryable
|
|
|
|
|
outcome.Detail = "publication is not in a failed state"
|
|
|
|
|
default:
|
|
|
|
|
tag, err := tx.Exec(ctx, cancelPublicationSQL, id, request.Now)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return result, fmt.Errorf("cancel publication %s: %w", id, err)
|
|
|
|
|
}
|
|
|
|
|
if tag.RowsAffected() == 0 {
|
|
|
|
|
outcome.Result = governance.RetryAlreadyRunning
|
|
|
|
|
} else {
|
|
|
|
|
if _, err := tx.Exec(ctx, cancelQueuedUploadJobsSQL, id, request.Now); err != nil {
|
|
|
|
|
return result, fmt.Errorf("cancel queued upload jobs: %w", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := tx.Exec(ctx, abandonPublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
|
|
|
|
|
return result, fmt.Errorf("abandon job failures: %w", err)
|
|
|
|
|
}
|
|
|
|
|
outcome.Result = governance.CancelCancelled
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
|
|
|
|
|
return result, fmt.Errorf("record cancel audit: %w", err)
|
|
|
|
|
}
|
|
|
|
|
switch outcome.Result {
|
|
|
|
|
case governance.CancelCancelled:
|
|
|
|
|
result.Started++
|
|
|
|
|
case governance.RetryAlreadyRunning:
|
|
|
|
|
result.AlreadyRunning++
|
|
|
|
|
default:
|
|
|
|
|
result.NotRetryable++
|
|
|
|
|
}
|
|
|
|
|
result.Outcomes = append(result.Outcomes, outcome)
|
|
|
|
|
}
|
|
|
|
|
result.Total = len(sorted)
|
|
|
|
|
return result, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const recordRetryAuditSQL = `
|
|
|
|
|
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
|
|
|
|
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
|
|
|