feat(monitor): 取消失败发布并修正危险按钮对比度

- Parse-monitor: the batch-retry button rendered red-on-red because
  weknoraMonitor.css overrode the global .danger style with text color
  only; make .danger a filled red button with proper hover/disabled states.
- Add operator cancel: 批量取消所选 / per-task 不再处理 buttons move
  failed/stalled publications to a new 'cancelled' state in one audited
  batch operation (POST /api/v1/test-publications/cancel-batch, idempotent,
  migration 000041 widens the CHECK constraints). Queued upload jobs are
  cancelled, open dead letters abandoned; source records and spool files
  stay intact so re-marking in the catalog can retry later.
- Queue status filter and labels learn 已取消; OpenAPI contract updated.
This commit is contained in:
姚章浩
2026-09-15 17:58:40 +08:00
parent 822896ea6c
commit 5037c05e93
13 changed files with 476 additions and 14 deletions
+138
View File
@@ -100,6 +100,144 @@ SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_
WHERE publication_id = $1 AND state = 'open'
`
// abandonPublicationFailuresSQL closes open dead letters the same way retry
// does, but records the operator's "stop trying" decision instead.
const abandonPublicationFailuresSQL = `
UPDATE job_failures
SET state = 'abandoned', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
WHERE publication_id = $1 AND state = 'open'
`
// cancelPublicationSQL moves a failed/stalled publication out of the retry
// loop. Rows already cancelled are left alone (idempotent replay).
const cancelPublicationSQL = `
UPDATE test_publications
SET processing_state = 'cancelled', updated_at = $2
WHERE id = $1
AND superseded_at IS NULL
AND (processing_state IN ('failed', 'stalled')
OR (ingest_state = 'failed' AND processing_state = 'pending'))
`
const cancelQueuedUploadJobsSQL = `
UPDATE jobs
SET state = 'cancelled', lease_owner = NULL, lease_until = NULL,
last_error = 'cancelled by operator', last_error_code = '', updated_at = $2
WHERE type = 'test.upload' AND payload->>'publication_id' = $1
AND state IN ('pending', 'retry')
`
// CancelPublicationsBatchWithOperation moves 1..100 failing publications to
// 'cancelled' in one operation envelope: per-item outcome, dead letters
// abandoned, queued upload jobs dropped, one audit row per item.
func (s *Store) CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
if operationID == "" || len(operationID) > 128 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
}
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
}
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
}
seen := make(map[string]bool, len(request.PublicationIDs))
ids := make([]string, 0, len(request.PublicationIDs))
for _, id := range request.PublicationIDs {
if id == "" || len(id) > 64 || seen[id] {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
}
seen[id] = true
ids = append(ids, id)
}
fingerprint, err := operationFingerprint(struct {
PublicationIDs []string `json:"publication_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
Kind string `json:"kind"`
}{ids, request.OperatorRef, request.Reason, "cancel"})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "test.publication.cancel_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := cancelPublicationsBatchTx(ctx, tx, request, ids, operationID)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
var result governance.RetryPublicationsBatchResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored cancel batch result")
}
return result, replayed, nil
}
func cancelPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.CancelPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
sorted := append([]string(nil), ids...)
for i := 1; i < len(sorted); i++ {
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
}
}
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
for _, id := range sorted {
var processingState, ingestState string
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, new(bool))
outcome := governance.RetryOutcome{PublicationID: id}
switch {
case errors.Is(err, pgx.ErrNoRows):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication not found"
case err != nil:
return result, fmt.Errorf("classify publication cancel: %w", err)
case processingState == "processing" || processingState == "finalizing":
outcome.Result = governance.RetryAlreadyRunning
case processingState == "cancelled":
// Idempotent replay of a previous cancel: count it as done.
outcome.Result = governance.CancelCancelled
case processingState == "completed":
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication already completed"
case processingState == "planned" || (processingState == "pending" && ingestState != "failed"):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication is not in a failed state"
default:
tag, err := tx.Exec(ctx, cancelPublicationSQL, id, request.Now)
if err != nil {
return result, fmt.Errorf("cancel publication %s: %w", id, err)
}
if tag.RowsAffected() == 0 {
outcome.Result = governance.RetryAlreadyRunning
} else {
if _, err := tx.Exec(ctx, cancelQueuedUploadJobsSQL, id, request.Now); err != nil {
return result, fmt.Errorf("cancel queued upload jobs: %w", err)
}
if _, err := tx.Exec(ctx, abandonPublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
return result, fmt.Errorf("abandon job failures: %w", err)
}
outcome.Result = governance.CancelCancelled
}
}
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
return result, fmt.Errorf("record cancel audit: %w", err)
}
switch outcome.Result {
case governance.CancelCancelled:
result.Started++
case governance.RetryAlreadyRunning:
result.AlreadyRunning++
default:
result.NotRetryable++
}
result.Outcomes = append(result.Outcomes, outcome)
}
result.Total = len(sorted)
return result, nil
}
const recordRetryAuditSQL = `
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
@@ -0,0 +1,11 @@
-- Roll back the cancelled state; any cancelled row is re-opened as failed
-- first so the tighter CHECK applies.
UPDATE test_publications SET processing_state = 'failed' WHERE processing_state = 'cancelled';
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
ALTER TABLE job_retry_operations DROP CONSTRAINT IF EXISTS job_retry_operations_outcome_check;
ALTER TABLE job_retry_operations ADD CONSTRAINT job_retry_operations_outcome_check
CHECK (outcome IN ('started', 'already_running', 'not_retryable'));
@@ -0,0 +1,11 @@
-- Documents whose parsing keeps failing (e.g. DingTalk exports an empty body)
-- can now be explicitly cancelled by an operator: the publication stops being
-- retryable and disappears from the attention lane, while the original record
-- stays untouched for a future re-mark.
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled', 'cancelled'));
ALTER TABLE job_retry_operations DROP CONSTRAINT IF EXISTS job_retry_operations_outcome_check;
ALTER TABLE job_retry_operations ADD CONSTRAINT job_retry_operations_outcome_check
CHECK (outcome IN ('started', 'already_running', 'not_retryable', 'cancelled'));
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 40 {
t.Fatalf("migration count = %d, want 40", len(migrations))
if len(migrations) != 41 {
t.Fatalf("migration count = %d, want 41", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])