feat(monitor): 取消失败发布并修正危险按钮对比度
- Parse-monitor: the batch-retry button rendered red-on-red because weknoraMonitor.css overrode the global .danger style with text color only; make .danger a filled red button with proper hover/disabled states. - Add operator cancel: 批量取消所选 / per-task 不再处理 buttons move failed/stalled publications to a new 'cancelled' state in one audited batch operation (POST /api/v1/test-publications/cancel-batch, idempotent, migration 000041 widens the CHECK constraints). Queued upload jobs are cancelled, open dead letters abandoned; source records and spool files stay intact so re-marking in the catalog can retry later. - Queue status filter and labels learn 已取消; OpenAPI contract updated.
This commit is contained in:
+129
-1
@@ -1422,7 +1422,8 @@
|
||||
"finalizing",
|
||||
"completed",
|
||||
"failed",
|
||||
"stalled"
|
||||
"stalled",
|
||||
"cancelled"
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -1941,6 +1942,133 @@
|
||||
"operationId": "retryPublicationsBatch"
|
||||
}
|
||||
},
|
||||
"/api/v1/test-publications/cancel-batch": {
|
||||
"post": {
|
||||
"summary": "服务端批量取消失败发布(不再自动重试)",
|
||||
"description": "用于源头即空文档等永久性失败:一次事务内把 failed/stalled 发布置为 cancelled,撤销排队中的上传任务并放弃对应死信;逐项返回 cancelled/already_running/not_retryable。原始记录与暂存文件保留,重新标记入库可再次处理。最多100项,操作人与原因由服务端会话注入并原子审计。",
|
||||
"parameters": [
|
||||
{
|
||||
"$ref": "#/components/parameters/IdempotencyKey"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"publication_ids": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 100,
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 500
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"publication_ids",
|
||||
"reason"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "批量取消结果(含逐项outcome)",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"batch_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"total": {
|
||||
"type": "integer"
|
||||
},
|
||||
"started": {
|
||||
"type": "integer",
|
||||
"description": "成功取消的项数"
|
||||
},
|
||||
"already_running": {
|
||||
"type": "integer"
|
||||
},
|
||||
"not_retryable": {
|
||||
"type": "integer",
|
||||
"description": "不可取消的项数(如已完成或不在失败态)"
|
||||
},
|
||||
"outcomes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"publication_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"result": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"cancelled",
|
||||
"already_running",
|
||||
"not_retryable"
|
||||
]
|
||||
},
|
||||
"detail": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"batch_key",
|
||||
"total",
|
||||
"started",
|
||||
"already_running",
|
||||
"not_retryable",
|
||||
"outcomes"
|
||||
]
|
||||
},
|
||||
"operation": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"created",
|
||||
"replayed"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"data",
|
||||
"operation"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"409": {
|
||||
"$ref": "#/components/responses/Conflict"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
}
|
||||
},
|
||||
"operationId": "cancelPublicationsBatch"
|
||||
}
|
||||
},
|
||||
"/api/v1/monitor/metrics": {
|
||||
"get": {
|
||||
"operationId": "getMonitorMetrics",
|
||||
|
||||
@@ -64,3 +64,20 @@ type RetryPublicationsBatchResult struct {
|
||||
NotRetryable int `json:"not_retryable"`
|
||||
Outcomes []RetryOutcome `json:"outcomes"`
|
||||
}
|
||||
|
||||
// CancelOutcome constants reuse the retry vocabulary plus "cancelled".
|
||||
const (
|
||||
CancelCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// CancelPublicationsBatchRequest is the operator's "stop retrying" batch:
|
||||
// 1..100 failed/stalled publications move to 'cancelled', their queued
|
||||
// upload jobs are dropped and their open dead letters are abandoned.
|
||||
// Source records and spooled artifacts are untouched; re-marking the file
|
||||
// in 来源与目录 can still create a fresh publication later.
|
||||
type CancelPublicationsBatchRequest struct {
|
||||
PublicationIDs []string `json:"publication_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
Now time.Time `json:"-"`
|
||||
}
|
||||
|
||||
@@ -133,11 +133,15 @@ const (
|
||||
ProcessingCompleted ProcessingState = "completed"
|
||||
ProcessingFailed ProcessingState = "failed"
|
||||
ProcessingStalled ProcessingState = "stalled"
|
||||
// ProcessingCancelled is the operator's "stop trying" decision on a
|
||||
// permanently failing document (e.g. an empty DingTalk export). The
|
||||
// publication leaves the attention lane; source data is untouched.
|
||||
ProcessingCancelled ProcessingState = "cancelled"
|
||||
)
|
||||
|
||||
func (state ProcessingState) Valid() bool {
|
||||
switch state {
|
||||
case ProcessingPending, ProcessingProcessing, ProcessingFinalizing, ProcessingCompleted, ProcessingFailed, ProcessingStalled:
|
||||
case ProcessingPending, ProcessingProcessing, ProcessingFinalizing, ProcessingCompleted, ProcessingFailed, ProcessingStalled, ProcessingCancelled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
||||
@@ -94,6 +94,7 @@ type Store interface {
|
||||
RetryPublicationWithOperation(ctx context.Context, publicationID string, now time.Time, operationID string) (bool, error)
|
||||
ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error)
|
||||
RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error)
|
||||
CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -385,6 +386,9 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("POST /api/v1/test-publications/retry-batch", func(w http.ResponseWriter, r *http.Request) {
|
||||
retryPublicationsBatch(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/test-publications/cancel-batch", func(w http.ResponseWriter, r *http.Request) {
|
||||
cancelPublicationsBatch(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/job-failures", func(w http.ResponseWriter, r *http.Request) {
|
||||
listJobFailures(w, r, config.Store)
|
||||
})
|
||||
@@ -1914,6 +1918,50 @@ func retryPublicationsBatch(w http.ResponseWriter, r *http.Request, config Confi
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": result, "operation": operation})
|
||||
}
|
||||
|
||||
func cancelPublicationsBatch(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
operationID, ok := requireOperationID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var input startTestPublicationsInput
|
||||
if err := decodeStrictJSON(r, &input); err != nil || len(input.PublicationIDs) == 0 || len(input.PublicationIDs) > 100 {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid cancel batch"})
|
||||
return
|
||||
}
|
||||
reason := strings.TrimSpace(input.Reason)
|
||||
if reason == "" || len(reason) > 500 {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "reason is required"})
|
||||
return
|
||||
}
|
||||
seen := make(map[string]bool, len(input.PublicationIDs))
|
||||
for _, publicationID := range input.PublicationIDs {
|
||||
if !validCatalogID(publicationID) || seen[publicationID] {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid publication_ids"})
|
||||
return
|
||||
}
|
||||
seen[publicationID] = true
|
||||
}
|
||||
result, replayed, err := config.Store.CancelPublicationsBatchWithOperation(r.Context(), governance.CancelPublicationsBatchRequest{
|
||||
PublicationIDs: input.PublicationIDs,
|
||||
OperatorRef: actorFor(r),
|
||||
Reason: reason,
|
||||
Now: config.Now().UTC(),
|
||||
}, operationID)
|
||||
if errors.Is(err, postgresstore.ErrOperationRequestConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not cancel publications"})
|
||||
return
|
||||
}
|
||||
operation := "created"
|
||||
if replayed {
|
||||
operation = "replayed"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": result, "operation": operation})
|
||||
}
|
||||
|
||||
func retryPublication(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
operationID, ok := requireOperationID(w, r)
|
||||
if !ok {
|
||||
@@ -1977,7 +2025,7 @@ func decodeStrictJSON(r *http.Request, destination any) error {
|
||||
|
||||
func validProcessingState(state string) bool {
|
||||
switch state {
|
||||
case "planned", "pending", "processing", "finalizing", "completed", "failed", "stalled":
|
||||
case "planned", "pending", "processing", "finalizing", "completed", "failed", "stalled", "cancelled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
||||
@@ -161,6 +161,49 @@ func TestHandlerListsJobFailuresAndRunsServerSideBatchRetry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRunsServerSideBatchCancel(t *testing.T) {
|
||||
store := &fakeJobStore{
|
||||
cancelBatchResult: governance.RetryPublicationsBatchResult{
|
||||
BatchKey: "op-cancel-1", Total: 2, Started: 1, NotRetryable: 1,
|
||||
Outcomes: []governance.RetryOutcome{{PublicationID: "tp-1", Result: "cancelled"}, {PublicationID: "tp-x", Result: "not_retryable", Detail: "publication not found"}},
|
||||
},
|
||||
}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/cancel-batch",
|
||||
strings.NewReader(`{"publication_ids":["tp-1","tp-x"],"reason":"源头为空文档,不再重试"}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", "op-cancel-1")
|
||||
handler.ServeHTTP(recorder, request)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("cancel-batch status = %d body %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.cancelBatchOperation != "op-cancel-1" || store.cancelBatchRequest == nil ||
|
||||
len(store.cancelBatchRequest.PublicationIDs) != 2 ||
|
||||
store.cancelBatchRequest.Reason != "源头为空文档,不再重试" || store.cancelBatchRequest.Now.IsZero() {
|
||||
t.Fatalf("cancel request = %+v op=%q", store.cancelBatchRequest, store.cancelBatchOperation)
|
||||
}
|
||||
|
||||
// Missing Idempotency-Key and missing reason are rejected before touching the store.
|
||||
recorder = httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/cancel-batch",
|
||||
strings.NewReader(`{"publication_ids":["tp-1"],"reason":"r"}`)))
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("missing key status = %d", recorder.Code)
|
||||
}
|
||||
recorder = httptest.NewRecorder()
|
||||
noReason := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/cancel-batch",
|
||||
strings.NewReader(`{"publication_ids":["tp-1"],"reason":""}`))
|
||||
noReason.Header.Set("Idempotency-Key", "op-cancel-2")
|
||||
handler.ServeHTTP(recorder, noReason)
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("missing reason status = %d", recorder.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerListsJobsWithJSONPayload(t *testing.T) {
|
||||
store := &fakeJobStore{listResult: []jobs.Job{{
|
||||
ID: "job-6",
|
||||
@@ -1438,6 +1481,11 @@ type fakeJobStore struct {
|
||||
retryBatchRequest *governance.RetryPublicationsBatchRequest
|
||||
retryBatchOperation string
|
||||
retryBatchResult governance.RetryPublicationsBatchResult
|
||||
cancelBatchRequest *governance.CancelPublicationsBatchRequest
|
||||
cancelBatchOperation string
|
||||
cancelBatchResult governance.RetryPublicationsBatchResult
|
||||
cancelBatchReplayed bool
|
||||
cancelBatchErr error
|
||||
retryBatchReplayed bool
|
||||
retryBatchErr error
|
||||
enqueued jobs.EnqueueRequest
|
||||
@@ -1837,6 +1885,12 @@ func (s *fakeJobStore) RetryPublicationsBatchWithOperation(_ context.Context, re
|
||||
return s.retryBatchResult, s.retryBatchReplayed, s.retryBatchErr
|
||||
}
|
||||
|
||||
func (s *fakeJobStore) CancelPublicationsBatchWithOperation(_ context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
||||
s.cancelBatchRequest = &request
|
||||
s.cancelBatchOperation = operationID
|
||||
return s.cancelBatchResult, s.cancelBatchReplayed, s.cancelBatchErr
|
||||
}
|
||||
|
||||
func (s *fakeJobStore) ListScopeRules(_ context.Context, connectionID string) ([]governance.ScopeRule, error) {
|
||||
s.scopeRuleConnectionID = connectionID
|
||||
return s.scopeRules, s.scopeRuleErr
|
||||
|
||||
@@ -100,6 +100,144 @@ SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_
|
||||
WHERE publication_id = $1 AND state = 'open'
|
||||
`
|
||||
|
||||
// abandonPublicationFailuresSQL closes open dead letters the same way retry
|
||||
// does, but records the operator's "stop trying" decision instead.
|
||||
const abandonPublicationFailuresSQL = `
|
||||
UPDATE job_failures
|
||||
SET state = 'abandoned', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
||||
WHERE publication_id = $1 AND state = 'open'
|
||||
`
|
||||
|
||||
// cancelPublicationSQL moves a failed/stalled publication out of the retry
|
||||
// loop. Rows already cancelled are left alone (idempotent replay).
|
||||
const cancelPublicationSQL = `
|
||||
UPDATE test_publications
|
||||
SET processing_state = 'cancelled', updated_at = $2
|
||||
WHERE id = $1
|
||||
AND superseded_at IS NULL
|
||||
AND (processing_state IN ('failed', 'stalled')
|
||||
OR (ingest_state = 'failed' AND processing_state = 'pending'))
|
||||
`
|
||||
|
||||
const cancelQueuedUploadJobsSQL = `
|
||||
UPDATE jobs
|
||||
SET state = 'cancelled', lease_owner = NULL, lease_until = NULL,
|
||||
last_error = 'cancelled by operator', last_error_code = '', updated_at = $2
|
||||
WHERE type = 'test.upload' AND payload->>'publication_id' = $1
|
||||
AND state IN ('pending', 'retry')
|
||||
`
|
||||
|
||||
// CancelPublicationsBatchWithOperation moves 1..100 failing publications to
|
||||
// 'cancelled' in one operation envelope: per-item outcome, dead letters
|
||||
// abandoned, queued upload jobs dropped, one audit row per item.
|
||||
func (s *Store) CancelPublicationsBatchWithOperation(ctx context.Context, request governance.CancelPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
||||
if operationID == "" || len(operationID) > 128 {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
|
||||
}
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
|
||||
}
|
||||
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
||||
}
|
||||
seen := make(map[string]bool, len(request.PublicationIDs))
|
||||
ids := make([]string, 0, len(request.PublicationIDs))
|
||||
for _, id := range request.PublicationIDs {
|
||||
if id == "" || len(id) > 64 || seen[id] {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
|
||||
}
|
||||
seen[id] = true
|
||||
ids = append(ids, id)
|
||||
}
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
PublicationIDs []string `json:"publication_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
Kind string `json:"kind"`
|
||||
}{ids, request.OperatorRef, request.Reason, "cancel"})
|
||||
if err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, err
|
||||
}
|
||||
raw, replayed, err := s.executeOperation(ctx, "test.publication.cancel_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := cancelPublicationsBatchTx(ctx, tx, request, ids, operationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, err
|
||||
}
|
||||
var result governance.RetryPublicationsBatchResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored cancel batch result")
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func cancelPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.CancelPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
|
||||
sorted := append([]string(nil), ids...)
|
||||
for i := 1; i < len(sorted); i++ {
|
||||
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
|
||||
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
||||
}
|
||||
}
|
||||
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
|
||||
for _, id := range sorted {
|
||||
var processingState, ingestState string
|
||||
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, new(bool))
|
||||
outcome := governance.RetryOutcome{PublicationID: id}
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication not found"
|
||||
case err != nil:
|
||||
return result, fmt.Errorf("classify publication cancel: %w", err)
|
||||
case processingState == "processing" || processingState == "finalizing":
|
||||
outcome.Result = governance.RetryAlreadyRunning
|
||||
case processingState == "cancelled":
|
||||
// Idempotent replay of a previous cancel: count it as done.
|
||||
outcome.Result = governance.CancelCancelled
|
||||
case processingState == "completed":
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication already completed"
|
||||
case processingState == "planned" || (processingState == "pending" && ingestState != "failed"):
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication is not in a failed state"
|
||||
default:
|
||||
tag, err := tx.Exec(ctx, cancelPublicationSQL, id, request.Now)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("cancel publication %s: %w", id, err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
outcome.Result = governance.RetryAlreadyRunning
|
||||
} else {
|
||||
if _, err := tx.Exec(ctx, cancelQueuedUploadJobsSQL, id, request.Now); err != nil {
|
||||
return result, fmt.Errorf("cancel queued upload jobs: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, abandonPublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
|
||||
return result, fmt.Errorf("abandon job failures: %w", err)
|
||||
}
|
||||
outcome.Result = governance.CancelCancelled
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
|
||||
return result, fmt.Errorf("record cancel audit: %w", err)
|
||||
}
|
||||
switch outcome.Result {
|
||||
case governance.CancelCancelled:
|
||||
result.Started++
|
||||
case governance.RetryAlreadyRunning:
|
||||
result.AlreadyRunning++
|
||||
default:
|
||||
result.NotRetryable++
|
||||
}
|
||||
result.Outcomes = append(result.Outcomes, outcome)
|
||||
}
|
||||
result.Total = len(sorted)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const recordRetryAuditSQL = `
|
||||
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Roll back the cancelled state; any cancelled row is re-opened as failed
|
||||
-- first so the tighter CHECK applies.
|
||||
UPDATE test_publications SET processing_state = 'failed' WHERE processing_state = 'cancelled';
|
||||
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
|
||||
|
||||
ALTER TABLE job_retry_operations DROP CONSTRAINT IF EXISTS job_retry_operations_outcome_check;
|
||||
ALTER TABLE job_retry_operations ADD CONSTRAINT job_retry_operations_outcome_check
|
||||
CHECK (outcome IN ('started', 'already_running', 'not_retryable'));
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Documents whose parsing keeps failing (e.g. DingTalk exports an empty body)
|
||||
-- can now be explicitly cancelled by an operator: the publication stops being
|
||||
-- retryable and disappears from the attention lane, while the original record
|
||||
-- stays untouched for a future re-mark.
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled', 'cancelled'));
|
||||
|
||||
ALTER TABLE job_retry_operations DROP CONSTRAINT IF EXISTS job_retry_operations_outcome_check;
|
||||
ALTER TABLE job_retry_operations ADD CONSTRAINT job_retry_operations_outcome_check
|
||||
CHECK (outcome IN ('started', 'already_running', 'not_retryable', 'cancelled'));
|
||||
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 40 {
|
||||
t.Fatalf("migration count = %d, want 40", len(migrations))
|
||||
if len(migrations) != 41 {
|
||||
t.Fatalf("migration count = %d, want 41", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
|
||||
@@ -614,6 +614,7 @@ const processingLabels: Record<ProcessingState, string> = {
|
||||
completed: '已完成',
|
||||
failed: '失败',
|
||||
stalled: '停滞',
|
||||
cancelled: '已取消',
|
||||
}
|
||||
|
||||
interface TestPublicationsProps {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { useContext, useEffect, useMemo, useState } from 'react'
|
||||
import { fetchMonitorFiles, fetchMonitorOverview, fetchRunMetrics } from './monitor'
|
||||
import type { RunMetrics } from './monitor'
|
||||
import type { KnowledgeProgress, MonitorFilePage, MonitorOverview, MonitorStatus } from './monitor'
|
||||
import { fetchJobFailures, fetchTestPublications, retryPublication, retryPublicationsBatch, startTestPublications } from './testPublications'
|
||||
import { fetchJobFailures, fetchTestPublications, retryPublication, retryPublicationsBatch, cancelPublicationsBatch, startTestPublications } from './testPublications'
|
||||
import type { JobFailure, ProcessingState, TestPublication, TestPublicationPage } from './testPublications'
|
||||
import { fetchActiveUISettings, settingsRefreshDelays } from './uiSettings'
|
||||
import { WorkspaceContextTabsSetter } from './workspaceContext'
|
||||
@@ -14,7 +14,7 @@ const gb = (value: number) => `${(value / 1073741824).toFixed(1)}`
|
||||
const scope = (value: string) => value === 'internal' ? '对内' : value === 'external' ? '对外' : '混合'
|
||||
const duration = (seconds: number) => seconds <= 0 ? '完成' : seconds < 60 ? `${seconds}秒` : seconds < 3600 ? `${Math.floor(seconds / 60)}分钟` : `${(seconds / 3600).toFixed(1)}小时`
|
||||
const statusLabels: Record<MonitorStatus, string> = { completed: 'completed', pending: 'pending', processing: 'processing', finalizing: 'finalizing', failed: 'failed', cancelled: 'cancelled' }
|
||||
const processingLabels: Record<ProcessingState, string> = { planned: '待入库', pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞' }
|
||||
const processingLabels: Record<ProcessingState, string> = { planned: '待入库', pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞', cancelled: '已取消' }
|
||||
|
||||
const secondsSince = (iso: string) => Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 1000))
|
||||
const adviceFor = (stage: string, code: string) => {
|
||||
@@ -85,6 +85,7 @@ interface ViewProps {
|
||||
loading: boolean
|
||||
error: string | null
|
||||
retryingIDs?: string[]
|
||||
cancellingIDs?: string[]
|
||||
starting?: boolean
|
||||
startMessage?: string
|
||||
batchMessage?: string
|
||||
@@ -98,6 +99,7 @@ interface ViewProps {
|
||||
onLaneChange?: (lane: string) => void
|
||||
onRetry?: (id: string) => void
|
||||
onBatchRetry?: (ids: string[]) => void
|
||||
onBatchCancel?: (ids: string[]) => void
|
||||
onStart?: (ids: string[]) => void
|
||||
onRefresh: () => void
|
||||
}
|
||||
@@ -116,6 +118,7 @@ export function WeKnoraMonitorView({
|
||||
loading,
|
||||
error,
|
||||
retryingIDs = [],
|
||||
cancellingIDs = [],
|
||||
starting = false,
|
||||
startMessage = '',
|
||||
batchMessage = '',
|
||||
@@ -127,6 +130,7 @@ export function WeKnoraMonitorView({
|
||||
onQueueScope = () => undefined,
|
||||
onRetry = () => undefined,
|
||||
onBatchRetry = () => undefined,
|
||||
onBatchCancel = () => undefined,
|
||||
onStart = () => undefined,
|
||||
onRefresh,
|
||||
}: ViewProps) {
|
||||
@@ -169,7 +173,7 @@ export function WeKnoraMonitorView({
|
||||
<div className="task-chain" aria-label={`处理链${item.title}`}>{chainSteps(item).map((step) => <span key={step.stage} className={`chain-step ${step.state}`}>{step.stage}{step.state === 'done' ? ' ✓' : step.state === 'bad' ? ' ✗' : step.state === 'active' ? ` 中 ${duration(secondsSince(item.updated_at))}` : ''}</span>)}</div>
|
||||
<div className={item.processing_state === 'failed' || item.processing_state === 'stalled' ? 'errline' : 'tiny muted-line'}>{item.processing_state === 'failed' || item.processing_state === 'stalled' ? adviceFor(failureStage(item), item.last_error) : item.processing_state === 'planned' && !item.can_start ? '正在从源头取回文件,完成后自动进入可处理状态' : item.can_start ? '文件已就绪,等待开始处理' : '无需操作,等待自动处理'} · <code>{item.id}</code></div>
|
||||
</div>
|
||||
<div className="inbox-task-actions">{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}</div>
|
||||
<div className="inbox-task-actions">{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}{item.can_retry ? <button disabled={cancellingIDs.includes(item.id) || retryingIDs.length > 0} title="不再自动重试;原始记录保留,重新标记入库可再次处理" onClick={() => onBatchCancel([item.id])}>{cancellingIDs.includes(item.id) ? '取消中…' : '不再处理'}</button> : null}</div>
|
||||
</article>
|
||||
const emptyLane = (text: string) => <div className="empty-state">{text}</div>
|
||||
return <main className="page-shell weknora-monitor inbox-layout">
|
||||
@@ -191,7 +195,7 @@ export function WeKnoraMonitorView({
|
||||
{fetchingPlanned.length ? <p className="tiny">其中 {fetchingPlanned.length} 项仍在从源头取件,取回后自动可处理。</p> : null}
|
||||
</section> : null}
|
||||
{lane === 'attention' ? <section className="inbox-section" aria-label="需人工处理">
|
||||
<div className="section-head"><h2>需要你出手({attentionItems.length})</h2><span className="spacer" /><button className="danger" disabled={!batchTargets.length || retryingIDs.length > 0} onClick={() => onBatchRetry(batchTargets)}>{retryingIDs.length ? '重试中…' : `批量重试所选(${batchTargets.length})`}</button></div>
|
||||
<div className="section-head"><h2>需要你出手({attentionItems.length})</h2><span className="spacer" /><button disabled={!batchTargets.length || cancellingIDs.length > 0 || retryingIDs.length > 0} title="不再自动重试(如源头就是空文档);原始记录保留,重新标记入库可再次处理" onClick={() => onBatchCancel(batchTargets)}>{cancellingIDs.length ? '取消中…' : `批量取消所选(${batchTargets.length})`}</button><button className="danger" disabled={!batchTargets.length || retryingIDs.length > 0} onClick={() => onBatchRetry(batchTargets)}>{retryingIDs.length ? '重试中…' : `批量重试所选(${batchTargets.length})`}</button></div>
|
||||
{batchMessage ? <p className="queue-batch-message" role="status">{batchMessage}</p> : null}
|
||||
{attentionItems.length ? attentionItems.map(actionTask) : emptyLane('没有失败或停滞的任务。')}
|
||||
<div className="section-head sub"><h3>死信(重试次数用完,系统不再自动处理)</h3><span className="tiny">“源头取件”=从钉钉/AList 把原始文件拉回本地;“上传解析”=把文件送进知识库做解析。死信只是失败任务的登记表,不会改动原始记录;重新入队=再给它一次机会</span></div>
|
||||
@@ -224,7 +228,7 @@ export function WeKnoraMonitorView({
|
||||
{doneItems.length ? doneItems.map(actionTask) : emptyLane('当前页没有已完成任务;用全部任务视图翻页查看。')}
|
||||
</section> : null}
|
||||
{lane === 'queue' ? <section className="inbox-section" aria-label="全部任务">
|
||||
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="planned">待入库</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option></select></label></div>
|
||||
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="planned">待入库</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option><option value="cancelled">已取消</option></select></label></div>
|
||||
{orderedQueue.length ? orderedQueue.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
|
||||
</section> : null}
|
||||
{metrics ? <footer className="monitor-strip" aria-label="真实运行指标"><div>1小时·成功 <b className={metrics.short_window.failure_rate !== null && metrics.short_window.failure_rate > 5 ? 'bad' : 'good'}>{metrics.short_window.success_rate === null ? '数据不足' : `${metrics.short_window.success_rate}%`}</b></div><div>24小时·成功 <b>{metrics.long_window.success_rate === null ? '数据不足' : `${metrics.long_window.success_rate}%`}</b></div><div>p50 {metrics.short_window.duration_p50_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p50_seconds))} · p95 {metrics.short_window.duration_p95_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p95_seconds))}</div><div>Worker {metrics.active_workers} · 等待 {metrics.queue.pending} · 运行 {metrics.queue.running}{metrics.queue.failed ? <span className="bad"> · 失败 {metrics.queue.failed}</span> : null}</div>{overview?.health.map((service) => <div key={service.key}>{service.name} <b className={service.readiness === 'ready' ? 'good' : service.readiness === 'blocked' || service.readiness === 'down' ? 'bad' : ''}>{service.readiness === 'ready' ? '可接单' : service.readiness === 'degraded' ? '降级' : service.readiness === 'not_configured' ? '未配置' : service.readiness === 'down' ? '不可达' : '阻断'}</b></div>)}{overview ? <div>CPU {overview.host.cpu_percent.toFixed(0)}% · 内存 {gb(overview.host.memory_used_bytes)}/{gb(overview.host.memory_total_bytes)}GB</div> : null}</footer> : null}
|
||||
@@ -250,6 +254,7 @@ export function WeKnoraMonitor() {
|
||||
const [documentRefreshMs, setDocumentRefreshMs] = useState(15000)
|
||||
const [resourceRefreshMs, setResourceRefreshMs] = useState(30000)
|
||||
const [retryingIDs, setRetryingIDs] = useState<string[]>([])
|
||||
const [cancellingIDs, setCancellingIDs] = useState<string[]>([])
|
||||
const [starting, setStarting] = useState(false)
|
||||
const [startMessage, setStartMessage] = useState('')
|
||||
const [batchMessage, setBatchMessage] = useState('')
|
||||
@@ -353,6 +358,21 @@ export function WeKnoraMonitor() {
|
||||
setRetryingIDs([])
|
||||
}
|
||||
}
|
||||
const cancelBatch = async (ids: string[]) => {
|
||||
setCancellingIDs(ids)
|
||||
setBatchMessage('')
|
||||
try {
|
||||
const result = await cancelPublicationsBatch(ids, '解析队列批量取消(不再重试)')
|
||||
setError(null)
|
||||
const skipped = result.already_running + result.not_retryable
|
||||
setBatchMessage(`已取消 ${result.started} 项,不再自动重试${skipped ? `,跳过 ${skipped} 项(处理中或不可取消)` : ''};原始记录未删除,重新标记入库可再次处理`)
|
||||
setDocumentRevision((value) => value + 1)
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : '批量取消失败')
|
||||
} finally {
|
||||
setCancellingIDs([])
|
||||
}
|
||||
}
|
||||
|
||||
const startPlans = async (ids: string[]) => {
|
||||
setStarting(true)
|
||||
@@ -368,5 +388,5 @@ export function WeKnoraMonitor() {
|
||||
setStarting(false)
|
||||
}
|
||||
}
|
||||
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onStart={(ids) => void startPlans(ids)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
|
||||
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} cancellingIDs={cancellingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onBatchCancel={(ids) => void cancelBatch(ids)} onStart={(ids) => void startPlans(ids)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { newOperationID } from './operationKey'
|
||||
export type IngestState = 'pending' | 'uploading' | 'uploaded' | 'failed'
|
||||
export type ProcessingState = 'planned' | 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled'
|
||||
export type ProcessingState = 'planned' | 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled' | 'cancelled'
|
||||
|
||||
export interface TestPublication {
|
||||
id: string
|
||||
@@ -50,7 +50,7 @@ export interface StartTestPublicationsResult {
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
|
||||
const isCount = (value: unknown): value is number => typeof value === 'number' && Number.isInteger(value) && value >= 0
|
||||
const ingestStates = new Set<IngestState>(['pending', 'uploading', 'uploaded', 'failed'])
|
||||
const processingStates = new Set<ProcessingState>(['planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'])
|
||||
const processingStates = new Set<ProcessingState>(['planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled', 'cancelled'])
|
||||
|
||||
function parseTestPublication(value: unknown): TestPublication {
|
||||
if (!isRecord(value) || typeof value.id !== 'string' || typeof value.title !== 'string' ||
|
||||
@@ -172,6 +172,34 @@ export async function retryPublicationsBatch(publicationIds: string[], reason: s
|
||||
|
||||
export interface JobFailure { id: string; job_id: string; job_type: string; publication_id?: string; source_item_id?: string; error_code: string; attempt: number; max_attempts_at_failure: number; change_set_id?: string; action?: string; target_kb_id?: string; state: 'open' | 'requeued' | 'abandoned'; created_at: string; doc_name?: string; last_error?: string }
|
||||
|
||||
export async function cancelPublicationsBatch(publicationIds: string[], reason: string): Promise<RetryPublicationsBatchResult> {
|
||||
const response = await fetch('/api/v1/test-publications/cancel-batch', {
|
||||
method: 'POST',
|
||||
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
|
||||
body: JSON.stringify({ publication_ids: publicationIds, reason }),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||||
throw new Error(body?.error ?? `批量取消失败(HTTP ${response.status})`)
|
||||
}
|
||||
const value = await response.json() as unknown
|
||||
if (!isRecord(value) || !isRecord(value.data) || !Array.isArray(value.data.outcomes)) throw new Error('批量取消响应无效')
|
||||
const data = value.data as Record<string, unknown>
|
||||
const outcomes = Array.isArray(data.outcomes) ? data.outcomes : []
|
||||
return {
|
||||
batch_key: typeof data.batch_key === 'string' ? data.batch_key : '',
|
||||
total: isCount(data.total) ? data.total : 0,
|
||||
started: isCount(data.started) ? data.started : 0,
|
||||
already_running: isCount(data.already_running) ? data.already_running : 0,
|
||||
not_retryable: isCount(data.not_retryable) ? data.not_retryable : 0,
|
||||
outcomes: outcomes.filter(isRecord).map((item) => ({
|
||||
publication_id: typeof item.publication_id === 'string' ? item.publication_id : '',
|
||||
result: item.result === 'started' || item.result === 'already_running' || item.result === 'not_retryable' ? item.result : 'not_retryable',
|
||||
detail: typeof item.detail === 'string' ? item.detail : undefined,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchJobFailures(signal?: AbortSignal): Promise<JobFailure[]> {
|
||||
const response = await fetch('/api/v1/job-failures?state=open&limit=50', { headers: { Accept: 'application/json' }, signal })
|
||||
if (!response.ok) throw new Error(`加载失败队列失败(HTTP ${response.status})`)
|
||||
|
||||
@@ -54,7 +54,9 @@
|
||||
.task-badge.add{color:var(--c-done)}.task-badge.update{color:var(--c-run)}
|
||||
.inbox-task-actions{display:flex;flex-direction:column;gap:6px;align-items:flex-end}
|
||||
.inbox-task-actions .primary{background:var(--ink-strong);color:var(--bg-panel);border-color:var(--ink-strong)}
|
||||
button.danger{color:var(--c-fail)}
|
||||
button.danger{background:var(--c-fail);color:#fff;border-color:var(--c-fail)}
|
||||
button.danger:hover:not(:disabled){background:#991b1b;border-color:#991b1b}
|
||||
button.danger:disabled{background:var(--c-fail-border);color:#fff;border-color:var(--c-fail-border)}
|
||||
.operator-input{font:inherit;padding:5px 8px;border:1px solid var(--border-strong);border-radius:6px;width:120px}
|
||||
.tiny{font-size:12px;color:var(--c-dead)}
|
||||
.monitor-strip{display:flex;overflow-x:auto;background:var(--bg-panel);border:1px solid var(--border-strong);border-radius:8px;font-size:12px}
|
||||
|
||||
Reference in New Issue
Block a user