fix(monitor): 修复取件死信无法重排与列表 NULL 标志导致的 500

- platform/postgres: listTestPublicationsSQL 的 can_start/can_retry 在 LEFT JOIN
  内容对象为空时算出 NULL,Go 侧 bool 扫描失败使整个列表接口 500,前端三个 lane
  全空(表现为“待入库 0 / 全部任务 0”)。改为 COALESCE(..., false)。
- 新增 POST /api/v1/job-failures/{id}/requeue:取件任务的幂等键绑在版本上,
  失败那次已占住它,重新标注同一文件不会再入队;现在按新轮次 (…:r<n>) 重排、
  复位意图取件状态并把死信标记为 requeued(幂等信封 + 审计字段)。
- web: 死信区对 artifact.fetch 显示「重新入队」;系统设置新增「知识库绑定」分区,
  挂上一直没被渲染的 RoutingBindingsSection,导入目标库可在页面上切换。
- 测试:真实 PostgreSQL 集成测试(NULL 折叠 + 重排轮次/幂等/拒绝)、前端死信按钮
  与设置分区渲染测试。
This commit is contained in:
hermes
2026-10-08 12:09:34 +08:00
parent 22a6bc5004
commit cddc4c90d1
12 changed files with 495 additions and 8 deletions
@@ -0,0 +1,135 @@
package postgres
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
// Two defects this pins down (both hit on 2026-10-08 while running the first
// end-to-end document):
//
// 1. listTestPublicationsSQL computed can_start/can_retry against a
// LEFT JOINed content object, so a planned row whose artifact was never
// fetched produced NULL — and scanning NULL into a Go bool made the whole
// /api/v1/test-publications call 500, which the monitor renders as three
// empty lanes ("待入库 0 / 全部任务 0").
// 2. An artifact.fetch dead letter carried no publication id, so the UI had no
// retry affordance and the fetch idempotency key (artifact-fetch:<version>)
// was permanently occupied by the failed attempt — re-marking the file
// could never re-enqueue it.
func TestPlannedPublicationFlagsFoldNullAndArtifactFetchRequeue(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err = ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-fetch','dingtalk','Fetch DT',true);
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-fetch','conn-fetch','node-1',repeat('b',64),'技术文档/环境部署/关闭PC2远程桌面弹窗.docx','关闭PC2远程桌面弹窗.docx','document','locator:h1','internal','configured_scope');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state)
VALUES('version-fetch','item-fetch','locator:h1',NULL,'unknown');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope)
VALUES('pub-fetch','version-fetch','kb-internal','planned','internal');
INSERT INTO jobs(id,type,payload,state,priority,max_attempts,idempotency_key,last_error,last_error_code)
VALUES('job-fetch-1','artifact.fetch',
'{"source_connection_id":"conn-fetch","node_id":"node-1","version_id":"version-fetch"}'::jsonb,
'failed',20,8,'artifact-fetch:version-fetch','artifact_fetch_rejected: http_401','artifact_invalid');
INSERT INTO job_failures(id,job_id,job_type,publication_id,error_code,last_error,attempt,max_attempts_at_failure,state)
VALUES('jf-fetch-1','job-fetch-1','artifact.fetch','','artifact_invalid','artifact_fetch_rejected: http_401',1,8,'open');
INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,decided_by,relative_path,name,locator_hash,fetch_state,last_error)
VALUES('conn-fetch','node-1','internal','tester','技术文档/环境部署/关闭PC2远程桌面弹窗.docx','关闭PC2远程桌面弹窗.docx',repeat('c',64),'failed','artifact_fetch_rejected: http_401');
`); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, time.October, 8, 3, 40, 0, 0, time.UTC)
// 1. The unfetched planned row must list cleanly with false flags, not error.
page, err := store.ListTestPublications(ctx, governance.TestPublicationFilter{Limit: 50})
if err != nil {
t.Fatalf("ListTestPublications() error = %v", err)
}
if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "pub-fetch" {
t.Fatalf("page = %#v", page)
}
if page.Items[0].CanStart || page.Items[0].CanRetry {
t.Fatalf("NULL flags must fold to false, got %#v", page.Items[0])
}
if page.Items[0].Title != "关闭PC2远程桌面弹窗.docx" {
t.Fatalf("title = %q", page.Items[0].Title)
}
// 2. Requeue opens a new round instead of replaying the occupied key.
request := governance.RequeueArtifactFetchRequest{
FailureID: "jf-fetch-1", OperatorRef: "tester", Reason: "钉钉会话已刷新", Now: now,
}
result, replayed, err := store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-1")
if err != nil {
t.Fatalf("RequeueArtifactFetchWithOperation() error = %v", err)
}
if replayed || result.AlreadyQueued {
t.Fatalf("first requeue must enqueue, got %#v replayed=%v", result, replayed)
}
if result.Round != 2 || result.VersionID != "version-fetch" || result.JobID == "" {
t.Fatalf("result = %#v", result)
}
var state, key string
if err = pool.QueryRow(ctx, `SELECT state, idempotency_key FROM jobs WHERE id = $1`, result.JobID).Scan(&state, &key); err != nil {
t.Fatal(err)
}
if state != "pending" || key != "artifact-fetch:version-fetch:r2" {
t.Fatalf("requeued job = %s / %s", state, key)
}
var failureState, fetchState string
if err = pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE id = 'jf-fetch-1'`).Scan(&failureState); err != nil {
t.Fatal(err)
}
if failureState != "requeued" {
t.Fatalf("dead letter state = %s", failureState)
}
if err = pool.QueryRow(ctx, `SELECT fetch_state FROM catalog_import_intents WHERE node_id = 'node-1'`).Scan(&fetchState); err != nil {
t.Fatal(err)
}
if fetchState != "pending" {
t.Fatalf("intent fetch state = %s", fetchState)
}
// Replaying the SAME operation must return the stored result, not enqueue again.
again, replayedAgain, err := store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-1")
if err != nil {
t.Fatalf("replay error = %v", err)
}
if !replayedAgain || again.JobID != result.JobID || again.Round != result.Round {
t.Fatalf("replay = %#v replayed=%v", again, replayedAgain)
}
var fetchJobs int
if err = pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type = 'artifact.fetch'`).Scan(&fetchJobs); err != nil {
t.Fatal(err)
}
if fetchJobs != 2 {
t.Fatalf("artifact.fetch job count = %d, want 2 (original + one requeue)", fetchJobs)
}
// A resolved dead letter is no longer requeueable.
if _, _, err = store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-2"); err == nil {
t.Fatal("second requeue of the same dead letter must fail")
} else if !errors.Is(err, ErrJobFailureNotRequeueable) {
t.Fatalf("unexpected error: %v", err)
}
}
+129
View File
@@ -11,6 +11,135 @@ import (
"knowledge-governance/internal/governance"
)
// ErrJobFailureNotRequeueable marks a dead letter that cannot be re-enqueued:
// not an artifact fetch, already resolved, or missing its source coordinates.
var ErrJobFailureNotRequeueable = errors.New("job failure is not requeueable")
const loadRequeueableArtifactFailureSQL = `
SELECT jf.state, jf.job_type,
COALESCE(j.payload->>'source_connection_id',''),
COALESCE(j.payload->>'node_id',''),
COALESCE(j.payload->>'version_id',''),
COALESCE(NULLIF(intent.name,''),'')
FROM job_failures jf
LEFT JOIN jobs j ON j.id = jf.job_id
LEFT JOIN catalog_import_intents intent
ON intent.source_connection_id = j.payload->>'source_connection_id'
AND intent.node_id = j.payload->>'node_id'
WHERE jf.id = $1
FOR UPDATE OF jf
`
const countArtifactFetchRoundsSQL = `
SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch' AND payload->>'version_id' = $1
`
const requeueArtifactFetchJobSQL = `
INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3)
ON CONFLICT (idempotency_key) DO NOTHING
`
const resetIntentFetchStateSQL = `
UPDATE catalog_import_intents SET fetch_state='pending', last_error='', updated_at=$3
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')
`
const resolveRequeuedFailureSQL = `
UPDATE job_failures
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
WHERE id = $1 AND state = 'open'
`
// RequeueArtifactFetchWithOperation re-enqueues one terminal 源头取件 dead
// letter under a new round-suffixed idempotency key, resets the intent's fetch
// state and closes the dead letter — all in one operation envelope.
func (s *Store) RequeueArtifactFetchWithOperation(ctx context.Context, request governance.RequeueArtifactFetchRequest, operationID string) (governance.RequeueArtifactFetchResult, bool, error) {
if operationID == "" || len(operationID) > 128 {
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid operation id")
}
if request.FailureID == "" || len(request.FailureID) > 64 {
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid failure id")
}
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 500 || request.Now.IsZero() {
return governance.RequeueArtifactFetchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
}
fingerprint, err := operationFingerprint(struct {
FailureID string `json:"failure_id"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{request.FailureID, request.OperatorRef, request.Reason})
if err != nil {
return governance.RequeueArtifactFetchResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "artifact.fetch.requeue", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := requeueArtifactFetchTx(ctx, tx, request)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.RequeueArtifactFetchResult{}, false, err
}
var result governance.RequeueArtifactFetchResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid stored requeue result")
}
return result, replayed, nil
}
func requeueArtifactFetchTx(ctx context.Context, tx pgx.Tx, request governance.RequeueArtifactFetchRequest) (governance.RequeueArtifactFetchResult, error) {
var state, jobType, connectionID, nodeID, versionID, docName string
err := tx.QueryRow(ctx, loadRequeueableArtifactFailureSQL, request.FailureID).
Scan(&state, &jobType, &connectionID, &nodeID, &versionID, &docName)
if errors.Is(err, pgx.ErrNoRows) {
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
}
if err != nil {
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("load requeueable failure: %w", err)
}
if jobType != "artifact.fetch" || state != "open" || connectionID == "" || nodeID == "" || versionID == "" {
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
}
var rounds int
if err := tx.QueryRow(ctx, countArtifactFetchRoundsSQL, versionID).Scan(&rounds); err != nil {
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("count artifact fetch rounds: %w", err)
}
round := rounds + 1
jobID, err := newOpaqueID("job-")
if err != nil {
return governance.RequeueArtifactFetchResult{}, err
}
payload, err := json.Marshal(map[string]string{
"source_connection_id": connectionID, "node_id": nodeID, "version_id": versionID,
})
if err != nil {
return governance.RequeueArtifactFetchResult{}, errors.New("encode artifact fetch job")
}
tag, err := tx.Exec(ctx, requeueArtifactFetchJobSQL, jobID, string(payload), request.Now,
fmt.Sprintf("artifact-fetch:%s:r%d", versionID, round))
if err != nil {
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("requeue artifact fetch: %w", err)
}
result := governance.RequeueArtifactFetchResult{
FailureID: request.FailureID, JobID: jobID, VersionID: versionID, Round: round, DocName: docName,
}
if tag.RowsAffected() == 0 {
// The round key already exists (concurrent requeue): report it instead of
// resolving the dead letter twice.
result.AlreadyQueued = true
return result, nil
}
if _, err := tx.Exec(ctx, resetIntentFetchStateSQL, connectionID, nodeID, request.Now); err != nil {
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("reset intent fetch state: %w", err)
}
if _, err := tx.Exec(ctx, resolveRequeuedFailureSQL, request.FailureID, request.Now, request.OperatorRef, request.Reason); err != nil {
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("resolve job failure: %w", err)
}
return result, nil
}
const listJobFailuresSQL = `
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
@@ -128,16 +128,20 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
) THEN 'updated' ELSE 'added' END AS change_type,
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
-- Both flags compare against a LEFT-JOINed content object, so the raw
-- expression is NULL (not false) for a row whose artifact was never
-- fetched; a NULL here makes the whole list fail to scan into a Go bool
-- and 500s the endpoint. Fold NULL to false.
COALESCE((p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
AND NOT EXISTS (SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)) AS can_start,
AND NOT EXISTS (SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)), false) AS can_start,
p.retry_count, p.last_error,
((p.processing_state IN ('failed', 'stalled')
COALESCE(((p.processing_state IN ('failed', 'stalled')
OR (p.ingest_state = 'failed' AND p.processing_state = 'pending'))
AND co.content_state = 'verified'
AND co.local_spool_ref <> '') AS can_retry,
AND co.local_spool_ref <> ''), false) AS can_retry,
p.updated_at
FROM test_publications p
JOIN source_item_versions siv ON siv.id = p.source_item_version_id