fix(monitor): 修复取件死信无法重排与列表 NULL 标志导致的 500
- platform/postgres: listTestPublicationsSQL 的 can_start/can_retry 在 LEFT JOIN
内容对象为空时算出 NULL,Go 侧 bool 扫描失败使整个列表接口 500,前端三个 lane
全空(表现为“待入库 0 / 全部任务 0”)。改为 COALESCE(..., false)。
- 新增 POST /api/v1/job-failures/{id}/requeue:取件任务的幂等键绑在版本上,
失败那次已占住它,重新标注同一文件不会再入队;现在按新轮次 (…:r<n>) 重排、
复位意图取件状态并把死信标记为 requeued(幂等信封 + 审计字段)。
- web: 死信区对 artifact.fetch 显示「重新入队」;系统设置新增「知识库绑定」分区,
挂上一直没被渲染的 RoutingBindingsSection,导入目标库可在页面上切换。
- 测试:真实 PostgreSQL 集成测试(NULL 折叠 + 重排轮次/幂等/拒绝)、前端死信按钮
与设置分区渲染测试。
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// Two defects this pins down (both hit on 2026-10-08 while running the first
|
||||
// end-to-end document):
|
||||
//
|
||||
// 1. listTestPublicationsSQL computed can_start/can_retry against a
|
||||
// LEFT JOINed content object, so a planned row whose artifact was never
|
||||
// fetched produced NULL — and scanning NULL into a Go bool made the whole
|
||||
// /api/v1/test-publications call 500, which the monitor renders as three
|
||||
// empty lanes ("待入库 0 / 全部任务 0").
|
||||
// 2. An artifact.fetch dead letter carried no publication id, so the UI had no
|
||||
// retry affordance and the fetch idempotency key (artifact-fetch:<version>)
|
||||
// was permanently occupied by the failed attempt — re-marking the file
|
||||
// could never re-enqueue it.
|
||||
func TestPlannedPublicationFlagsFoldNullAndArtifactFetchRequeue(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err = ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-fetch','dingtalk','Fetch DT',true);
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-fetch','conn-fetch','node-1',repeat('b',64),'技术文档/环境部署/关闭PC2远程桌面弹窗.docx','关闭PC2远程桌面弹窗.docx','document','locator:h1','internal','configured_scope');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state)
|
||||
VALUES('version-fetch','item-fetch','locator:h1',NULL,'unknown');
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope)
|
||||
VALUES('pub-fetch','version-fetch','kb-internal','planned','internal');
|
||||
INSERT INTO jobs(id,type,payload,state,priority,max_attempts,idempotency_key,last_error,last_error_code)
|
||||
VALUES('job-fetch-1','artifact.fetch',
|
||||
'{"source_connection_id":"conn-fetch","node_id":"node-1","version_id":"version-fetch"}'::jsonb,
|
||||
'failed',20,8,'artifact-fetch:version-fetch','artifact_fetch_rejected: http_401','artifact_invalid');
|
||||
INSERT INTO job_failures(id,job_id,job_type,publication_id,error_code,last_error,attempt,max_attempts_at_failure,state)
|
||||
VALUES('jf-fetch-1','job-fetch-1','artifact.fetch','','artifact_invalid','artifact_fetch_rejected: http_401',1,8,'open');
|
||||
INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,decided_by,relative_path,name,locator_hash,fetch_state,last_error)
|
||||
VALUES('conn-fetch','node-1','internal','tester','技术文档/环境部署/关闭PC2远程桌面弹窗.docx','关闭PC2远程桌面弹窗.docx',repeat('c',64),'failed','artifact_fetch_rejected: http_401');
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, time.October, 8, 3, 40, 0, 0, time.UTC)
|
||||
|
||||
// 1. The unfetched planned row must list cleanly with false flags, not error.
|
||||
page, err := store.ListTestPublications(ctx, governance.TestPublicationFilter{Limit: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("ListTestPublications() error = %v", err)
|
||||
}
|
||||
if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "pub-fetch" {
|
||||
t.Fatalf("page = %#v", page)
|
||||
}
|
||||
if page.Items[0].CanStart || page.Items[0].CanRetry {
|
||||
t.Fatalf("NULL flags must fold to false, got %#v", page.Items[0])
|
||||
}
|
||||
if page.Items[0].Title != "关闭PC2远程桌面弹窗.docx" {
|
||||
t.Fatalf("title = %q", page.Items[0].Title)
|
||||
}
|
||||
|
||||
// 2. Requeue opens a new round instead of replaying the occupied key.
|
||||
request := governance.RequeueArtifactFetchRequest{
|
||||
FailureID: "jf-fetch-1", OperatorRef: "tester", Reason: "钉钉会话已刷新", Now: now,
|
||||
}
|
||||
result, replayed, err := store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-1")
|
||||
if err != nil {
|
||||
t.Fatalf("RequeueArtifactFetchWithOperation() error = %v", err)
|
||||
}
|
||||
if replayed || result.AlreadyQueued {
|
||||
t.Fatalf("first requeue must enqueue, got %#v replayed=%v", result, replayed)
|
||||
}
|
||||
if result.Round != 2 || result.VersionID != "version-fetch" || result.JobID == "" {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
var state, key string
|
||||
if err = pool.QueryRow(ctx, `SELECT state, idempotency_key FROM jobs WHERE id = $1`, result.JobID).Scan(&state, &key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != "pending" || key != "artifact-fetch:version-fetch:r2" {
|
||||
t.Fatalf("requeued job = %s / %s", state, key)
|
||||
}
|
||||
var failureState, fetchState string
|
||||
if err = pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE id = 'jf-fetch-1'`).Scan(&failureState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failureState != "requeued" {
|
||||
t.Fatalf("dead letter state = %s", failureState)
|
||||
}
|
||||
if err = pool.QueryRow(ctx, `SELECT fetch_state FROM catalog_import_intents WHERE node_id = 'node-1'`).Scan(&fetchState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fetchState != "pending" {
|
||||
t.Fatalf("intent fetch state = %s", fetchState)
|
||||
}
|
||||
|
||||
// Replaying the SAME operation must return the stored result, not enqueue again.
|
||||
again, replayedAgain, err := store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-1")
|
||||
if err != nil {
|
||||
t.Fatalf("replay error = %v", err)
|
||||
}
|
||||
if !replayedAgain || again.JobID != result.JobID || again.Round != result.Round {
|
||||
t.Fatalf("replay = %#v replayed=%v", again, replayedAgain)
|
||||
}
|
||||
var fetchJobs int
|
||||
if err = pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type = 'artifact.fetch'`).Scan(&fetchJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fetchJobs != 2 {
|
||||
t.Fatalf("artifact.fetch job count = %d, want 2 (original + one requeue)", fetchJobs)
|
||||
}
|
||||
|
||||
// A resolved dead letter is no longer requeueable.
|
||||
if _, _, err = store.RequeueArtifactFetchWithOperation(ctx, request, "op-requeue-2"); err == nil {
|
||||
t.Fatal("second requeue of the same dead letter must fail")
|
||||
} else if !errors.Is(err, ErrJobFailureNotRequeueable) {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,135 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// ErrJobFailureNotRequeueable marks a dead letter that cannot be re-enqueued:
|
||||
// not an artifact fetch, already resolved, or missing its source coordinates.
|
||||
var ErrJobFailureNotRequeueable = errors.New("job failure is not requeueable")
|
||||
|
||||
const loadRequeueableArtifactFailureSQL = `
|
||||
SELECT jf.state, jf.job_type,
|
||||
COALESCE(j.payload->>'source_connection_id',''),
|
||||
COALESCE(j.payload->>'node_id',''),
|
||||
COALESCE(j.payload->>'version_id',''),
|
||||
COALESCE(NULLIF(intent.name,''),'')
|
||||
FROM job_failures jf
|
||||
LEFT JOIN jobs j ON j.id = jf.job_id
|
||||
LEFT JOIN catalog_import_intents intent
|
||||
ON intent.source_connection_id = j.payload->>'source_connection_id'
|
||||
AND intent.node_id = j.payload->>'node_id'
|
||||
WHERE jf.id = $1
|
||||
FOR UPDATE OF jf
|
||||
`
|
||||
|
||||
const countArtifactFetchRoundsSQL = `
|
||||
SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch' AND payload->>'version_id' = $1
|
||||
`
|
||||
|
||||
const requeueArtifactFetchJobSQL = `
|
||||
INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
||||
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3)
|
||||
ON CONFLICT (idempotency_key) DO NOTHING
|
||||
`
|
||||
|
||||
const resetIntentFetchStateSQL = `
|
||||
UPDATE catalog_import_intents SET fetch_state='pending', last_error='', updated_at=$3
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')
|
||||
`
|
||||
|
||||
const resolveRequeuedFailureSQL = `
|
||||
UPDATE job_failures
|
||||
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
||||
WHERE id = $1 AND state = 'open'
|
||||
`
|
||||
|
||||
// RequeueArtifactFetchWithOperation re-enqueues one terminal 源头取件 dead
|
||||
// letter under a new round-suffixed idempotency key, resets the intent's fetch
|
||||
// state and closes the dead letter — all in one operation envelope.
|
||||
func (s *Store) RequeueArtifactFetchWithOperation(ctx context.Context, request governance.RequeueArtifactFetchRequest, operationID string) (governance.RequeueArtifactFetchResult, bool, error) {
|
||||
if operationID == "" || len(operationID) > 128 {
|
||||
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid operation id")
|
||||
}
|
||||
if request.FailureID == "" || len(request.FailureID) > 64 {
|
||||
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid failure id")
|
||||
}
|
||||
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 500 || request.Now.IsZero() {
|
||||
return governance.RequeueArtifactFetchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
||||
}
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
FailureID string `json:"failure_id"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
}{request.FailureID, request.OperatorRef, request.Reason})
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, false, err
|
||||
}
|
||||
raw, replayed, err := s.executeOperation(ctx, "artifact.fetch.requeue", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := requeueArtifactFetchTx(ctx, tx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, false, err
|
||||
}
|
||||
var result governance.RequeueArtifactFetchResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, false, errors.New("invalid stored requeue result")
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func requeueArtifactFetchTx(ctx context.Context, tx pgx.Tx, request governance.RequeueArtifactFetchRequest) (governance.RequeueArtifactFetchResult, error) {
|
||||
var state, jobType, connectionID, nodeID, versionID, docName string
|
||||
err := tx.QueryRow(ctx, loadRequeueableArtifactFailureSQL, request.FailureID).
|
||||
Scan(&state, &jobType, &connectionID, &nodeID, &versionID, &docName)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
|
||||
}
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("load requeueable failure: %w", err)
|
||||
}
|
||||
if jobType != "artifact.fetch" || state != "open" || connectionID == "" || nodeID == "" || versionID == "" {
|
||||
return governance.RequeueArtifactFetchResult{}, ErrJobFailureNotRequeueable
|
||||
}
|
||||
var rounds int
|
||||
if err := tx.QueryRow(ctx, countArtifactFetchRoundsSQL, versionID).Scan(&rounds); err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("count artifact fetch rounds: %w", err)
|
||||
}
|
||||
round := rounds + 1
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{
|
||||
"source_connection_id": connectionID, "node_id": nodeID, "version_id": versionID,
|
||||
})
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, errors.New("encode artifact fetch job")
|
||||
}
|
||||
tag, err := tx.Exec(ctx, requeueArtifactFetchJobSQL, jobID, string(payload), request.Now,
|
||||
fmt.Sprintf("artifact-fetch:%s:r%d", versionID, round))
|
||||
if err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("requeue artifact fetch: %w", err)
|
||||
}
|
||||
result := governance.RequeueArtifactFetchResult{
|
||||
FailureID: request.FailureID, JobID: jobID, VersionID: versionID, Round: round, DocName: docName,
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
// The round key already exists (concurrent requeue): report it instead of
|
||||
// resolving the dead letter twice.
|
||||
result.AlreadyQueued = true
|
||||
return result, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, resetIntentFetchStateSQL, connectionID, nodeID, request.Now); err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("reset intent fetch state: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, resolveRequeuedFailureSQL, request.FailureID, request.Now, request.OperatorRef, request.Reason); err != nil {
|
||||
return governance.RequeueArtifactFetchResult{}, fmt.Errorf("resolve job failure: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const listJobFailuresSQL = `
|
||||
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
|
||||
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
|
||||
|
||||
@@ -128,16 +128,20 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
|
||||
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
|
||||
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
|
||||
) THEN 'updated' ELSE 'added' END AS change_type,
|
||||
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
|
||||
-- Both flags compare against a LEFT-JOINed content object, so the raw
|
||||
-- expression is NULL (not false) for a row whose artifact was never
|
||||
-- fetched; a NULL here makes the whole list fail to scan into a Go bool
|
||||
-- and 500s the endpoint. Fold NULL to false.
|
||||
COALESCE((p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
|
||||
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
|
||||
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
|
||||
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)) AS can_start,
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)), false) AS can_start,
|
||||
p.retry_count, p.last_error,
|
||||
((p.processing_state IN ('failed', 'stalled')
|
||||
COALESCE(((p.processing_state IN ('failed', 'stalled')
|
||||
OR (p.ingest_state = 'failed' AND p.processing_state = 'pending'))
|
||||
AND co.content_state = 'verified'
|
||||
AND co.local_spool_ref <> '') AS can_retry,
|
||||
AND co.local_spool_ref <> ''), false) AS can_retry,
|
||||
p.updated_at
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
|
||||
|
||||
Reference in New Issue
Block a user