fix(library): 修正启动收敛误判并提供精确核对
build / build (push) Failing after 1m20s

ReconcileLibraryJobs 不再把 completed capture queue 当成不存在,保留待核对
pending 封存作业;failed/cancelled/missing 仍按真实队列状态处置。管理员核对
入口只允许精确的启动误判形态,并要求 completed queue、export succeeded、
唯一 test_sealed 版本、来源身份与完整归档复核。新增 library_job_repairs
不可变审计表,在同一事务内 CAS 保存原 state/stage/error/recovery/finished_at
及本次 actor/原因/证据后修复;真实失败/取消不开放。

验证:真实隔离 PostgreSQL 启动收敛 RED/GREEN、启动→handler 纵向 pending 与
系统误判 failed、审计保留/幂等、负例;真实 Go 全量与 make verify 17 项通过。
This commit is contained in:
姚章浩
2026-11-06 10:51:21 +08:00
parent 146ac832a2
commit 2eff9447e1
12 changed files with 610 additions and 12 deletions
@@ -35,10 +35,17 @@ CREATE TEMP TABLE library_jobs(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
state varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL DEFAULT '',
stage varchar(32) NOT NULL DEFAULT '', snapshot_id varchar(64) NOT NULL DEFAULT '',
error text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz);
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz);
CREATE TEMP TABLE library_job_stages(
job_id varchar(64) NOT NULL, stage varchar(32) NOT NULL, state varchar(16) NOT NULL,
detail text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, PRIMARY KEY(job_id,stage));
CREATE TEMP TABLE library_job_repairs(
id varchar(64) PRIMARY KEY, job_id varchar(64) NOT NULL, scope varchar(16) NOT NULL,
kind varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL, snapshot_id varchar(64) NOT NULL,
original_state varchar(16) NOT NULL, original_stage varchar(32) NOT NULL DEFAULT '',
original_error text NOT NULL DEFAULT '', original_recovery varchar(255) NOT NULL DEFAULT '',
original_finished_at timestamptz, reason varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
evidence jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL);
CREATE TEMP TABLE library_versions(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(24) NOT NULL,
snapshot_id varchar(64) NOT NULL DEFAULT '', version_label varchar(64) NOT NULL DEFAULT '',
@@ -94,4 +101,31 @@ INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,created_at)
if updated, err := store.ReconcileLibraryCaptureTerminal(ctx, "libseal-other", "internal", "libcap-3", "snap-3", now); err != nil || updated {
t.Fatalf("different existing snapshot must not be overwritten: updated=%v err=%v", updated, err)
}
legacyFinished := now.Add(-time.Minute).Truncate(time.Microsecond)
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,queue_job_id,error,recovery,updated_at,finished_at)
VALUES('libseal-misclassified','internal','seal','failed','libcap-4',$1,$2,$3,$3)`,
"队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", legacyFinished); err != nil {
t.Fatal(err)
}
repaired, err := store.ReconcileMisclassifiedLibraryCapture(ctx, "libseal-misclassified", "internal", "libcap-4", "snap-4",
"", "队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", &legacyFinished,
"librepair-test", "tester", "test repair", map[string]any{"archive_verified": true}, now)
if err != nil || !repaired {
t.Fatalf("misclassification repair=%v err=%v", repaired, err)
}
var originalState, originalError string
var originalFinished *time.Time
if err := pool.QueryRow(ctx, `SELECT original_state, original_error, original_finished_at FROM library_job_repairs WHERE id='librepair-test'`).
Scan(&originalState, &originalError, &originalFinished); err != nil {
t.Fatal(err)
}
if originalState != "failed" || originalError == "" || originalFinished == nil {
t.Fatalf("audit lost original fields: state=%q error=%q finished=%v", originalState, originalError, originalFinished)
}
if repaired, err := store.ReconcileMisclassifiedLibraryCapture(ctx, "libseal-misclassified", "internal", "libcap-4", "snap-4",
"", "队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", &legacyFinished,
"librepair-test-2", "tester", "test repair", map[string]any{}, now); err != nil || repaired {
t.Fatalf("second repair must be false: repaired=%v err=%v", repaired, err)
}
}
+75 -5
View File
@@ -346,6 +346,73 @@ WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$3
return tag.RowsAffected() == 1, nil
}
// ReconcileMisclassifiedLibraryCapture repairs ONLY the known startup
// misclassification: a failed seal whose queue actually completed, with no
// stage/snapshot and the exact legacy error/recovery. The original fields are
// copied to the immutable audit row and the job update is CAS-bound to them;
// any other failed/cancelled row is rejected.
func (s *Store) ReconcileMisclassifiedLibraryCapture(ctx context.Context, jobID, scope, queueJobID, snapshotID,
expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time,
auditID, actor, reason string, evidence map[string]any, now time.Time) (bool, error) {
if jobID == "" || scope == "" || queueJobID == "" || snapshotID == "" || auditID == "" {
return false, errors.New("misclassification repair requires job/scope/queue/snapshot/audit")
}
rawEvidence, err := json.Marshal(evidence)
if err != nil {
return false, fmt.Errorf("encode repair evidence: %w", err)
}
tx, err := s.db.Begin(ctx)
if err != nil {
return false, fmt.Errorf("begin repair: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
var state, stage, errText, recovery, currentSnapshot string
var finished *time.Time
if err := tx.QueryRow(ctx, `
SELECT state, stage, error, recovery, snapshot_id, finished_at
FROM library_jobs WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$3
FOR UPDATE`, jobID, scope, queueJobID).Scan(&state, &stage, &errText, &recovery, &currentSnapshot, &finished); err != nil {
return false, fmt.Errorf("read repair target: %w", err)
}
if state != "failed" || stage != expectedStage || errText != expectedError ||
recovery != expectedRecovery || currentSnapshot != "" ||
(finished == nil) != (expectedFinished == nil) ||
(finished != nil && expectedFinished != nil && !finished.Equal(*expectedFinished)) {
return false, nil
}
tag, err := tx.Exec(ctx, `
UPDATE library_jobs
SET state='succeeded', stage='export', error='', recovery='', snapshot_id=$3,
updated_at=$4, finished_at=$4
WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$5
AND state='failed' AND stage=$6 AND error=$7 AND recovery=$8 AND snapshot_id=''
AND finished_at IS NOT DISTINCT FROM $9`,
jobID, scope, snapshotID, now.UTC(), queueJobID, expectedStage, expectedError, expectedRecovery, expectedFinished)
if err != nil {
return false, fmt.Errorf("repair misclassified seal: %w", err)
}
if tag.RowsAffected() != 1 {
return false, nil
}
tag, err = tx.Exec(ctx, `
INSERT INTO library_job_repairs
(id, job_id, scope, kind, queue_job_id, snapshot_id, original_state, original_stage,
original_error, original_recovery, original_finished_at, reason, actor, evidence, created_at)
VALUES ($1,$2,$3,'seal',$4,$5,'failed',$6,$7,$8,$9,$10,$11,$12,$13)
ON CONFLICT (id) DO NOTHING`, auditID, jobID, scope, queueJobID, snapshotID,
expectedStage, expectedError, expectedRecovery, expectedFinished, reason, actor, rawEvidence, now.UTC())
if err != nil {
return false, fmt.Errorf("write repair audit: %w", err)
}
if tag.RowsAffected() != 1 {
return false, errors.New("repair audit already exists")
}
if err := tx.Commit(ctx); err != nil {
return false, fmt.Errorf("commit repair: %w", err)
}
return true, nil
}
// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。
//
// 与旧实现的区别(F02/F09 的根因):
@@ -724,7 +791,7 @@ func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time, kinds [
if len(kinds) == 0 {
return 0, errors.New("reconcile library jobs requires the kinds this executor owns")
}
if _, err := s.db.Exec(ctx, `
tag, err := s.db.Exec(ctx, `
UPDATE library_jobs AS lj
SET state='failed', error='队列中已无对应作业,业务作业未收到结果',
recovery=CASE WHEN lj.kind IN ('sync','restore','recover','baseline')
@@ -737,12 +804,15 @@ WHERE lj.state IN ('pending','running')
AND lj.queue_job_id <> ''
AND NOT EXISTS (
SELECT 1 FROM jobs AS j
WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry')
)`, now.UTC(), kinds, s.libraryTargetID); err != nil {
WHERE j.id = lj.queue_job_id
AND j.state IN ('pending','running','retry','completed')
)`, now.UTC(), kinds, s.libraryTargetID)
if err != nil {
return 0, fmt.Errorf("reconcile library jobs: %w", err)
}
first := int(tag.RowsAffected())
// 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。
tag, err := s.db.Exec(ctx, `
tag, err = s.db.Exec(ctx, `
UPDATE library_jobs
SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1
WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND (kind='seal' OR target_id=$4) AND created_at < $2`,
@@ -750,7 +820,7 @@ WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND
if err != nil {
return 0, fmt.Errorf("reconcile unattached library jobs: %w", err)
}
return int(tag.RowsAffected()), nil
return first + int(tag.RowsAffected()), nil
}
// LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。
@@ -0,0 +1,79 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// The worker startup reconciliation must not turn a completed capture queue
// into the legacy "queue does not exist" failure. This is a real PostgreSQL
// fixture with the actual varchar shapes; the RED case is the old SQL which
// only accepted pending/running/retry.
func TestStartupReconcileKeepsCompletedCaptureJob(t *testing.T) {
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Fatal("isolated database required")
}
cfg.MaxConns = 1
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_temp"
ctx := context.Background()
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
now := time.Now().UTC()
if _, err := pool.Exec(ctx, `
CREATE TEMP TABLE library_jobs(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
state varchar(16) NOT NULL, stage varchar(32) NOT NULL DEFAULT '',
queue_job_id varchar(64) NOT NULL DEFAULT '', target_id text NOT NULL DEFAULT '',
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '',
snapshot_id varchar(64) NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz, created_at timestamptz NOT NULL DEFAULT now());
CREATE TEMP TABLE jobs(id varchar(64) PRIMARY KEY, type varchar(64) NOT NULL, state varchar(16) NOT NULL);`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,queue_job_id,updated_at)
VALUES('libseal-completed','internal','seal','pending','q-completed',$1),
('libseal-failed','internal','seal','pending','q-failed',$1),
('libseal-missing','internal','seal','pending','q-missing',$1)`, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,state) VALUES('q-completed','library.capture','completed'),
('q-failed','library.capture','failed')`); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"})
if err != nil {
t.Fatal(err)
}
var state, errText string
if err := pool.QueryRow(ctx, `SELECT state, error FROM library_jobs WHERE id='libseal-completed'`).Scan(&state, &errText); err != nil {
t.Fatal(err)
}
if state != "pending" || errText != "" {
t.Fatalf("completed queue was misclassified: state=%q error=%q", state, errText)
}
if n != 2 {
t.Fatalf("reconciled=%d, want only failed/missing queue jobs (2)", n)
}
for _, id := range []string{"libseal-failed", "libseal-missing"} {
var got string
if err := pool.QueryRow(ctx, `SELECT state FROM library_jobs WHERE id=$1`, id).Scan(&got); err != nil {
t.Fatal(err)
}
if got != "failed" {
t.Fatalf("%s state=%q, want failed", id, got)
}
}
}
@@ -0,0 +1 @@
DROP TABLE IF EXISTS library_job_repairs;
@@ -0,0 +1,19 @@
-- Immutable audit for the only permitted automatic/system-misclassification repair.
CREATE TABLE library_job_repairs (
id VARCHAR(64) PRIMARY KEY,
job_id VARCHAR(64) NOT NULL,
scope VARCHAR(16) NOT NULL,
kind VARCHAR(16) NOT NULL,
queue_job_id VARCHAR(64) NOT NULL,
snapshot_id VARCHAR(64) NOT NULL,
original_state VARCHAR(16) NOT NULL,
original_stage VARCHAR(32) NOT NULL DEFAULT '',
original_error TEXT NOT NULL DEFAULT '',
original_recovery VARCHAR(255) NOT NULL DEFAULT '',
original_finished_at TIMESTAMPTZ,
reason VARCHAR(255) NOT NULL,
actor VARCHAR(128) NOT NULL DEFAULT '',
evidence JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL
);
CREATE UNIQUE INDEX library_job_repairs_identity ON library_job_repairs(job_id, snapshot_id);
@@ -11,11 +11,14 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 79 {
t.Fatalf("migration count = %d, want 79", len(migrations))
if len(migrations) != 80 {
t.Fatalf("migration count = %d, want 80", len(migrations))
}
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
// 证据按 target_id 落库(换目标旧证据自动失效)。
if repair := migrations[79]; repair.Version != 80 {
t.Fatalf("migration 80 = %#v", repair)
}
if verification := migrations[78]; verification.Version != 79 ||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
!strings.Contains(verification.UpSQL, "'verify'") {