ReconcileLibraryJobs 不再把 completed capture queue 当成不存在,保留待核对 pending 封存作业;failed/cancelled/missing 仍按真实队列状态处置。管理员核对 入口只允许精确的启动误判形态,并要求 completed queue、export succeeded、 唯一 test_sealed 版本、来源身份与完整归档复核。新增 library_job_repairs 不可变审计表,在同一事务内 CAS 保存原 state/stage/error/recovery/finished_at 及本次 actor/原因/证据后修复;真实失败/取消不开放。 验证:真实隔离 PostgreSQL 启动收敛 RED/GREEN、启动→handler 纵向 pending 与 系统误判 failed、审计保留/幂等、负例;真实 Go 全量与 make verify 17 项通过。
This commit is contained in:
@@ -35,10 +35,17 @@ CREATE TEMP TABLE library_jobs(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
||||
state varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL DEFAULT '',
|
||||
stage varchar(32) NOT NULL DEFAULT '', snapshot_id varchar(64) NOT NULL DEFAULT '',
|
||||
error text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz);
|
||||
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz);
|
||||
CREATE TEMP TABLE library_job_stages(
|
||||
job_id varchar(64) NOT NULL, stage varchar(32) NOT NULL, state varchar(16) NOT NULL,
|
||||
detail text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, PRIMARY KEY(job_id,stage));
|
||||
CREATE TEMP TABLE library_job_repairs(
|
||||
id varchar(64) PRIMARY KEY, job_id varchar(64) NOT NULL, scope varchar(16) NOT NULL,
|
||||
kind varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL, snapshot_id varchar(64) NOT NULL,
|
||||
original_state varchar(16) NOT NULL, original_stage varchar(32) NOT NULL DEFAULT '',
|
||||
original_error text NOT NULL DEFAULT '', original_recovery varchar(255) NOT NULL DEFAULT '',
|
||||
original_finished_at timestamptz, reason varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
|
||||
evidence jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL);
|
||||
CREATE TEMP TABLE library_versions(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(24) NOT NULL,
|
||||
snapshot_id varchar(64) NOT NULL DEFAULT '', version_label varchar(64) NOT NULL DEFAULT '',
|
||||
@@ -94,4 +101,31 @@ INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,created_at)
|
||||
if updated, err := store.ReconcileLibraryCaptureTerminal(ctx, "libseal-other", "internal", "libcap-3", "snap-3", now); err != nil || updated {
|
||||
t.Fatalf("different existing snapshot must not be overwritten: updated=%v err=%v", updated, err)
|
||||
}
|
||||
|
||||
legacyFinished := now.Add(-time.Minute).Truncate(time.Microsecond)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,queue_job_id,error,recovery,updated_at,finished_at)
|
||||
VALUES('libseal-misclassified','internal','seal','failed','libcap-4',$1,$2,$3,$3)`,
|
||||
"队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", legacyFinished); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repaired, err := store.ReconcileMisclassifiedLibraryCapture(ctx, "libseal-misclassified", "internal", "libcap-4", "snap-4",
|
||||
"", "队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", &legacyFinished,
|
||||
"librepair-test", "tester", "test repair", map[string]any{"archive_verified": true}, now)
|
||||
if err != nil || !repaired {
|
||||
t.Fatalf("misclassification repair=%v err=%v", repaired, err)
|
||||
}
|
||||
var originalState, originalError string
|
||||
var originalFinished *time.Time
|
||||
if err := pool.QueryRow(ctx, `SELECT original_state, original_error, original_finished_at FROM library_job_repairs WHERE id='librepair-test'`).
|
||||
Scan(&originalState, &originalError, &originalFinished); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if originalState != "failed" || originalError == "" || originalFinished == nil {
|
||||
t.Fatalf("audit lost original fields: state=%q error=%q finished=%v", originalState, originalError, originalFinished)
|
||||
}
|
||||
if repaired, err := store.ReconcileMisclassifiedLibraryCapture(ctx, "libseal-misclassified", "internal", "libcap-4", "snap-4",
|
||||
"", "队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", &legacyFinished,
|
||||
"librepair-test-2", "tester", "test repair", map[string]any{}, now); err != nil || repaired {
|
||||
t.Fatalf("second repair must be false: repaired=%v err=%v", repaired, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,6 +346,73 @@ WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$3
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// ReconcileMisclassifiedLibraryCapture repairs ONLY the known startup
|
||||
// misclassification: a failed seal whose queue actually completed, with no
|
||||
// stage/snapshot and the exact legacy error/recovery. The original fields are
|
||||
// copied to the immutable audit row and the job update is CAS-bound to them;
|
||||
// any other failed/cancelled row is rejected.
|
||||
func (s *Store) ReconcileMisclassifiedLibraryCapture(ctx context.Context, jobID, scope, queueJobID, snapshotID,
|
||||
expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time,
|
||||
auditID, actor, reason string, evidence map[string]any, now time.Time) (bool, error) {
|
||||
if jobID == "" || scope == "" || queueJobID == "" || snapshotID == "" || auditID == "" {
|
||||
return false, errors.New("misclassification repair requires job/scope/queue/snapshot/audit")
|
||||
}
|
||||
rawEvidence, err := json.Marshal(evidence)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("encode repair evidence: %w", err)
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin repair: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var state, stage, errText, recovery, currentSnapshot string
|
||||
var finished *time.Time
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT state, stage, error, recovery, snapshot_id, finished_at
|
||||
FROM library_jobs WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$3
|
||||
FOR UPDATE`, jobID, scope, queueJobID).Scan(&state, &stage, &errText, &recovery, ¤tSnapshot, &finished); err != nil {
|
||||
return false, fmt.Errorf("read repair target: %w", err)
|
||||
}
|
||||
if state != "failed" || stage != expectedStage || errText != expectedError ||
|
||||
recovery != expectedRecovery || currentSnapshot != "" ||
|
||||
(finished == nil) != (expectedFinished == nil) ||
|
||||
(finished != nil && expectedFinished != nil && !finished.Equal(*expectedFinished)) {
|
||||
return false, nil
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs
|
||||
SET state='succeeded', stage='export', error='', recovery='', snapshot_id=$3,
|
||||
updated_at=$4, finished_at=$4
|
||||
WHERE id=$1 AND scope=$2 AND kind='seal' AND queue_job_id=$5
|
||||
AND state='failed' AND stage=$6 AND error=$7 AND recovery=$8 AND snapshot_id=''
|
||||
AND finished_at IS NOT DISTINCT FROM $9`,
|
||||
jobID, scope, snapshotID, now.UTC(), queueJobID, expectedStage, expectedError, expectedRecovery, expectedFinished)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("repair misclassified seal: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return false, nil
|
||||
}
|
||||
tag, err = tx.Exec(ctx, `
|
||||
INSERT INTO library_job_repairs
|
||||
(id, job_id, scope, kind, queue_job_id, snapshot_id, original_state, original_stage,
|
||||
original_error, original_recovery, original_finished_at, reason, actor, evidence, created_at)
|
||||
VALUES ($1,$2,$3,'seal',$4,$5,'failed',$6,$7,$8,$9,$10,$11,$12,$13)
|
||||
ON CONFLICT (id) DO NOTHING`, auditID, jobID, scope, queueJobID, snapshotID,
|
||||
expectedStage, expectedError, expectedRecovery, expectedFinished, reason, actor, rawEvidence, now.UTC())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("write repair audit: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return false, errors.New("repair audit already exists")
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit repair: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。
|
||||
//
|
||||
// 与旧实现的区别(F02/F09 的根因):
|
||||
@@ -724,7 +791,7 @@ func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time, kinds [
|
||||
if len(kinds) == 0 {
|
||||
return 0, errors.New("reconcile library jobs requires the kinds this executor owns")
|
||||
}
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_jobs AS lj
|
||||
SET state='failed', error='队列中已无对应作业,业务作业未收到结果',
|
||||
recovery=CASE WHEN lj.kind IN ('sync','restore','recover','baseline')
|
||||
@@ -737,12 +804,15 @@ WHERE lj.state IN ('pending','running')
|
||||
AND lj.queue_job_id <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM jobs AS j
|
||||
WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry')
|
||||
)`, now.UTC(), kinds, s.libraryTargetID); err != nil {
|
||||
WHERE j.id = lj.queue_job_id
|
||||
AND j.state IN ('pending','running','retry','completed')
|
||||
)`, now.UTC(), kinds, s.libraryTargetID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reconcile library jobs: %w", err)
|
||||
}
|
||||
first := int(tag.RowsAffected())
|
||||
// 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
tag, err = s.db.Exec(ctx, `
|
||||
UPDATE library_jobs
|
||||
SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1
|
||||
WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND (kind='seal' OR target_id=$4) AND created_at < $2`,
|
||||
@@ -750,7 +820,7 @@ WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reconcile unattached library jobs: %w", err)
|
||||
}
|
||||
return int(tag.RowsAffected()), nil
|
||||
return first + int(tag.RowsAffected()), nil
|
||||
}
|
||||
|
||||
// LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// The worker startup reconciliation must not turn a completed capture queue
|
||||
// into the legacy "queue does not exist" failure. This is a real PostgreSQL
|
||||
// fixture with the actual varchar shapes; the RED case is the old SQL which
|
||||
// only accepted pending/running/retry.
|
||||
func TestStartupReconcileKeepsCompletedCaptureJob(t *testing.T) {
|
||||
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||||
t.Fatal("isolated database required")
|
||||
}
|
||||
cfg.MaxConns = 1
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_temp"
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
now := time.Now().UTC()
|
||||
if _, err := pool.Exec(ctx, `
|
||||
CREATE TEMP TABLE library_jobs(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
||||
state varchar(16) NOT NULL, stage varchar(32) NOT NULL DEFAULT '',
|
||||
queue_job_id varchar(64) NOT NULL DEFAULT '', target_id text NOT NULL DEFAULT '',
|
||||
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '',
|
||||
snapshot_id varchar(64) NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, finished_at timestamptz, created_at timestamptz NOT NULL DEFAULT now());
|
||||
CREATE TEMP TABLE jobs(id varchar(64) PRIMARY KEY, type varchar(64) NOT NULL, state varchar(16) NOT NULL);`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,queue_job_id,updated_at)
|
||||
VALUES('libseal-completed','internal','seal','pending','q-completed',$1),
|
||||
('libseal-failed','internal','seal','pending','q-failed',$1),
|
||||
('libseal-missing','internal','seal','pending','q-missing',$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,state) VALUES('q-completed','library.capture','completed'),
|
||||
('q-failed','library.capture','failed')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var state, errText string
|
||||
if err := pool.QueryRow(ctx, `SELECT state, error FROM library_jobs WHERE id='libseal-completed'`).Scan(&state, &errText); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != "pending" || errText != "" {
|
||||
t.Fatalf("completed queue was misclassified: state=%q error=%q", state, errText)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("reconciled=%d, want only failed/missing queue jobs (2)", n)
|
||||
}
|
||||
for _, id := range []string{"libseal-failed", "libseal-missing"} {
|
||||
var got string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_jobs WHERE id=$1`, id).Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "failed" {
|
||||
t.Fatalf("%s state=%q, want failed", id, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS library_job_repairs;
|
||||
@@ -0,0 +1,19 @@
|
||||
-- Immutable audit for the only permitted automatic/system-misclassification repair.
|
||||
CREATE TABLE library_job_repairs (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
job_id VARCHAR(64) NOT NULL,
|
||||
scope VARCHAR(16) NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL,
|
||||
queue_job_id VARCHAR(64) NOT NULL,
|
||||
snapshot_id VARCHAR(64) NOT NULL,
|
||||
original_state VARCHAR(16) NOT NULL,
|
||||
original_stage VARCHAR(32) NOT NULL DEFAULT '',
|
||||
original_error TEXT NOT NULL DEFAULT '',
|
||||
original_recovery VARCHAR(255) NOT NULL DEFAULT '',
|
||||
original_finished_at TIMESTAMPTZ,
|
||||
reason VARCHAR(255) NOT NULL,
|
||||
actor VARCHAR(128) NOT NULL DEFAULT '',
|
||||
evidence JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX library_job_repairs_identity ON library_job_repairs(job_id, snapshot_id);
|
||||
@@ -11,11 +11,14 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 79 {
|
||||
t.Fatalf("migration count = %d, want 79", len(migrations))
|
||||
if len(migrations) != 80 {
|
||||
t.Fatalf("migration count = %d, want 80", len(migrations))
|
||||
}
|
||||
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
|
||||
// 证据按 target_id 落库(换目标旧证据自动失效)。
|
||||
if repair := migrations[79]; repair.Version != 80 {
|
||||
t.Fatalf("migration 80 = %#v", repair)
|
||||
}
|
||||
if verification := migrations[78]; verification.Version != 79 ||
|
||||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
|
||||
!strings.Contains(verification.UpSQL, "'verify'") {
|
||||
|
||||
Reference in New Issue
Block a user