fix(library): 修正启动收敛误判并提供精确核对
build / build (push) Failing after 1m20s

ReconcileLibraryJobs 不再把 completed capture queue 当成不存在,保留待核对
pending 封存作业;failed/cancelled/missing 仍按真实队列状态处置。管理员核对
入口只允许精确的启动误判形态,并要求 completed queue、export succeeded、
唯一 test_sealed 版本、来源身份与完整归档复核。新增 library_job_repairs
不可变审计表,在同一事务内 CAS 保存原 state/stage/error/recovery/finished_at
及本次 actor/原因/证据后修复;真实失败/取消不开放。

验证:真实隔离 PostgreSQL 启动收敛 RED/GREEN、启动→handler 纵向 pending 与
系统误判 failed、审计保留/幂等、负例;真实 Go 全量与 make verify 17 项通过。
This commit is contained in:
姚章浩
2026-11-06 10:51:21 +08:00
parent 146ac832a2
commit 2eff9447e1
12 changed files with 610 additions and 12 deletions
+35 -2
View File
@@ -24,8 +24,14 @@ type libraryCaptureReconciler interface {
LibraryTestSealedVersionForJob(ctx context.Context, scope, jobID string) (governance.LibraryVersion, bool, error)
EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error)
ReconcileLibraryCaptureTerminal(ctx context.Context, jobID, scope, queueJobID, snapshotID string, now time.Time) (bool, error)
ReconcileMisclassifiedLibraryCapture(ctx context.Context, jobID, scope, queueJobID, snapshotID, expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time, auditID, actor, reason string, evidence map[string]any, now time.Time) (bool, error)
}
const (
legacyOrphanError = "队列中已无对应作业,业务作业未收到结果"
legacyOrphanRecovery = "封存作业由 agent 恢复来源;确认来源 healthy 后可重发"
)
type libraryCaptureQueueReader interface {
QueueJob(ctx context.Context, id string) (jobs.Job, bool, error)
}
@@ -61,8 +67,11 @@ func libraryCaptureReconcile(w http.ResponseWriter, r *http.Request, config Conf
conflict(w, "只有封存作业可以核对封存结果")
return
}
if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning && job.State != governance.LibraryJobSucceeded {
conflict(w, "作业已是失败/取消终态,不能改写成成功")
systemMisclassified := job.State == governance.LibraryJobFailed && job.Stage == "" &&
job.Error == legacyOrphanError && job.Recovery == legacyOrphanRecovery && job.SnapshotID == ""
if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning &&
job.State != governance.LibraryJobSucceeded && !systemMisclassified {
conflict(w, "作业是真实失败/取消终态,不能改写成成功")
return
}
if job.QueueJobID == "" {
@@ -176,6 +185,30 @@ func libraryCaptureReconcile(w http.ResponseWriter, r *http.Request, config Conf
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build reconcile audit digest"})
return
}
if systemMisclassified {
auditID := "librepair-" + digest[:16]
updated, repairErr := store.ReconcileMisclassifiedLibraryCapture(r.Context(), job.ID, job.Scope,
job.QueueJobID, version.SnapshotID, job.Stage, legacyOrphanError, legacyOrphanRecovery,
job.FinishedAt, auditID, actor, "completed capture queue + export succeeded + exact version + archive verified; legacy orphan misclassification", map[string]any{
"original_state": job.State, "original_error": job.Error, "original_recovery": job.Recovery,
"original_finished_at": job.FinishedAt, "queue_job_id": queue.ID,
"snapshot_id": version.SnapshotID, "version_id": version.ID, "archive_verified": true,
}, now)
if repairErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot persist misclassification repair"})
return
}
if !updated {
conflict(w, "原误判行状态/错误/完成时间已变化,拒绝覆盖")
return
}
writeJSON(w, http.StatusOK, map[string]any{
"repaired": true, "already_succeeded": false, "misclassification_repaired": true,
"job_id": job.ID, "snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
"stage": "export", "version_id": version.ID, "audit_repair_id": auditID,
})
return
}
opID := "libop-capture-reconcile-" + digest[:16]
op, _, err := store.EnsureLibraryOperation(r.Context(), governance.LibraryOperation{
ID: opID, Scope: job.Scope, Kind: governance.LibraryJobRecover,
@@ -99,6 +99,30 @@ func (f *fakeLibraryStore) ReconcileLibraryCaptureTerminal(_ context.Context, jo
}
return false, nil
}
func (f *fakeLibraryStore) ReconcileMisclassifiedLibraryCapture(_ context.Context, jobID, scope, queueJobID, snapshotID, expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time, _ string, _ string, _ string, _ map[string]any, _ time.Time) (bool, error) {
if f.reconcileDenied {
return false, nil
}
for index := range f.jobs {
j := &f.jobs[index]
if j.ID != jobID || j.Scope != scope || j.Kind != governance.LibraryJobSeal || j.QueueJobID != queueJobID ||
j.State != governance.LibraryJobFailed || j.Stage != expectedStage || j.Error != expectedError ||
j.Recovery != expectedRecovery || j.SnapshotID != "" || (j.FinishedAt == nil) != (expectedFinished == nil) ||
(j.FinishedAt != nil && expectedFinished != nil && !j.FinishedAt.Equal(*expectedFinished)) {
return false, nil
}
j.State = governance.LibraryJobSucceeded
j.SnapshotID = snapshotID
j.Stage = "export"
f.lastState = governance.LibraryJobSucceeded
return true, nil
}
return false, nil
}
var _ libraryCaptureReconciler = (*fakeLibraryStore)(nil)
func runReconcile(t *testing.T, config Config, jobID string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/"+jobID+"/reconcile-capture", nil)
@@ -271,3 +295,67 @@ func TestLibraryCaptureReconcileConditionalWriteRejectsConcurrentState(t *testin
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
}
}
func TestLibraryCaptureReconcileRepairsExactLegacyMisclassification(t *testing.T) {
store, config, job, _ := reconcileFixture(t)
finished := time.Now().UTC()
job.State = governance.LibraryJobFailed
job.Stage = ""
job.Error = legacyOrphanError
job.Recovery = legacyOrphanRecovery
job.FinishedAt = &finished
store.jobs[0] = job
rec := runReconcile(t, config, job.ID)
if rec.Code != http.StatusOK {
t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &body)
if body["misclassification_repaired"] != true || store.lastState != governance.LibraryJobSucceeded {
t.Fatalf("body=%v advanced=%q", body, store.lastState)
}
}
func TestLibraryCaptureReconcileRejectsOtherFailedError(t *testing.T) {
store, config, job, _ := reconcileFixture(t)
finished := time.Now().UTC()
job.State = governance.LibraryJobFailed
job.Stage = ""
job.Error = "capture export failed"
job.Recovery = legacyOrphanRecovery
job.FinishedAt = &finished
store.jobs[0] = job
rec := runReconcile(t, config, job.ID)
if rec.Code != http.StatusConflict || store.lastState != "" {
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
}
}
func TestLibraryCaptureReconcileRejectsRealFailureAndCancel(t *testing.T) {
for _, state := range []governance.LibraryJobState{governance.LibraryJobFailed, governance.LibraryJobCancelled} {
store, config, job, _ := reconcileFixture(t)
job.State = state
job.Error = "real failure"
store.jobs[0] = job
rec := runReconcile(t, config, job.ID)
if rec.Code != http.StatusConflict || store.lastState != "" {
t.Fatalf("state=%s code=%d advanced=%q", state, rec.Code, store.lastState)
}
}
}
func TestLibraryCaptureReconcileMisclassificationCASRejectsChange(t *testing.T) {
store, config, job, _ := reconcileFixture(t)
finished := time.Now().UTC()
job.State = governance.LibraryJobFailed
job.Stage = ""
job.Error = legacyOrphanError
job.Recovery = legacyOrphanRecovery
job.FinishedAt = &finished
store.jobs[0] = job
store.reconcileDenied = true
rec := runReconcile(t, config, job.ID)
if rec.Code != http.StatusConflict || store.lastState != "" {
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
}
}
@@ -0,0 +1,204 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// Vertical PostgreSQL path: the real worker startup reconciliation must leave
// a pending seal with a completed capture queue intact, then the actual HTTP
// handler must verify archive/version/stage and repair it. This is not a fake
// store or a direct helper test.
func verticalReconcileFixture(t *testing.T) (*postgresstore.Store, Config, string, *pgxpool.Pool) {
t.Helper()
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Fatal("isolated database required")
}
cfg.MaxConns = 1
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_temp"
ctx := context.Background()
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pool.Close)
if _, err := pool.Exec(ctx, `
CREATE TEMP TABLE library_jobs(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
state varchar(16) NOT NULL, stage varchar(32) NOT NULL DEFAULT '',
idempotency_key varchar(255) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL DEFAULT '',
queue_job_id varchar(64) NOT NULL DEFAULT '', snapshot_id varchar(64) NOT NULL DEFAULT '',
target_snapshot_id varchar(64) NOT NULL DEFAULT '', actor varchar(128) NOT NULL DEFAULT '',
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '',
operation_id varchar(64) NOT NULL DEFAULT '', attempt_epoch integer NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, finished_at timestamptz,
target_id text NOT NULL DEFAULT '');
CREATE TEMP TABLE library_job_stages(
job_id varchar(64) NOT NULL, stage varchar(32) NOT NULL, state varchar(16) NOT NULL,
total bigint, completed bigint, bytes_total bigint, bytes_done bigint,
detail text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, PRIMARY KEY(job_id,stage));
CREATE TEMP TABLE library_versions(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(24) NOT NULL,
snapshot_id varchar(64) NOT NULL DEFAULT '', version_label varchar(64) NOT NULL DEFAULT '',
source_job_id varchar(64) NOT NULL DEFAULT '', operation_id varchar(64) NOT NULL DEFAULT '',
attempt_epoch integer NOT NULL DEFAULT 0, evidence jsonb NOT NULL DEFAULT '{}'::jsonb,
created_by varchar(128) NOT NULL DEFAULT '', created_at timestamptz NOT NULL, superseded_at timestamptz);
CREATE TEMP TABLE library_job_repairs(
id varchar(64) PRIMARY KEY, job_id varchar(64) NOT NULL, scope varchar(16) NOT NULL,
kind varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL, snapshot_id varchar(64) NOT NULL,
original_state varchar(16) NOT NULL, original_stage varchar(32) NOT NULL DEFAULT '',
original_error text NOT NULL DEFAULT '', original_recovery varchar(255) NOT NULL DEFAULT '',
original_finished_at timestamptz, reason varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
evidence jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL);
CREATE TEMP TABLE library_operations(
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
target_snapshot_id varchar(64) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL,
operation_key varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
created_at timestamptz NOT NULL, target_id text NOT NULL DEFAULT '',
first_backup_snapshot_id varchar(64), first_backup_bytes bigint, first_backup_at timestamptz,
writeset_digest varchar(64), writeset_rows bigint, target_fingerprint varchar(128), anchors_at timestamptz);
CREATE UNIQUE INDEX library_operations_key_test ON library_operations(scope,kind,operation_key);
CREATE TEMP TABLE jobs(
id varchar(64) PRIMARY KEY, type varchar(64) NOT NULL, payload jsonb NOT NULL,
state varchar(16) NOT NULL, priority integer NOT NULL DEFAULT 0, available_at timestamptz NOT NULL,
lease_owner varchar(128), lease_until timestamptz, attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL DEFAULT 1, last_error text, idempotency_key varchar(255) NOT NULL,
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL);`); err != nil {
t.Fatal(err)
}
root := libraryFixtureRoot(t)
snapshotID := sealSharedVersions(t, root, 1, 4)[0]
archive, err := librarysnapshot.Open(root, 256*1024*1024)
if err != nil {
t.Fatal(err)
}
manifest, err := archive.Load(snapshotID)
archive.Close()
if err != nil {
t.Fatal(err)
}
config := Config{LibraryArchiveRoot: root,
LibrarySourceProject: "fixture-project", LibrarySourceApp: "fixture-app",
LibrarySourceTenantID: manifest.TenantID, LibrarySourceInternalKB: manifest.KnowledgeBaseID}
source := configuredLibrarySource(config, "internal")
sourceID, err := source.ID()
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical"})
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
VALUES('q-vertical','library.capture',$1,'completed',$2,'library.capture:vertical',$2,$2)`, payload, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
VALUES('ver-vertical','internal','test_sealed',$1,'libseal-vertical',$2,$3)`, snapshotID,
map[string]any{"source_id": sourceID, "capture_job": "vertical"}, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
VALUES('libseal-vertical','export','succeeded','已封存快照', $1)`, now); err != nil {
t.Fatal(err)
}
store := postgresstore.NewStore(pool)
config.Store = store
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,created_at,updated_at)
VALUES('libseal-vertical','internal','seal','pending','k','d','q-vertical',$1,$1)`, now); err != nil {
t.Fatal(err)
}
return store, config, snapshotID, pool
}
func TestStartupReconcileThenHandlerRepairsPendingCapture(t *testing.T) {
store, config, _, _ := verticalReconcileFixture(t)
ctx := context.Background()
now := time.Now().UTC()
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
t.Fatalf("startup reconcile n=%d err=%v", n, err)
}
job, err := store.LibraryJob(ctx, "libseal-vertical")
if err != nil || job.State != governance.LibraryJobPending {
t.Fatalf("startup changed job: %+v err=%v", job, err)
}
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical/reconcile-capture", nil)
req.SetPathValue("id", "libseal-vertical")
rec := httptest.NewRecorder()
libraryCaptureReconcile(rec, req, config)
if rec.Code != http.StatusOK {
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
}
job, err = store.LibraryJob(ctx, "libseal-vertical")
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID == "" {
t.Fatalf("handler did not repair: %+v err=%v", job, err)
}
}
func TestStartupReconcilePreservesFailedCapturesForExactReconcile(t *testing.T) {
store, config, snapshotID, pool := verticalReconcileFixture(t)
ctx := context.Background()
now := time.Now().UTC()
source := configuredLibrarySource(config, "internal")
sourceID, err := source.ID()
if err != nil {
t.Fatal(err)
}
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical-failed"})
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
VALUES('q-failed','library.capture',$1,'completed',$2,'library.capture:vertical-failed',$2,$2)`, payload, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
VALUES('ver-failed','internal','test_sealed',$1,'libseal-vertical-failed',$2,$3)`,
snapshotID, map[string]any{"source_id": sourceID, "capture_job": "vertical-failed"}, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
VALUES('libseal-vertical-failed','export','succeeded','已封存快照', $1)`, now); err != nil {
t.Fatal(err)
}
finished := now.Truncate(time.Microsecond)
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,error,recovery,created_at,updated_at,finished_at)
VALUES('libseal-vertical-failed','internal','seal','failed','kf','df','q-failed',$1,$2,$3,$3,$3)`,
"队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", finished); err != nil {
t.Fatal(err)
}
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
t.Fatalf("startup reconcile n=%d err=%v", n, err)
}
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical-failed/reconcile-capture", nil)
req.SetPathValue("id", "libseal-vertical-failed")
rec := httptest.NewRecorder()
libraryCaptureReconcile(rec, req, config)
if rec.Code != http.StatusOK {
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
}
job, err := store.LibraryJob(ctx, "libseal-vertical-failed")
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID != snapshotID {
t.Fatalf("failed misclassification not repaired: %+v err=%v", job, err)
}
var originalState, originalError string
if err := pool.QueryRow(ctx, `SELECT original_state, original_error FROM library_job_repairs WHERE job_id='libseal-vertical-failed'`).
Scan(&originalState, &originalError); err != nil {
t.Fatal(err)
}
if originalState != "failed" || originalError == "" {
t.Fatalf("audit missing original: state=%q error=%q", originalState, originalError)
}
}