2eff9447e1
build / build (push) Failing after 1m20s
ReconcileLibraryJobs 不再把 completed capture queue 当成不存在,保留待核对 pending 封存作业;failed/cancelled/missing 仍按真实队列状态处置。管理员核对 入口只允许精确的启动误判形态,并要求 completed queue、export succeeded、 唯一 test_sealed 版本、来源身份与完整归档复核。新增 library_job_repairs 不可变审计表,在同一事务内 CAS 保存原 state/stage/error/recovery/finished_at 及本次 actor/原因/证据后修复;真实失败/取消不开放。 验证:真实隔离 PostgreSQL 启动收敛 RED/GREEN、启动→handler 纵向 pending 与 系统误判 failed、审计保留/幂等、负例;真实 Go 全量与 make verify 17 项通过。
205 lines
10 KiB
Go
205 lines
10 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
postgresstore "knowledge-governance/internal/platform/postgres"
|
|
)
|
|
|
|
// Vertical PostgreSQL path: the real worker startup reconciliation must leave
|
|
// a pending seal with a completed capture queue intact, then the actual HTTP
|
|
// handler must verify archive/version/stage and repair it. This is not a fake
|
|
// store or a direct helper test.
|
|
func verticalReconcileFixture(t *testing.T) (*postgresstore.Store, Config, string, *pgxpool.Pool) {
|
|
t.Helper()
|
|
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
|
if dsn == "" {
|
|
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
|
|
}
|
|
cfg, err := pgxpool.ParseConfig(dsn)
|
|
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
|
t.Fatal("isolated database required")
|
|
}
|
|
cfg.MaxConns = 1
|
|
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_temp"
|
|
ctx := context.Background()
|
|
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(pool.Close)
|
|
if _, err := pool.Exec(ctx, `
|
|
CREATE TEMP TABLE library_jobs(
|
|
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
|
state varchar(16) NOT NULL, stage varchar(32) NOT NULL DEFAULT '',
|
|
idempotency_key varchar(255) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL DEFAULT '',
|
|
queue_job_id varchar(64) NOT NULL DEFAULT '', snapshot_id varchar(64) NOT NULL DEFAULT '',
|
|
target_snapshot_id varchar(64) NOT NULL DEFAULT '', actor varchar(128) NOT NULL DEFAULT '',
|
|
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '',
|
|
operation_id varchar(64) NOT NULL DEFAULT '', attempt_epoch integer NOT NULL DEFAULT 0,
|
|
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, finished_at timestamptz,
|
|
target_id text NOT NULL DEFAULT '');
|
|
CREATE TEMP TABLE library_job_stages(
|
|
job_id varchar(64) NOT NULL, stage varchar(32) NOT NULL, state varchar(16) NOT NULL,
|
|
total bigint, completed bigint, bytes_total bigint, bytes_done bigint,
|
|
detail text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, PRIMARY KEY(job_id,stage));
|
|
CREATE TEMP TABLE library_versions(
|
|
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(24) NOT NULL,
|
|
snapshot_id varchar(64) NOT NULL DEFAULT '', version_label varchar(64) NOT NULL DEFAULT '',
|
|
source_job_id varchar(64) NOT NULL DEFAULT '', operation_id varchar(64) NOT NULL DEFAULT '',
|
|
attempt_epoch integer NOT NULL DEFAULT 0, evidence jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
created_by varchar(128) NOT NULL DEFAULT '', created_at timestamptz NOT NULL, superseded_at timestamptz);
|
|
CREATE TEMP TABLE library_job_repairs(
|
|
id varchar(64) PRIMARY KEY, job_id varchar(64) NOT NULL, scope varchar(16) NOT NULL,
|
|
kind varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL, snapshot_id varchar(64) NOT NULL,
|
|
original_state varchar(16) NOT NULL, original_stage varchar(32) NOT NULL DEFAULT '',
|
|
original_error text NOT NULL DEFAULT '', original_recovery varchar(255) NOT NULL DEFAULT '',
|
|
original_finished_at timestamptz, reason varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
|
|
evidence jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL);
|
|
CREATE TEMP TABLE library_operations(
|
|
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
|
target_snapshot_id varchar(64) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL,
|
|
operation_key varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
|
|
created_at timestamptz NOT NULL, target_id text NOT NULL DEFAULT '',
|
|
first_backup_snapshot_id varchar(64), first_backup_bytes bigint, first_backup_at timestamptz,
|
|
writeset_digest varchar(64), writeset_rows bigint, target_fingerprint varchar(128), anchors_at timestamptz);
|
|
CREATE UNIQUE INDEX library_operations_key_test ON library_operations(scope,kind,operation_key);
|
|
CREATE TEMP TABLE jobs(
|
|
id varchar(64) PRIMARY KEY, type varchar(64) NOT NULL, payload jsonb NOT NULL,
|
|
state varchar(16) NOT NULL, priority integer NOT NULL DEFAULT 0, available_at timestamptz NOT NULL,
|
|
lease_owner varchar(128), lease_until timestamptz, attempt integer NOT NULL DEFAULT 0,
|
|
max_attempts integer NOT NULL DEFAULT 1, last_error text, idempotency_key varchar(255) NOT NULL,
|
|
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL);`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
root := libraryFixtureRoot(t)
|
|
snapshotID := sealSharedVersions(t, root, 1, 4)[0]
|
|
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
manifest, err := archive.Load(snapshotID)
|
|
archive.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
config := Config{LibraryArchiveRoot: root,
|
|
LibrarySourceProject: "fixture-project", LibrarySourceApp: "fixture-app",
|
|
LibrarySourceTenantID: manifest.TenantID, LibrarySourceInternalKB: manifest.KnowledgeBaseID}
|
|
source := configuredLibrarySource(config, "internal")
|
|
sourceID, err := source.ID()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Now().UTC()
|
|
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical"})
|
|
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
|
|
VALUES('q-vertical','library.capture',$1,'completed',$2,'library.capture:vertical',$2,$2)`, payload, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
|
|
VALUES('ver-vertical','internal','test_sealed',$1,'libseal-vertical',$2,$3)`, snapshotID,
|
|
map[string]any{"source_id": sourceID, "capture_job": "vertical"}, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
|
|
VALUES('libseal-vertical','export','succeeded','已封存快照', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := postgresstore.NewStore(pool)
|
|
config.Store = store
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,created_at,updated_at)
|
|
VALUES('libseal-vertical','internal','seal','pending','k','d','q-vertical',$1,$1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return store, config, snapshotID, pool
|
|
}
|
|
|
|
func TestStartupReconcileThenHandlerRepairsPendingCapture(t *testing.T) {
|
|
store, config, _, _ := verticalReconcileFixture(t)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC()
|
|
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
|
|
t.Fatalf("startup reconcile n=%d err=%v", n, err)
|
|
}
|
|
job, err := store.LibraryJob(ctx, "libseal-vertical")
|
|
if err != nil || job.State != governance.LibraryJobPending {
|
|
t.Fatalf("startup changed job: %+v err=%v", job, err)
|
|
}
|
|
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical/reconcile-capture", nil)
|
|
req.SetPathValue("id", "libseal-vertical")
|
|
rec := httptest.NewRecorder()
|
|
libraryCaptureReconcile(rec, req, config)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
job, err = store.LibraryJob(ctx, "libseal-vertical")
|
|
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID == "" {
|
|
t.Fatalf("handler did not repair: %+v err=%v", job, err)
|
|
}
|
|
}
|
|
|
|
func TestStartupReconcilePreservesFailedCapturesForExactReconcile(t *testing.T) {
|
|
store, config, snapshotID, pool := verticalReconcileFixture(t)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC()
|
|
source := configuredLibrarySource(config, "internal")
|
|
sourceID, err := source.ID()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical-failed"})
|
|
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
|
|
VALUES('q-failed','library.capture',$1,'completed',$2,'library.capture:vertical-failed',$2,$2)`, payload, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
|
|
VALUES('ver-failed','internal','test_sealed',$1,'libseal-vertical-failed',$2,$3)`,
|
|
snapshotID, map[string]any{"source_id": sourceID, "capture_job": "vertical-failed"}, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
|
|
VALUES('libseal-vertical-failed','export','succeeded','已封存快照', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finished := now.Truncate(time.Microsecond)
|
|
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,error,recovery,created_at,updated_at,finished_at)
|
|
VALUES('libseal-vertical-failed','internal','seal','failed','kf','df','q-failed',$1,$2,$3,$3,$3)`,
|
|
"队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", finished); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
|
|
t.Fatalf("startup reconcile n=%d err=%v", n, err)
|
|
}
|
|
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical-failed/reconcile-capture", nil)
|
|
req.SetPathValue("id", "libseal-vertical-failed")
|
|
rec := httptest.NewRecorder()
|
|
libraryCaptureReconcile(rec, req, config)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
job, err := store.LibraryJob(ctx, "libseal-vertical-failed")
|
|
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID != snapshotID {
|
|
t.Fatalf("failed misclassification not repaired: %+v err=%v", job, err)
|
|
}
|
|
var originalState, originalError string
|
|
if err := pool.QueryRow(ctx, `SELECT original_state, original_error FROM library_job_repairs WHERE job_id='libseal-vertical-failed'`).
|
|
Scan(&originalState, &originalError); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if originalState != "failed" || originalError == "" {
|
|
t.Fatalf("audit missing original: state=%q error=%q", originalState, originalError)
|
|
}
|
|
}
|