Files
knowledge-governance/internal/platform/postgres/library_identity_integration_test.go
T
姚章浩 a7d643635a
build / build (push) Successful in 1m27s
fix(library): 守卫按行取作业身份、同作业多代次、原作业续作与执行域准入
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2,
导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的
owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。

R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引
(job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与
operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、
新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律
ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。

R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式
结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。

R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release
路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。

同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点
改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、
恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。

测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例;
go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
2026-10-30 17:20:40 +08:00

526 lines
24 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package postgres
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/librarymaintenance"
)
// 真实 PG 上的执行身份/栅栏用例。
//
// 为什么必须是真库(父的复核项 R1/R2):守卫的失败形态是"SQL 条件永远不匹配"——
// 参数序号、列引用、NULL 与空串的差别在内存 fake 里根本不存在。字符串断言证明不了
// 一条 UPDATE 真的命中了一行。
func newIdentityStore(t *testing.T) (*Store, *pgxpool.Pool) {
t.Helper()
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect isolated database: %v", err)
}
t.Cleanup(pool.Close)
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("apply migrations: %v", err)
}
return NewStore(pool), pool
}
// seedOperationJob 建一条"有 operation 的业务作业",返回 operation 与作业 id。
func seedOperationJob(t *testing.T, store *Store, pool *pgxpool.Pool, scope, jobID, kind string, now time.Time) string {
t.Helper()
ctx := context.Background()
op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: governance.LibraryOperationID(scope, governance.LibraryJobKind(kind), jobID),
Scope: scope, Kind: governance.LibraryJobKind(kind), TargetSnapshotID: "snap-target",
RequestDigest: "digest-" + jobID, OperationKey: jobID, Actor: "tester", CreatedAt: now.UTC(),
})
if err != nil {
t.Fatalf("ensure operation: %v", err)
}
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
ID: jobID, Scope: scope, Kind: governance.LibraryJobKind(kind), OperationID: op.ID,
IdempotencyKey: "library.job:" + op.ID, RequestDigest: "digest-" + jobID,
TargetSnapshotID: "snap-target", Actor: "tester",
}, governance.SyncStages(), now); err != nil {
t.Fatalf("create library job: %v", err)
}
// 维护窗口由这个作业持有(实例级 + scope 级)。
if _, err := pool.Exec(ctx, `
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state,
note, acquired_at, updated_at)
VALUES ($1,$2,'target-identity','acquired','app:running,store:running','test',$3,$3)
ON CONFLICT (scope) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, released_at=NULL,
state='acquired', updated_at=EXCLUDED.updated_at`,
scope, jobID, now.UTC()); err != nil {
t.Fatalf("seed scope maintenance: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, state,
original_service_state, note, acquired_at, updated_at)
VALUES ('inst-1',$1,$2,'fingerprint-1','acquired','app:running','test',$3,$3)
ON CONFLICT (instance) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, owner_scope=EXCLUDED.owner_scope,
released_at=NULL, state='acquired', updated_at=EXCLUDED.updated_at`,
jobID, scope, now.UTC()); err != nil {
t.Fatalf("seed instance maintenance: %v", err)
}
return op.ID
}
// settleJob 把一个作业行推到终态:同一 scope 只允许一条在飞作业(部分唯一索引),
// 所以"再建一条兄弟作业"之前必须先让它落地。
func settleJob(t *testing.T, pool *pgxpool.Pool, jobID string, now time.Time) {
t.Helper()
if _, err := pool.Exec(context.Background(),
`UPDATE library_jobs SET state='failed', finished_at=$2, updated_at=$2 WHERE id=$1`,
jobID, now.UTC()); err != nil {
t.Fatalf("settle job %s: %v", jobID, err)
}
}
// R1:维护类守卫的作业 id 必须取自被更新行自己的 owner_job_id,而不是参数 $1。
//
// 旧实现把 `j.id=$1` 写死,$1 是 scope/instance ⇒ 守卫永远匹配不到作业,
// **正常**的维护状态写入与释放全部失败。这里逐个入口走一遍真库。
func TestLibraryMaintenanceFenceUsesRowOwner(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
jobID := "libsync-fence-1"
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
if err != nil {
t.Fatalf("begin attempt: %v", err)
}
if attempt.OperationID == "" || attempt.Epoch != 1 {
t.Fatalf("attempt = %+v", attempt)
}
// scope 级:状态、入口、释放都必须命中($1 是 scope,不是作业 id)。
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", jobID, "applying", "写目标", attempt, now); err != nil {
t.Fatalf("更新 scope 维护状态失败(守卫没命中作业):%v", err)
}
if err := store.SetLibraryMaintenanceIngressGuarded(ctx, "internal", jobID, "closed", "入口已关", attempt, now); err != nil {
t.Fatalf("记录 scope 入口状态失败:%v", err)
}
// 实例级:$1 是 instance。
if err := store.UpdateInstanceMaintenanceStateGuarded(ctx, "inst-1", jobID, "applying", "写目标", attempt, now); err != nil {
t.Fatalf("更新实例维护状态失败:%v", err)
}
if err := store.SetInstanceMaintenanceIngressGuarded(ctx, "inst-1", jobID, "closed", "入口已关", attempt, now); err != nil {
t.Fatalf("记录实例入口状态失败:%v", err)
}
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); err != nil {
t.Fatalf("释放 scope 维护失败(正常路径被守卫挡死):%v", err)
}
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", jobID, attempt, now); err != nil {
t.Fatalf("释放实例维护失败:%v", err)
}
var released int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_maintenance WHERE released_at IS NOT NULL`).Scan(&released); err != nil {
t.Fatal(err)
}
if released != 1 {
t.Fatalf("scope 维护应有 1 行已释放,实际 %d", released)
}
// 负例(同一代码路径):作业 id 不匹配时守卫必须**拒绝**,而不是默默返回成功。
// 这条与上面的正例一起,才证明 "$1 是 scope、作业 id 取自行" 的契约真的在起作用。
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", "libsync-other-job", attempt, now); err == nil {
t.Fatal("作业 id 不匹配时释放实例维护必须失败(守卫没看作业身份)")
}
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", "libsync-other-job", "applying", "x", attempt, now); err == nil {
t.Fatal("作业 id 不匹配时更新维护状态必须失败")
}
// 幂等:再释放一次是"已释放",不是"未持有"。
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); !errors.Is(err, ErrMaintenanceReleased) {
t.Fatalf("重复释放应返回 ErrMaintenanceReleased,实际 %v", err)
}
}
// R2:同一作业连续三次认领 ⇒ 三个不可变代次;旧代次的写必须失败。
func TestBeginLibraryAttemptSameJobThreeGenerations(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
jobID := "libsync-gen-1"
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
var epochs []int
for round := 1; round <= 3; round++ {
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker#q#"+string(rune('0'+round)), 10*time.Minute, now.Add(time.Duration(round)*time.Minute))
if err != nil {
t.Fatalf("第 %d 次认领失败:%v", round, err)
}
epochs = append(epochs, attempt.Epoch)
// 结束上一代(模拟执行完毕/崩溃后接手):直接 Finish 或让它过期。
if err := store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptCompleted, "round done", now); err != nil {
t.Fatalf("第 %d 次结束代次失败:%v", round, err)
}
}
if len(epochs) != 3 || epochs[0] != 1 || epochs[1] != 2 || epochs[2] != 3 {
t.Fatalf("epoch 必须按 operation 单调递增:%v", epochs)
}
// 历史不可变:三次认领留下三行。
var rows int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
t.Fatal(err)
}
if rows != 3 {
t.Fatalf("同一作业三次认领必须保留 3 条代次(不可变历史),实际 %d", rows)
}
// 作业行指向最后一代。
job, err := store.LibraryJob(ctx, jobID)
if err != nil {
t.Fatal(err)
}
if job.AttemptEpoch != 3 {
t.Fatalf("作业行 attempt_epoch = %d, want 3", job.AttemptEpoch)
}
// 旧代次的守卫写必须失败(它的 attempt 已 completed)。
stale := governance.LibraryAttempt{OperationID: opID, Epoch: 2, JobID: jobID, Owner: "worker#q#2"}
if err := store.AdvanceLibraryJobGuarded(ctx, stale, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
t.Fatalf("旧代次写业务状态必须被拒绝,实际 %v", err)
}
// 迟到的 finish 也必须拒绝:否则旧执行能把别人正在跑的那一代"结束"掉。
if err := store.FinishLibraryAttempt(ctx, stale, governance.LibraryAttemptCompleted, "late finish", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
t.Fatalf("旧代次的 finish 必须被拒绝,实际 %v", err)
}
}
// R2:同一次投递重复调用是幂等的(同一代次,不新增行);别人的有效租约必须让路。
func TestBeginLibraryAttemptIdempotentRepeatAndHeldLease(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
jobID := "libsync-repeat-1"
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
if err != nil {
t.Fatal(err)
}
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now.Add(time.Second))
if err != nil {
t.Fatalf("同一次投递重复调用必须幂等:%v", err)
}
if second.Epoch != first.Epoch {
t.Fatalf("重复调用不应新增代次:%d vs %d", second.Epoch, first.Epoch)
}
var rows int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
t.Fatal(err)
}
if rows != 1 {
t.Fatalf("幂等重复调用不应新增代次行,实际 %d", rows)
}
// 另一个执行者(不同 owner,租约未过期)必须被拒绝。
if _, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#0", 10*time.Minute, now.Add(2*time.Second)); !errors.Is(err, ErrLibraryAttemptHeld) {
t.Fatalf("别的执行者持有未过期租约时必须拒绝,实际 %v", err)
}
// 租约过期后允许接管,并结束旧代次。
later := now.Add(20 * time.Minute)
takeover, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#1", 10*time.Minute, later)
if err != nil {
t.Fatalf("租约过期后必须允许接管:%v", err)
}
if takeover.Epoch != first.Epoch+1 {
t.Fatalf("接管必须开新代次:%d", takeover.Epoch)
}
var oldState string
if err := pool.QueryRow(ctx, `SELECT state FROM library_attempts WHERE operation_id=$1 AND epoch=$2`,
opID, first.Epoch).Scan(&oldState); err != nil {
t.Fatal(err)
}
if oldState != "abandoned" {
t.Fatalf("被接管的旧代次应标为 abandoned,实际 %q", oldState)
}
// 旧代次的续租必须失败(失租即停)。
if err := store.RenewLibraryAttempt(ctx, first, time.Minute, later); !errors.Is(err, governance.ErrLibraryEpochStale) {
t.Fatalf("旧代次续租必须失败,实际 %v", err)
}
}
// R2:同一 operation 不同作业(续作留下的兄弟行)——新代次激活后,旧作业行的守卫必须失效。
func TestSiblingJobGuardInvalidatedByNewAttempt(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
firstJob := "libsync-sib-1"
opID := seedOperationJob(t, store, pool, "internal", firstJob, "sync", now)
firstAttempt, err := store.BeginLibraryAttempt(ctx, firstJob, "worker-a#q1#0", 10*time.Minute, now)
if err != nil {
t.Fatal(err)
}
settleJob(t, pool, firstJob, now)
// 兄弟作业行(同一 operation,模拟历史 `-resume` 行/并行执行者)。
secondJob := firstJob + "-resume"
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
ID: secondJob, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: opID,
IdempotencyKey: "library.job:" + opID + ":resume", RequestDigest: "digest-sib",
TargetSnapshotID: "snap-target", Actor: "tester",
}, governance.SyncStages(), now); err != nil {
t.Fatalf("create sibling job: %v", err)
}
// 结束第一代后,由兄弟作业接管同一 operation。
if err := store.FinishLibraryAttempt(ctx, firstAttempt, governance.LibraryAttemptFailed, "crashed", now); err != nil {
t.Fatal(err)
}
settleJob(t, pool, firstJob, now)
secondAttempt, err := store.BeginLibraryAttempt(ctx, secondJob, "worker-b#q2#0", 10*time.Minute, now.Add(time.Minute))
if err != nil {
t.Fatalf("兄弟作业接管失败:%v", err)
}
if secondAttempt.OperationID != opID || secondAttempt.Epoch != firstAttempt.Epoch+1 {
t.Fatalf("兄弟作业必须共用同一 operation 并开新一代:%+v", secondAttempt)
}
// 旧作业行上的 epoch/owner 字段还在,但它的守卫必须失败。
if err := store.AdvanceLibraryJobGuarded(ctx, firstAttempt, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
t.Fatalf("旧作业的守卫必须失效,实际 %v", err)
}
var siblingLease string
if err := pool.QueryRow(ctx, `SELECT lease_owner FROM library_jobs WHERE id=$1`, firstJob).Scan(&siblingLease); err != nil {
t.Fatal(err)
}
if siblingLease != "" {
t.Fatalf("兄弟接管后旧作业行租约必须被清空,实际 %q", siblingLease)
}
}
// 首次锚点:事务 + 回读核对,NULL 与空串不清混;覆盖被拒绝。
func TestRecordOperationAnchorsTransactionalAndNullSafe(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
jobID := "libsync-anchor-1"
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
// 只登记快照 id(字节/写集合未知)⇒ 对应列必须是 NULL,而不是 0/空串。
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
FirstBackupSnapshotID: "backup-snap-1",
}, now); err != nil {
t.Fatalf("首次登记锚点失败:%v", err)
}
var bytesNull, rowsNull, digestNull bool
if err := pool.QueryRow(ctx, `
SELECT first_backup_bytes IS NULL, writeset_rows IS NULL, writeset_digest IS NULL
FROM library_operations WHERE id=$1`, opID).Scan(&bytesNull, &rowsNull, &digestNull); err != nil {
t.Fatal(err)
}
if !bytesNull || !rowsNull || !digestNull {
t.Fatalf("未知锚点必须留 NULL:bytes_null=%v rows_null=%v digest_null=%v", bytesNull, rowsNull, digestNull)
}
// 补齐字节与写集合:允许填空。
bytes := int64(4096)
rows := int64(119)
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
FirstBackupBytes: &bytes, WriteSetRows: &rows, WriteSetDigest: "ws-digest-1",
}, now); err != nil {
t.Fatalf("补齐锚点失败:%v", err)
}
var gotBytes int64
var gotDigest string
if err := pool.QueryRow(ctx, `SELECT first_backup_bytes, writeset_digest FROM library_operations WHERE id=$1`,
opID).Scan(&gotBytes, &gotDigest); err != nil {
t.Fatal(err)
}
if gotBytes != bytes || gotDigest != "ws-digest-1" {
t.Fatalf("锚点回读不一致:%d/%s", gotBytes, gotDigest)
}
// 覆盖被拒绝(字节与写集合都不许换)。
other := int64(1)
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{FirstBackupBytes: &other}, now); err == nil {
t.Fatal("用不同字节覆盖首次锚点必须失败")
}
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{WriteSetDigest: "ws-digest-2"}, now); err == nil {
t.Fatal("用不同写集合摘要覆盖首次锚点必须失败")
}
// 幂等:同一份锚点重复登记成功且不改值。
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
FirstBackupSnapshotID: "backup-snap-1", FirstBackupBytes: &bytes, WriteSetDigest: "ws-digest-1",
}, now); err != nil {
t.Fatalf("同一份锚点重复登记必须幂等成功:%v", err)
}
}
// current 指针:旧意图后来登记的事实**不能**把 current 推回历史;新意图(恢复)可以。
func TestLibraryCurrentOrderingRejectsLateOldIntent(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
base := time.Now().UTC()
jobA, jobB := "libsync-ord-a", "libsync-ord-b"
opA := seedOperationJob(t, store, pool, "internal", jobA, "sync", base)
settleJob(t, pool, jobA, base)
opB := seedOperationJob(t, store, pool, "internal", jobB, "sync", base.Add(time.Minute))
effective := func(opID, jobID, snapshot string, at time.Time) error {
return store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", snapshot, opID),
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot,
SourceJobID: jobID, OperationID: opID, CreatedBy: "tester", CreatedAt: at,
Evidence: map[string]any{"backup_snapshot": "backup-" + snapshot},
})
}
if err := effective(opA, jobA, "snap-a", base); err != nil {
t.Fatalf("登记 A 失败:%v", err)
}
if err := effective(opB, jobB, "snap-b", base.Add(time.Minute)); err != nil {
t.Fatalf("登记 B 失败:%v", err)
}
current, found, err := store.LibraryCurrentVersion(ctx, "internal")
if err != nil || !found || current.SnapshotID != "snap-b" {
t.Fatalf("current 应为 snap-b:%v %+v", err, current)
}
// 旧意图(A,创建时间更早)后来又登记了一条新事实:必须被拒绝,不能把 current 拉回去。
late := governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", "snap-a2", opA),
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-a2",
SourceJobID: jobA, OperationID: opA, CreatedBy: "tester", CreatedAt: base.Add(2 * time.Hour),
Evidence: map[string]any{"backup_snapshot": "backup-snap-a2"},
}
if err := store.RecordLibraryVersion(ctx, late); !errors.Is(err, governance.ErrLibraryCurrentStale) {
t.Fatalf("旧意图的迟到事实必须被拒绝,实际 %v", err)
}
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
if err != nil || !found || current.SnapshotID != "snap-b" {
t.Fatalf("current 不应被旧意图推回:%v %+v", err, current)
}
// 恢复是**新意图**:内容更旧(backup),意图更新 ⇒ 允许推进。
recoveryOp, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: governance.LibraryOperationID("internal", governance.LibraryJobRecover, "window-1"),
Scope: "internal", Kind: governance.LibraryJobRecover, TargetSnapshotID: "backup-snap-a",
RequestDigest: "recover-digest", OperationKey: "window-1", Actor: "tester",
CreatedAt: base.Add(3 * time.Hour),
})
if err != nil {
t.Fatalf("登记恢复意图失败:%v", err)
}
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionRecoveryVerified, "internal", "backup-snap-a", recoveryOp.ID),
Scope: "internal", Kind: governance.LibraryVersionRecoveryVerified, SnapshotID: "backup-snap-a",
SourceJobID: jobA, OperationID: recoveryOp.ID, CreatedBy: "worker", CreatedAt: base.Add(3 * time.Hour),
Evidence: map[string]any{"backup_snapshot": "backup-snap-a", "verify_digest": "digest"},
}); err != nil {
t.Fatalf("恢复事实必须能推进 current:%v", err)
}
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
if err != nil || !found || current.SnapshotID != "backup-snap-a" || current.Kind != governance.LibraryVersionRecoveryVerified {
t.Fatalf("恢复后 current 应指向预备份:%v %+v", err, current)
}
}
// R3:续作沿用原业务作业(维护窗口 owner 不变),只新增执行代次。
func TestPrepareLibraryJobResumeKeepsSameJobAndWindow(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
jobID := "libsync-resume-1"
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
if err != nil {
t.Fatal(err)
}
if err := store.AdvanceLibraryJobGuarded(ctx, first, governance.LibraryJobFailed, "database",
"写库失败", "按预备份恢复", "backup-snap-1", "", now); err != nil {
t.Fatalf("标记失败:%v", err)
}
// 维护窗口仍由**原作业**持有(这就是续作必须复用原行的原因)。
held, err := store.LibraryMaintenanceOf(ctx, "internal")
if err != nil || held == nil || held.OwnerJobID != jobID || held.ReleasedAt != nil {
t.Fatalf("维护窗口应仍由原作业持有:%v %+v", err, held)
}
prepared, resumed, err := store.PrepareLibraryJobResume(ctx, jobID, now)
if err != nil || !resumed {
t.Fatalf("续作准备失败:%v resumed=%v", err, resumed)
}
if prepared.ID != jobID || prepared.State != governance.LibraryJobPending {
t.Fatalf("续作必须是同一作业行且回到 pending:%+v", prepared)
}
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#1", 10*time.Minute, now.Add(time.Minute))
if err != nil {
t.Fatalf("续作代次失败:%v", err)
}
if second.Epoch != first.Epoch+1 || second.OperationID != first.OperationID {
t.Fatalf("续作必须是同一 operation 的新代次:%+v vs %+v", second, first)
}
// 新代次可以用原窗口释放(owner 仍是同一个作业)。
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, second, now.Add(2*time.Minute)); err != nil {
t.Fatalf("续作代次释放原窗口失败:%v", err)
}
}
// 编译期确认维护层级常量可用(避免测试里手写魔法字符串与实现漂移)。
var _ = librarymaintenance.LevelInstance
// 迁移 76 的 down/up 幂等:治理库里这一层只加身份表与 current 指针,
// 撤下来不能带走任何历史事件(版本行必须留着)。真实 PG 上逐条执行,
// 不靠"迁移文件存在"这种静态证据。
func TestMigration76DownAndUpKeepsVersionHistory(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
seedOperationJob(t, store, pool, "internal", "libsync-mig76", "sync", now)
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: governance.LibraryOperationID("internal", governance.LibraryJobSync, "legacy-op"),
Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "s",
RequestDigest: "d", OperationKey: "k", Actor: "t", CreatedAt: now,
}); err != nil {
t.Fatal(err)
}
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: "libver-mig76-keep", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
SnapshotID: "snap-mig76", SourceJobID: "libsync-mig76", CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": "snap-mig76"}, CreatedAt: now,
}); err != nil {
t.Fatal(err)
}
all, err := loadMigrations()
if err != nil {
t.Fatal(err)
}
var target migration
for _, item := range all {
if item.Version == 76 {
target = item
}
}
if target.Version == 0 {
t.Fatal("migration 76 不存在")
}
if _, err := pool.Exec(ctx, target.DownSQL); err != nil {
t.Fatalf("撤销 migration 76 失败:%v", err)
}
var versions int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_versions WHERE id='libver-mig76-keep'`).Scan(&versions); err != nil {
t.Fatalf("撤销后版本历史不可读:%v", err)
}
if versions != 1 {
t.Fatalf("撤销 identity 层不许删版本事件:count=%d", versions)
}
if _, err := pool.Exec(ctx, target.UpSQL); err != nil {
t.Fatalf("重新应用 migration 76 失败:%v", err)
}
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: "libver-mig76-after", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
SnapshotID: "snap-mig76-b", SourceJobID: "libsync-mig76", CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": "snap-mig76-b"}, CreatedAt: now.Add(time.Minute),
}); err != nil {
t.Fatalf("重新应用后写入失败:%v", err)
}
}