a7d643635a
build / build (push) Successful in 1m27s
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2, 导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的 owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。 R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引 (job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与 operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、 新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律 ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。 R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式 结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。 R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release 路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。 同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点 改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、 恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。 测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例; go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
526 lines
24 KiB
Go
526 lines
24 KiB
Go
package postgres
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"os"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
)
|
||
|
||
// 真实 PG 上的执行身份/栅栏用例。
|
||
//
|
||
// 为什么必须是真库(父的复核项 R1/R2):守卫的失败形态是"SQL 条件永远不匹配"——
|
||
// 参数序号、列引用、NULL 与空串的差别在内存 fake 里根本不存在。字符串断言证明不了
|
||
// 一条 UPDATE 真的命中了一行。
|
||
|
||
func newIdentityStore(t *testing.T) (*Store, *pgxpool.Pool) {
|
||
t.Helper()
|
||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||
if databaseURL == "" {
|
||
t.Skip("TEST_DATABASE_URL is not set")
|
||
}
|
||
ctx := context.Background()
|
||
pool, err := pgxpool.New(ctx, databaseURL)
|
||
if err != nil {
|
||
t.Fatalf("connect isolated database: %v", err)
|
||
}
|
||
t.Cleanup(pool.Close)
|
||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||
t.Fatalf("apply migrations: %v", err)
|
||
}
|
||
return NewStore(pool), pool
|
||
}
|
||
|
||
// seedOperationJob 建一条"有 operation 的业务作业",返回 operation 与作业 id。
|
||
func seedOperationJob(t *testing.T, store *Store, pool *pgxpool.Pool, scope, jobID, kind string, now time.Time) string {
|
||
t.Helper()
|
||
ctx := context.Background()
|
||
op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||
ID: governance.LibraryOperationID(scope, governance.LibraryJobKind(kind), jobID),
|
||
Scope: scope, Kind: governance.LibraryJobKind(kind), TargetSnapshotID: "snap-target",
|
||
RequestDigest: "digest-" + jobID, OperationKey: jobID, Actor: "tester", CreatedAt: now.UTC(),
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("ensure operation: %v", err)
|
||
}
|
||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
|
||
ID: jobID, Scope: scope, Kind: governance.LibraryJobKind(kind), OperationID: op.ID,
|
||
IdempotencyKey: "library.job:" + op.ID, RequestDigest: "digest-" + jobID,
|
||
TargetSnapshotID: "snap-target", Actor: "tester",
|
||
}, governance.SyncStages(), now); err != nil {
|
||
t.Fatalf("create library job: %v", err)
|
||
}
|
||
// 维护窗口由这个作业持有(实例级 + scope 级)。
|
||
if _, err := pool.Exec(ctx, `
|
||
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state,
|
||
note, acquired_at, updated_at)
|
||
VALUES ($1,$2,'target-identity','acquired','app:running,store:running','test',$3,$3)
|
||
ON CONFLICT (scope) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, released_at=NULL,
|
||
state='acquired', updated_at=EXCLUDED.updated_at`,
|
||
scope, jobID, now.UTC()); err != nil {
|
||
t.Fatalf("seed scope maintenance: %v", err)
|
||
}
|
||
if _, err := pool.Exec(ctx, `
|
||
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, state,
|
||
original_service_state, note, acquired_at, updated_at)
|
||
VALUES ('inst-1',$1,$2,'fingerprint-1','acquired','app:running','test',$3,$3)
|
||
ON CONFLICT (instance) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, owner_scope=EXCLUDED.owner_scope,
|
||
released_at=NULL, state='acquired', updated_at=EXCLUDED.updated_at`,
|
||
jobID, scope, now.UTC()); err != nil {
|
||
t.Fatalf("seed instance maintenance: %v", err)
|
||
}
|
||
return op.ID
|
||
}
|
||
|
||
// settleJob 把一个作业行推到终态:同一 scope 只允许一条在飞作业(部分唯一索引),
|
||
// 所以"再建一条兄弟作业"之前必须先让它落地。
|
||
func settleJob(t *testing.T, pool *pgxpool.Pool, jobID string, now time.Time) {
|
||
t.Helper()
|
||
if _, err := pool.Exec(context.Background(),
|
||
`UPDATE library_jobs SET state='failed', finished_at=$2, updated_at=$2 WHERE id=$1`,
|
||
jobID, now.UTC()); err != nil {
|
||
t.Fatalf("settle job %s: %v", jobID, err)
|
||
}
|
||
}
|
||
|
||
// R1:维护类守卫的作业 id 必须取自被更新行自己的 owner_job_id,而不是参数 $1。
|
||
//
|
||
// 旧实现把 `j.id=$1` 写死,$1 是 scope/instance ⇒ 守卫永远匹配不到作业,
|
||
// **正常**的维护状态写入与释放全部失败。这里逐个入口走一遍真库。
|
||
func TestLibraryMaintenanceFenceUsesRowOwner(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
jobID := "libsync-fence-1"
|
||
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||
|
||
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||
if err != nil {
|
||
t.Fatalf("begin attempt: %v", err)
|
||
}
|
||
if attempt.OperationID == "" || attempt.Epoch != 1 {
|
||
t.Fatalf("attempt = %+v", attempt)
|
||
}
|
||
|
||
// scope 级:状态、入口、释放都必须命中($1 是 scope,不是作业 id)。
|
||
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", jobID, "applying", "写目标", attempt, now); err != nil {
|
||
t.Fatalf("更新 scope 维护状态失败(守卫没命中作业):%v", err)
|
||
}
|
||
if err := store.SetLibraryMaintenanceIngressGuarded(ctx, "internal", jobID, "closed", "入口已关", attempt, now); err != nil {
|
||
t.Fatalf("记录 scope 入口状态失败:%v", err)
|
||
}
|
||
// 实例级:$1 是 instance。
|
||
if err := store.UpdateInstanceMaintenanceStateGuarded(ctx, "inst-1", jobID, "applying", "写目标", attempt, now); err != nil {
|
||
t.Fatalf("更新实例维护状态失败:%v", err)
|
||
}
|
||
if err := store.SetInstanceMaintenanceIngressGuarded(ctx, "inst-1", jobID, "closed", "入口已关", attempt, now); err != nil {
|
||
t.Fatalf("记录实例入口状态失败:%v", err)
|
||
}
|
||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); err != nil {
|
||
t.Fatalf("释放 scope 维护失败(正常路径被守卫挡死):%v", err)
|
||
}
|
||
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", jobID, attempt, now); err != nil {
|
||
t.Fatalf("释放实例维护失败:%v", err)
|
||
}
|
||
var released int
|
||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_maintenance WHERE released_at IS NOT NULL`).Scan(&released); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if released != 1 {
|
||
t.Fatalf("scope 维护应有 1 行已释放,实际 %d", released)
|
||
}
|
||
// 负例(同一代码路径):作业 id 不匹配时守卫必须**拒绝**,而不是默默返回成功。
|
||
// 这条与上面的正例一起,才证明 "$1 是 scope、作业 id 取自行" 的契约真的在起作用。
|
||
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", "libsync-other-job", attempt, now); err == nil {
|
||
t.Fatal("作业 id 不匹配时释放实例维护必须失败(守卫没看作业身份)")
|
||
}
|
||
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", "libsync-other-job", "applying", "x", attempt, now); err == nil {
|
||
t.Fatal("作业 id 不匹配时更新维护状态必须失败")
|
||
}
|
||
// 幂等:再释放一次是"已释放",不是"未持有"。
|
||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); !errors.Is(err, ErrMaintenanceReleased) {
|
||
t.Fatalf("重复释放应返回 ErrMaintenanceReleased,实际 %v", err)
|
||
}
|
||
}
|
||
|
||
// R2:同一作业连续三次认领 ⇒ 三个不可变代次;旧代次的写必须失败。
|
||
func TestBeginLibraryAttemptSameJobThreeGenerations(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
jobID := "libsync-gen-1"
|
||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||
|
||
var epochs []int
|
||
for round := 1; round <= 3; round++ {
|
||
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker#q#"+string(rune('0'+round)), 10*time.Minute, now.Add(time.Duration(round)*time.Minute))
|
||
if err != nil {
|
||
t.Fatalf("第 %d 次认领失败:%v", round, err)
|
||
}
|
||
epochs = append(epochs, attempt.Epoch)
|
||
// 结束上一代(模拟执行完毕/崩溃后接手):直接 Finish 或让它过期。
|
||
if err := store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptCompleted, "round done", now); err != nil {
|
||
t.Fatalf("第 %d 次结束代次失败:%v", round, err)
|
||
}
|
||
}
|
||
if len(epochs) != 3 || epochs[0] != 1 || epochs[1] != 2 || epochs[2] != 3 {
|
||
t.Fatalf("epoch 必须按 operation 单调递增:%v", epochs)
|
||
}
|
||
// 历史不可变:三次认领留下三行。
|
||
var rows int
|
||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rows != 3 {
|
||
t.Fatalf("同一作业三次认领必须保留 3 条代次(不可变历史),实际 %d", rows)
|
||
}
|
||
// 作业行指向最后一代。
|
||
job, err := store.LibraryJob(ctx, jobID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if job.AttemptEpoch != 3 {
|
||
t.Fatalf("作业行 attempt_epoch = %d, want 3", job.AttemptEpoch)
|
||
}
|
||
// 旧代次的守卫写必须失败(它的 attempt 已 completed)。
|
||
stale := governance.LibraryAttempt{OperationID: opID, Epoch: 2, JobID: jobID, Owner: "worker#q#2"}
|
||
if err := store.AdvanceLibraryJobGuarded(ctx, stale, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||
t.Fatalf("旧代次写业务状态必须被拒绝,实际 %v", err)
|
||
}
|
||
// 迟到的 finish 也必须拒绝:否则旧执行能把别人正在跑的那一代"结束"掉。
|
||
if err := store.FinishLibraryAttempt(ctx, stale, governance.LibraryAttemptCompleted, "late finish", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||
t.Fatalf("旧代次的 finish 必须被拒绝,实际 %v", err)
|
||
}
|
||
}
|
||
|
||
// R2:同一次投递重复调用是幂等的(同一代次,不新增行);别人的有效租约必须让路。
|
||
func TestBeginLibraryAttemptIdempotentRepeatAndHeldLease(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
jobID := "libsync-repeat-1"
|
||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||
|
||
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now.Add(time.Second))
|
||
if err != nil {
|
||
t.Fatalf("同一次投递重复调用必须幂等:%v", err)
|
||
}
|
||
if second.Epoch != first.Epoch {
|
||
t.Fatalf("重复调用不应新增代次:%d vs %d", second.Epoch, first.Epoch)
|
||
}
|
||
var rows int
|
||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rows != 1 {
|
||
t.Fatalf("幂等重复调用不应新增代次行,实际 %d", rows)
|
||
}
|
||
// 另一个执行者(不同 owner,租约未过期)必须被拒绝。
|
||
if _, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#0", 10*time.Minute, now.Add(2*time.Second)); !errors.Is(err, ErrLibraryAttemptHeld) {
|
||
t.Fatalf("别的执行者持有未过期租约时必须拒绝,实际 %v", err)
|
||
}
|
||
// 租约过期后允许接管,并结束旧代次。
|
||
later := now.Add(20 * time.Minute)
|
||
takeover, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#1", 10*time.Minute, later)
|
||
if err != nil {
|
||
t.Fatalf("租约过期后必须允许接管:%v", err)
|
||
}
|
||
if takeover.Epoch != first.Epoch+1 {
|
||
t.Fatalf("接管必须开新代次:%d", takeover.Epoch)
|
||
}
|
||
var oldState string
|
||
if err := pool.QueryRow(ctx, `SELECT state FROM library_attempts WHERE operation_id=$1 AND epoch=$2`,
|
||
opID, first.Epoch).Scan(&oldState); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if oldState != "abandoned" {
|
||
t.Fatalf("被接管的旧代次应标为 abandoned,实际 %q", oldState)
|
||
}
|
||
// 旧代次的续租必须失败(失租即停)。
|
||
if err := store.RenewLibraryAttempt(ctx, first, time.Minute, later); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||
t.Fatalf("旧代次续租必须失败,实际 %v", err)
|
||
}
|
||
}
|
||
|
||
// R2:同一 operation 不同作业(续作留下的兄弟行)——新代次激活后,旧作业行的守卫必须失效。
|
||
func TestSiblingJobGuardInvalidatedByNewAttempt(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
firstJob := "libsync-sib-1"
|
||
opID := seedOperationJob(t, store, pool, "internal", firstJob, "sync", now)
|
||
|
||
firstAttempt, err := store.BeginLibraryAttempt(ctx, firstJob, "worker-a#q1#0", 10*time.Minute, now)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
settleJob(t, pool, firstJob, now)
|
||
// 兄弟作业行(同一 operation,模拟历史 `-resume` 行/并行执行者)。
|
||
secondJob := firstJob + "-resume"
|
||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
|
||
ID: secondJob, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: opID,
|
||
IdempotencyKey: "library.job:" + opID + ":resume", RequestDigest: "digest-sib",
|
||
TargetSnapshotID: "snap-target", Actor: "tester",
|
||
}, governance.SyncStages(), now); err != nil {
|
||
t.Fatalf("create sibling job: %v", err)
|
||
}
|
||
// 结束第一代后,由兄弟作业接管同一 operation。
|
||
if err := store.FinishLibraryAttempt(ctx, firstAttempt, governance.LibraryAttemptFailed, "crashed", now); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
settleJob(t, pool, firstJob, now)
|
||
secondAttempt, err := store.BeginLibraryAttempt(ctx, secondJob, "worker-b#q2#0", 10*time.Minute, now.Add(time.Minute))
|
||
if err != nil {
|
||
t.Fatalf("兄弟作业接管失败:%v", err)
|
||
}
|
||
if secondAttempt.OperationID != opID || secondAttempt.Epoch != firstAttempt.Epoch+1 {
|
||
t.Fatalf("兄弟作业必须共用同一 operation 并开新一代:%+v", secondAttempt)
|
||
}
|
||
// 旧作业行上的 epoch/owner 字段还在,但它的守卫必须失败。
|
||
if err := store.AdvanceLibraryJobGuarded(ctx, firstAttempt, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||
t.Fatalf("旧作业的守卫必须失效,实际 %v", err)
|
||
}
|
||
var siblingLease string
|
||
if err := pool.QueryRow(ctx, `SELECT lease_owner FROM library_jobs WHERE id=$1`, firstJob).Scan(&siblingLease); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if siblingLease != "" {
|
||
t.Fatalf("兄弟接管后旧作业行租约必须被清空,实际 %q", siblingLease)
|
||
}
|
||
}
|
||
|
||
// 首次锚点:事务 + 回读核对,NULL 与空串不清混;覆盖被拒绝。
|
||
func TestRecordOperationAnchorsTransactionalAndNullSafe(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
jobID := "libsync-anchor-1"
|
||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||
|
||
// 只登记快照 id(字节/写集合未知)⇒ 对应列必须是 NULL,而不是 0/空串。
|
||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||
FirstBackupSnapshotID: "backup-snap-1",
|
||
}, now); err != nil {
|
||
t.Fatalf("首次登记锚点失败:%v", err)
|
||
}
|
||
var bytesNull, rowsNull, digestNull bool
|
||
if err := pool.QueryRow(ctx, `
|
||
SELECT first_backup_bytes IS NULL, writeset_rows IS NULL, writeset_digest IS NULL
|
||
FROM library_operations WHERE id=$1`, opID).Scan(&bytesNull, &rowsNull, &digestNull); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !bytesNull || !rowsNull || !digestNull {
|
||
t.Fatalf("未知锚点必须留 NULL:bytes_null=%v rows_null=%v digest_null=%v", bytesNull, rowsNull, digestNull)
|
||
}
|
||
// 补齐字节与写集合:允许填空。
|
||
bytes := int64(4096)
|
||
rows := int64(119)
|
||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||
FirstBackupBytes: &bytes, WriteSetRows: &rows, WriteSetDigest: "ws-digest-1",
|
||
}, now); err != nil {
|
||
t.Fatalf("补齐锚点失败:%v", err)
|
||
}
|
||
var gotBytes int64
|
||
var gotDigest string
|
||
if err := pool.QueryRow(ctx, `SELECT first_backup_bytes, writeset_digest FROM library_operations WHERE id=$1`,
|
||
opID).Scan(&gotBytes, &gotDigest); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if gotBytes != bytes || gotDigest != "ws-digest-1" {
|
||
t.Fatalf("锚点回读不一致:%d/%s", gotBytes, gotDigest)
|
||
}
|
||
// 覆盖被拒绝(字节与写集合都不许换)。
|
||
other := int64(1)
|
||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{FirstBackupBytes: &other}, now); err == nil {
|
||
t.Fatal("用不同字节覆盖首次锚点必须失败")
|
||
}
|
||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{WriteSetDigest: "ws-digest-2"}, now); err == nil {
|
||
t.Fatal("用不同写集合摘要覆盖首次锚点必须失败")
|
||
}
|
||
// 幂等:同一份锚点重复登记成功且不改值。
|
||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||
FirstBackupSnapshotID: "backup-snap-1", FirstBackupBytes: &bytes, WriteSetDigest: "ws-digest-1",
|
||
}, now); err != nil {
|
||
t.Fatalf("同一份锚点重复登记必须幂等成功:%v", err)
|
||
}
|
||
}
|
||
|
||
// current 指针:旧意图后来登记的事实**不能**把 current 推回历史;新意图(恢复)可以。
|
||
func TestLibraryCurrentOrderingRejectsLateOldIntent(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
base := time.Now().UTC()
|
||
jobA, jobB := "libsync-ord-a", "libsync-ord-b"
|
||
opA := seedOperationJob(t, store, pool, "internal", jobA, "sync", base)
|
||
settleJob(t, pool, jobA, base)
|
||
opB := seedOperationJob(t, store, pool, "internal", jobB, "sync", base.Add(time.Minute))
|
||
|
||
effective := func(opID, jobID, snapshot string, at time.Time) error {
|
||
return store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", snapshot, opID),
|
||
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot,
|
||
SourceJobID: jobID, OperationID: opID, CreatedBy: "tester", CreatedAt: at,
|
||
Evidence: map[string]any{"backup_snapshot": "backup-" + snapshot},
|
||
})
|
||
}
|
||
if err := effective(opA, jobA, "snap-a", base); err != nil {
|
||
t.Fatalf("登记 A 失败:%v", err)
|
||
}
|
||
if err := effective(opB, jobB, "snap-b", base.Add(time.Minute)); err != nil {
|
||
t.Fatalf("登记 B 失败:%v", err)
|
||
}
|
||
current, found, err := store.LibraryCurrentVersion(ctx, "internal")
|
||
if err != nil || !found || current.SnapshotID != "snap-b" {
|
||
t.Fatalf("current 应为 snap-b:%v %+v", err, current)
|
||
}
|
||
// 旧意图(A,创建时间更早)后来又登记了一条新事实:必须被拒绝,不能把 current 拉回去。
|
||
late := governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", "snap-a2", opA),
|
||
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-a2",
|
||
SourceJobID: jobA, OperationID: opA, CreatedBy: "tester", CreatedAt: base.Add(2 * time.Hour),
|
||
Evidence: map[string]any{"backup_snapshot": "backup-snap-a2"},
|
||
}
|
||
if err := store.RecordLibraryVersion(ctx, late); !errors.Is(err, governance.ErrLibraryCurrentStale) {
|
||
t.Fatalf("旧意图的迟到事实必须被拒绝,实际 %v", err)
|
||
}
|
||
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
|
||
if err != nil || !found || current.SnapshotID != "snap-b" {
|
||
t.Fatalf("current 不应被旧意图推回:%v %+v", err, current)
|
||
}
|
||
|
||
// 恢复是**新意图**:内容更旧(backup),意图更新 ⇒ 允许推进。
|
||
recoveryOp, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||
ID: governance.LibraryOperationID("internal", governance.LibraryJobRecover, "window-1"),
|
||
Scope: "internal", Kind: governance.LibraryJobRecover, TargetSnapshotID: "backup-snap-a",
|
||
RequestDigest: "recover-digest", OperationKey: "window-1", Actor: "tester",
|
||
CreatedAt: base.Add(3 * time.Hour),
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("登记恢复意图失败:%v", err)
|
||
}
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionRecoveryVerified, "internal", "backup-snap-a", recoveryOp.ID),
|
||
Scope: "internal", Kind: governance.LibraryVersionRecoveryVerified, SnapshotID: "backup-snap-a",
|
||
SourceJobID: jobA, OperationID: recoveryOp.ID, CreatedBy: "worker", CreatedAt: base.Add(3 * time.Hour),
|
||
Evidence: map[string]any{"backup_snapshot": "backup-snap-a", "verify_digest": "digest"},
|
||
}); err != nil {
|
||
t.Fatalf("恢复事实必须能推进 current:%v", err)
|
||
}
|
||
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
|
||
if err != nil || !found || current.SnapshotID != "backup-snap-a" || current.Kind != governance.LibraryVersionRecoveryVerified {
|
||
t.Fatalf("恢复后 current 应指向预备份:%v %+v", err, current)
|
||
}
|
||
}
|
||
|
||
// R3:续作沿用原业务作业(维护窗口 owner 不变),只新增执行代次。
|
||
func TestPrepareLibraryJobResumeKeepsSameJobAndWindow(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
jobID := "libsync-resume-1"
|
||
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||
|
||
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := store.AdvanceLibraryJobGuarded(ctx, first, governance.LibraryJobFailed, "database",
|
||
"写库失败", "按预备份恢复", "backup-snap-1", "", now); err != nil {
|
||
t.Fatalf("标记失败:%v", err)
|
||
}
|
||
// 维护窗口仍由**原作业**持有(这就是续作必须复用原行的原因)。
|
||
held, err := store.LibraryMaintenanceOf(ctx, "internal")
|
||
if err != nil || held == nil || held.OwnerJobID != jobID || held.ReleasedAt != nil {
|
||
t.Fatalf("维护窗口应仍由原作业持有:%v %+v", err, held)
|
||
}
|
||
prepared, resumed, err := store.PrepareLibraryJobResume(ctx, jobID, now)
|
||
if err != nil || !resumed {
|
||
t.Fatalf("续作准备失败:%v resumed=%v", err, resumed)
|
||
}
|
||
if prepared.ID != jobID || prepared.State != governance.LibraryJobPending {
|
||
t.Fatalf("续作必须是同一作业行且回到 pending:%+v", prepared)
|
||
}
|
||
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#1", 10*time.Minute, now.Add(time.Minute))
|
||
if err != nil {
|
||
t.Fatalf("续作代次失败:%v", err)
|
||
}
|
||
if second.Epoch != first.Epoch+1 || second.OperationID != first.OperationID {
|
||
t.Fatalf("续作必须是同一 operation 的新代次:%+v vs %+v", second, first)
|
||
}
|
||
// 新代次可以用原窗口释放(owner 仍是同一个作业)。
|
||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, second, now.Add(2*time.Minute)); err != nil {
|
||
t.Fatalf("续作代次释放原窗口失败:%v", err)
|
||
}
|
||
}
|
||
|
||
// 编译期确认维护层级常量可用(避免测试里手写魔法字符串与实现漂移)。
|
||
var _ = librarymaintenance.LevelInstance
|
||
|
||
// 迁移 76 的 down/up 幂等:治理库里这一层只加身份表与 current 指针,
|
||
// 撤下来不能带走任何历史事件(版本行必须留着)。真实 PG 上逐条执行,
|
||
// 不靠"迁移文件存在"这种静态证据。
|
||
func TestMigration76DownAndUpKeepsVersionHistory(t *testing.T) {
|
||
store, pool := newIdentityStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
seedOperationJob(t, store, pool, "internal", "libsync-mig76", "sync", now)
|
||
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||
ID: governance.LibraryOperationID("internal", governance.LibraryJobSync, "legacy-op"),
|
||
Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "s",
|
||
RequestDigest: "d", OperationKey: "k", Actor: "t", CreatedAt: now,
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: "libver-mig76-keep", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
|
||
SnapshotID: "snap-mig76", SourceJobID: "libsync-mig76", CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": "snap-mig76"}, CreatedAt: now,
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
all, err := loadMigrations()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var target migration
|
||
for _, item := range all {
|
||
if item.Version == 76 {
|
||
target = item
|
||
}
|
||
}
|
||
if target.Version == 0 {
|
||
t.Fatal("migration 76 不存在")
|
||
}
|
||
if _, err := pool.Exec(ctx, target.DownSQL); err != nil {
|
||
t.Fatalf("撤销 migration 76 失败:%v", err)
|
||
}
|
||
var versions int
|
||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_versions WHERE id='libver-mig76-keep'`).Scan(&versions); err != nil {
|
||
t.Fatalf("撤销后版本历史不可读:%v", err)
|
||
}
|
||
if versions != 1 {
|
||
t.Fatalf("撤销 identity 层不许删版本事件:count=%d", versions)
|
||
}
|
||
if _, err := pool.Exec(ctx, target.UpSQL); err != nil {
|
||
t.Fatalf("重新应用 migration 76 失败:%v", err)
|
||
}
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: "libver-mig76-after", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
|
||
SnapshotID: "snap-mig76-b", SourceJobID: "libsync-mig76", CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": "snap-mig76-b"}, CreatedAt: now.Add(time.Minute),
|
||
}); err != nil {
|
||
t.Fatalf("重新应用后写入失败:%v", err)
|
||
}
|
||
}
|