R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2, 导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的 owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。 R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引 (job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与 operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、 新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律 ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。 R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式 结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。 R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release 路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。 同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点 改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、 恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。 测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例; go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
This commit is contained in:
@@ -0,0 +1,525 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/librarymaintenance"
|
||||
)
|
||||
|
||||
// 真实 PG 上的执行身份/栅栏用例。
|
||||
//
|
||||
// 为什么必须是真库(父的复核项 R1/R2):守卫的失败形态是"SQL 条件永远不匹配"——
|
||||
// 参数序号、列引用、NULL 与空串的差别在内存 fake 里根本不存在。字符串断言证明不了
|
||||
// 一条 UPDATE 真的命中了一行。
|
||||
|
||||
func newIdentityStore(t *testing.T) (*Store, *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect isolated database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("apply migrations: %v", err)
|
||||
}
|
||||
return NewStore(pool), pool
|
||||
}
|
||||
|
||||
// seedOperationJob 建一条"有 operation 的业务作业",返回 operation 与作业 id。
|
||||
func seedOperationJob(t *testing.T, store *Store, pool *pgxpool.Pool, scope, jobID, kind string, now time.Time) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: governance.LibraryOperationID(scope, governance.LibraryJobKind(kind), jobID),
|
||||
Scope: scope, Kind: governance.LibraryJobKind(kind), TargetSnapshotID: "snap-target",
|
||||
RequestDigest: "digest-" + jobID, OperationKey: jobID, Actor: "tester", CreatedAt: now.UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ensure operation: %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
|
||||
ID: jobID, Scope: scope, Kind: governance.LibraryJobKind(kind), OperationID: op.ID,
|
||||
IdempotencyKey: "library.job:" + op.ID, RequestDigest: "digest-" + jobID,
|
||||
TargetSnapshotID: "snap-target", Actor: "tester",
|
||||
}, governance.SyncStages(), now); err != nil {
|
||||
t.Fatalf("create library job: %v", err)
|
||||
}
|
||||
// 维护窗口由这个作业持有(实例级 + scope 级)。
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state,
|
||||
note, acquired_at, updated_at)
|
||||
VALUES ($1,$2,'target-identity','acquired','app:running,store:running','test',$3,$3)
|
||||
ON CONFLICT (scope) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, released_at=NULL,
|
||||
state='acquired', updated_at=EXCLUDED.updated_at`,
|
||||
scope, jobID, now.UTC()); err != nil {
|
||||
t.Fatalf("seed scope maintenance: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, state,
|
||||
original_service_state, note, acquired_at, updated_at)
|
||||
VALUES ('inst-1',$1,$2,'fingerprint-1','acquired','app:running','test',$3,$3)
|
||||
ON CONFLICT (instance) DO UPDATE SET owner_job_id=EXCLUDED.owner_job_id, owner_scope=EXCLUDED.owner_scope,
|
||||
released_at=NULL, state='acquired', updated_at=EXCLUDED.updated_at`,
|
||||
jobID, scope, now.UTC()); err != nil {
|
||||
t.Fatalf("seed instance maintenance: %v", err)
|
||||
}
|
||||
return op.ID
|
||||
}
|
||||
|
||||
// settleJob 把一个作业行推到终态:同一 scope 只允许一条在飞作业(部分唯一索引),
|
||||
// 所以"再建一条兄弟作业"之前必须先让它落地。
|
||||
func settleJob(t *testing.T, pool *pgxpool.Pool, jobID string, now time.Time) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(context.Background(),
|
||||
`UPDATE library_jobs SET state='failed', finished_at=$2, updated_at=$2 WHERE id=$1`,
|
||||
jobID, now.UTC()); err != nil {
|
||||
t.Fatalf("settle job %s: %v", jobID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// R1:维护类守卫的作业 id 必须取自被更新行自己的 owner_job_id,而不是参数 $1。
|
||||
//
|
||||
// 旧实现把 `j.id=$1` 写死,$1 是 scope/instance ⇒ 守卫永远匹配不到作业,
|
||||
// **正常**的维护状态写入与释放全部失败。这里逐个入口走一遍真库。
|
||||
func TestLibraryMaintenanceFenceUsesRowOwner(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
jobID := "libsync-fence-1"
|
||||
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
|
||||
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatalf("begin attempt: %v", err)
|
||||
}
|
||||
if attempt.OperationID == "" || attempt.Epoch != 1 {
|
||||
t.Fatalf("attempt = %+v", attempt)
|
||||
}
|
||||
|
||||
// scope 级:状态、入口、释放都必须命中($1 是 scope,不是作业 id)。
|
||||
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", jobID, "applying", "写目标", attempt, now); err != nil {
|
||||
t.Fatalf("更新 scope 维护状态失败(守卫没命中作业):%v", err)
|
||||
}
|
||||
if err := store.SetLibraryMaintenanceIngressGuarded(ctx, "internal", jobID, "closed", "入口已关", attempt, now); err != nil {
|
||||
t.Fatalf("记录 scope 入口状态失败:%v", err)
|
||||
}
|
||||
// 实例级:$1 是 instance。
|
||||
if err := store.UpdateInstanceMaintenanceStateGuarded(ctx, "inst-1", jobID, "applying", "写目标", attempt, now); err != nil {
|
||||
t.Fatalf("更新实例维护状态失败:%v", err)
|
||||
}
|
||||
if err := store.SetInstanceMaintenanceIngressGuarded(ctx, "inst-1", jobID, "closed", "入口已关", attempt, now); err != nil {
|
||||
t.Fatalf("记录实例入口状态失败:%v", err)
|
||||
}
|
||||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); err != nil {
|
||||
t.Fatalf("释放 scope 维护失败(正常路径被守卫挡死):%v", err)
|
||||
}
|
||||
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", jobID, attempt, now); err != nil {
|
||||
t.Fatalf("释放实例维护失败:%v", err)
|
||||
}
|
||||
var released int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_maintenance WHERE released_at IS NOT NULL`).Scan(&released); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if released != 1 {
|
||||
t.Fatalf("scope 维护应有 1 行已释放,实际 %d", released)
|
||||
}
|
||||
// 负例(同一代码路径):作业 id 不匹配时守卫必须**拒绝**,而不是默默返回成功。
|
||||
// 这条与上面的正例一起,才证明 "$1 是 scope、作业 id 取自行" 的契约真的在起作用。
|
||||
if err := store.ReleaseInstanceMaintenanceGuarded(ctx, "inst-1", "libsync-other-job", attempt, now); err == nil {
|
||||
t.Fatal("作业 id 不匹配时释放实例维护必须失败(守卫没看作业身份)")
|
||||
}
|
||||
if err := store.UpdateLibraryMaintenanceStateGuarded(ctx, "internal", "libsync-other-job", "applying", "x", attempt, now); err == nil {
|
||||
t.Fatal("作业 id 不匹配时更新维护状态必须失败")
|
||||
}
|
||||
// 幂等:再释放一次是"已释放",不是"未持有"。
|
||||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, attempt, now); !errors.Is(err, ErrMaintenanceReleased) {
|
||||
t.Fatalf("重复释放应返回 ErrMaintenanceReleased,实际 %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// R2:同一作业连续三次认领 ⇒ 三个不可变代次;旧代次的写必须失败。
|
||||
func TestBeginLibraryAttemptSameJobThreeGenerations(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
jobID := "libsync-gen-1"
|
||||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
|
||||
var epochs []int
|
||||
for round := 1; round <= 3; round++ {
|
||||
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker#q#"+string(rune('0'+round)), 10*time.Minute, now.Add(time.Duration(round)*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("第 %d 次认领失败:%v", round, err)
|
||||
}
|
||||
epochs = append(epochs, attempt.Epoch)
|
||||
// 结束上一代(模拟执行完毕/崩溃后接手):直接 Finish 或让它过期。
|
||||
if err := store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptCompleted, "round done", now); err != nil {
|
||||
t.Fatalf("第 %d 次结束代次失败:%v", round, err)
|
||||
}
|
||||
}
|
||||
if len(epochs) != 3 || epochs[0] != 1 || epochs[1] != 2 || epochs[2] != 3 {
|
||||
t.Fatalf("epoch 必须按 operation 单调递增:%v", epochs)
|
||||
}
|
||||
// 历史不可变:三次认领留下三行。
|
||||
var rows int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 3 {
|
||||
t.Fatalf("同一作业三次认领必须保留 3 条代次(不可变历史),实际 %d", rows)
|
||||
}
|
||||
// 作业行指向最后一代。
|
||||
job, err := store.LibraryJob(ctx, jobID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.AttemptEpoch != 3 {
|
||||
t.Fatalf("作业行 attempt_epoch = %d, want 3", job.AttemptEpoch)
|
||||
}
|
||||
// 旧代次的守卫写必须失败(它的 attempt 已 completed)。
|
||||
stale := governance.LibraryAttempt{OperationID: opID, Epoch: 2, JobID: jobID, Owner: "worker#q#2"}
|
||||
if err := store.AdvanceLibraryJobGuarded(ctx, stale, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||||
t.Fatalf("旧代次写业务状态必须被拒绝,实际 %v", err)
|
||||
}
|
||||
// 迟到的 finish 也必须拒绝:否则旧执行能把别人正在跑的那一代"结束"掉。
|
||||
if err := store.FinishLibraryAttempt(ctx, stale, governance.LibraryAttemptCompleted, "late finish", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||||
t.Fatalf("旧代次的 finish 必须被拒绝,实际 %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// R2:同一次投递重复调用是幂等的(同一代次,不新增行);别人的有效租约必须让路。
|
||||
func TestBeginLibraryAttemptIdempotentRepeatAndHeldLease(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
jobID := "libsync-repeat-1"
|
||||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
|
||||
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now.Add(time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("同一次投递重复调用必须幂等:%v", err)
|
||||
}
|
||||
if second.Epoch != first.Epoch {
|
||||
t.Fatalf("重复调用不应新增代次:%d vs %d", second.Epoch, first.Epoch)
|
||||
}
|
||||
var rows int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_attempts WHERE operation_id=$1`, opID).Scan(&rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("幂等重复调用不应新增代次行,实际 %d", rows)
|
||||
}
|
||||
// 另一个执行者(不同 owner,租约未过期)必须被拒绝。
|
||||
if _, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#0", 10*time.Minute, now.Add(2*time.Second)); !errors.Is(err, ErrLibraryAttemptHeld) {
|
||||
t.Fatalf("别的执行者持有未过期租约时必须拒绝,实际 %v", err)
|
||||
}
|
||||
// 租约过期后允许接管,并结束旧代次。
|
||||
later := now.Add(20 * time.Minute)
|
||||
takeover, err := store.BeginLibraryAttempt(ctx, jobID, "worker-b#q1#1", 10*time.Minute, later)
|
||||
if err != nil {
|
||||
t.Fatalf("租约过期后必须允许接管:%v", err)
|
||||
}
|
||||
if takeover.Epoch != first.Epoch+1 {
|
||||
t.Fatalf("接管必须开新代次:%d", takeover.Epoch)
|
||||
}
|
||||
var oldState string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_attempts WHERE operation_id=$1 AND epoch=$2`,
|
||||
opID, first.Epoch).Scan(&oldState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if oldState != "abandoned" {
|
||||
t.Fatalf("被接管的旧代次应标为 abandoned,实际 %q", oldState)
|
||||
}
|
||||
// 旧代次的续租必须失败(失租即停)。
|
||||
if err := store.RenewLibraryAttempt(ctx, first, time.Minute, later); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||||
t.Fatalf("旧代次续租必须失败,实际 %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// R2:同一 operation 不同作业(续作留下的兄弟行)——新代次激活后,旧作业行的守卫必须失效。
|
||||
func TestSiblingJobGuardInvalidatedByNewAttempt(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
firstJob := "libsync-sib-1"
|
||||
opID := seedOperationJob(t, store, pool, "internal", firstJob, "sync", now)
|
||||
|
||||
firstAttempt, err := store.BeginLibraryAttempt(ctx, firstJob, "worker-a#q1#0", 10*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settleJob(t, pool, firstJob, now)
|
||||
// 兄弟作业行(同一 operation,模拟历史 `-resume` 行/并行执行者)。
|
||||
secondJob := firstJob + "-resume"
|
||||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
|
||||
ID: secondJob, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: opID,
|
||||
IdempotencyKey: "library.job:" + opID + ":resume", RequestDigest: "digest-sib",
|
||||
TargetSnapshotID: "snap-target", Actor: "tester",
|
||||
}, governance.SyncStages(), now); err != nil {
|
||||
t.Fatalf("create sibling job: %v", err)
|
||||
}
|
||||
// 结束第一代后,由兄弟作业接管同一 operation。
|
||||
if err := store.FinishLibraryAttempt(ctx, firstAttempt, governance.LibraryAttemptFailed, "crashed", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settleJob(t, pool, firstJob, now)
|
||||
secondAttempt, err := store.BeginLibraryAttempt(ctx, secondJob, "worker-b#q2#0", 10*time.Minute, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("兄弟作业接管失败:%v", err)
|
||||
}
|
||||
if secondAttempt.OperationID != opID || secondAttempt.Epoch != firstAttempt.Epoch+1 {
|
||||
t.Fatalf("兄弟作业必须共用同一 operation 并开新一代:%+v", secondAttempt)
|
||||
}
|
||||
// 旧作业行上的 epoch/owner 字段还在,但它的守卫必须失败。
|
||||
if err := store.AdvanceLibraryJobGuarded(ctx, firstAttempt, governance.LibraryJobRunning, "", "", "", "", "", now); !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||||
t.Fatalf("旧作业的守卫必须失效,实际 %v", err)
|
||||
}
|
||||
var siblingLease string
|
||||
if err := pool.QueryRow(ctx, `SELECT lease_owner FROM library_jobs WHERE id=$1`, firstJob).Scan(&siblingLease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if siblingLease != "" {
|
||||
t.Fatalf("兄弟接管后旧作业行租约必须被清空,实际 %q", siblingLease)
|
||||
}
|
||||
}
|
||||
|
||||
// 首次锚点:事务 + 回读核对,NULL 与空串不清混;覆盖被拒绝。
|
||||
func TestRecordOperationAnchorsTransactionalAndNullSafe(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
jobID := "libsync-anchor-1"
|
||||
opID := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
|
||||
// 只登记快照 id(字节/写集合未知)⇒ 对应列必须是 NULL,而不是 0/空串。
|
||||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||||
FirstBackupSnapshotID: "backup-snap-1",
|
||||
}, now); err != nil {
|
||||
t.Fatalf("首次登记锚点失败:%v", err)
|
||||
}
|
||||
var bytesNull, rowsNull, digestNull bool
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT first_backup_bytes IS NULL, writeset_rows IS NULL, writeset_digest IS NULL
|
||||
FROM library_operations WHERE id=$1`, opID).Scan(&bytesNull, &rowsNull, &digestNull); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytesNull || !rowsNull || !digestNull {
|
||||
t.Fatalf("未知锚点必须留 NULL:bytes_null=%v rows_null=%v digest_null=%v", bytesNull, rowsNull, digestNull)
|
||||
}
|
||||
// 补齐字节与写集合:允许填空。
|
||||
bytes := int64(4096)
|
||||
rows := int64(119)
|
||||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||||
FirstBackupBytes: &bytes, WriteSetRows: &rows, WriteSetDigest: "ws-digest-1",
|
||||
}, now); err != nil {
|
||||
t.Fatalf("补齐锚点失败:%v", err)
|
||||
}
|
||||
var gotBytes int64
|
||||
var gotDigest string
|
||||
if err := pool.QueryRow(ctx, `SELECT first_backup_bytes, writeset_digest FROM library_operations WHERE id=$1`,
|
||||
opID).Scan(&gotBytes, &gotDigest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotBytes != bytes || gotDigest != "ws-digest-1" {
|
||||
t.Fatalf("锚点回读不一致:%d/%s", gotBytes, gotDigest)
|
||||
}
|
||||
// 覆盖被拒绝(字节与写集合都不许换)。
|
||||
other := int64(1)
|
||||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{FirstBackupBytes: &other}, now); err == nil {
|
||||
t.Fatal("用不同字节覆盖首次锚点必须失败")
|
||||
}
|
||||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{WriteSetDigest: "ws-digest-2"}, now); err == nil {
|
||||
t.Fatal("用不同写集合摘要覆盖首次锚点必须失败")
|
||||
}
|
||||
// 幂等:同一份锚点重复登记成功且不改值。
|
||||
if err := store.RecordOperationAnchors(ctx, opID, governance.LibraryAnchors{
|
||||
FirstBackupSnapshotID: "backup-snap-1", FirstBackupBytes: &bytes, WriteSetDigest: "ws-digest-1",
|
||||
}, now); err != nil {
|
||||
t.Fatalf("同一份锚点重复登记必须幂等成功:%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// current 指针:旧意图后来登记的事实**不能**把 current 推回历史;新意图(恢复)可以。
|
||||
func TestLibraryCurrentOrderingRejectsLateOldIntent(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
base := time.Now().UTC()
|
||||
jobA, jobB := "libsync-ord-a", "libsync-ord-b"
|
||||
opA := seedOperationJob(t, store, pool, "internal", jobA, "sync", base)
|
||||
settleJob(t, pool, jobA, base)
|
||||
opB := seedOperationJob(t, store, pool, "internal", jobB, "sync", base.Add(time.Minute))
|
||||
|
||||
effective := func(opID, jobID, snapshot string, at time.Time) error {
|
||||
return store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", snapshot, opID),
|
||||
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot,
|
||||
SourceJobID: jobID, OperationID: opID, CreatedBy: "tester", CreatedAt: at,
|
||||
Evidence: map[string]any{"backup_snapshot": "backup-" + snapshot},
|
||||
})
|
||||
}
|
||||
if err := effective(opA, jobA, "snap-a", base); err != nil {
|
||||
t.Fatalf("登记 A 失败:%v", err)
|
||||
}
|
||||
if err := effective(opB, jobB, "snap-b", base.Add(time.Minute)); err != nil {
|
||||
t.Fatalf("登记 B 失败:%v", err)
|
||||
}
|
||||
current, found, err := store.LibraryCurrentVersion(ctx, "internal")
|
||||
if err != nil || !found || current.SnapshotID != "snap-b" {
|
||||
t.Fatalf("current 应为 snap-b:%v %+v", err, current)
|
||||
}
|
||||
// 旧意图(A,创建时间更早)后来又登记了一条新事实:必须被拒绝,不能把 current 拉回去。
|
||||
late := governance.LibraryVersion{
|
||||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, "internal", "snap-a2", opA),
|
||||
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-a2",
|
||||
SourceJobID: jobA, OperationID: opA, CreatedBy: "tester", CreatedAt: base.Add(2 * time.Hour),
|
||||
Evidence: map[string]any{"backup_snapshot": "backup-snap-a2"},
|
||||
}
|
||||
if err := store.RecordLibraryVersion(ctx, late); !errors.Is(err, governance.ErrLibraryCurrentStale) {
|
||||
t.Fatalf("旧意图的迟到事实必须被拒绝,实际 %v", err)
|
||||
}
|
||||
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
|
||||
if err != nil || !found || current.SnapshotID != "snap-b" {
|
||||
t.Fatalf("current 不应被旧意图推回:%v %+v", err, current)
|
||||
}
|
||||
|
||||
// 恢复是**新意图**:内容更旧(backup),意图更新 ⇒ 允许推进。
|
||||
recoveryOp, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: governance.LibraryOperationID("internal", governance.LibraryJobRecover, "window-1"),
|
||||
Scope: "internal", Kind: governance.LibraryJobRecover, TargetSnapshotID: "backup-snap-a",
|
||||
RequestDigest: "recover-digest", OperationKey: "window-1", Actor: "tester",
|
||||
CreatedAt: base.Add(3 * time.Hour),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("登记恢复意图失败:%v", err)
|
||||
}
|
||||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: governance.LibraryVersionEventID(governance.LibraryVersionRecoveryVerified, "internal", "backup-snap-a", recoveryOp.ID),
|
||||
Scope: "internal", Kind: governance.LibraryVersionRecoveryVerified, SnapshotID: "backup-snap-a",
|
||||
SourceJobID: jobA, OperationID: recoveryOp.ID, CreatedBy: "worker", CreatedAt: base.Add(3 * time.Hour),
|
||||
Evidence: map[string]any{"backup_snapshot": "backup-snap-a", "verify_digest": "digest"},
|
||||
}); err != nil {
|
||||
t.Fatalf("恢复事实必须能推进 current:%v", err)
|
||||
}
|
||||
current, found, err = store.LibraryCurrentVersion(ctx, "internal")
|
||||
if err != nil || !found || current.SnapshotID != "backup-snap-a" || current.Kind != governance.LibraryVersionRecoveryVerified {
|
||||
t.Fatalf("恢复后 current 应指向预备份:%v %+v", err, current)
|
||||
}
|
||||
}
|
||||
|
||||
// R3:续作沿用原业务作业(维护窗口 owner 不变),只新增执行代次。
|
||||
func TestPrepareLibraryJobResumeKeepsSameJobAndWindow(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
jobID := "libsync-resume-1"
|
||||
seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
|
||||
first, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#0", 10*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.AdvanceLibraryJobGuarded(ctx, first, governance.LibraryJobFailed, "database",
|
||||
"写库失败", "按预备份恢复", "backup-snap-1", "", now); err != nil {
|
||||
t.Fatalf("标记失败:%v", err)
|
||||
}
|
||||
// 维护窗口仍由**原作业**持有(这就是续作必须复用原行的原因)。
|
||||
held, err := store.LibraryMaintenanceOf(ctx, "internal")
|
||||
if err != nil || held == nil || held.OwnerJobID != jobID || held.ReleasedAt != nil {
|
||||
t.Fatalf("维护窗口应仍由原作业持有:%v %+v", err, held)
|
||||
}
|
||||
prepared, resumed, err := store.PrepareLibraryJobResume(ctx, jobID, now)
|
||||
if err != nil || !resumed {
|
||||
t.Fatalf("续作准备失败:%v resumed=%v", err, resumed)
|
||||
}
|
||||
if prepared.ID != jobID || prepared.State != governance.LibraryJobPending {
|
||||
t.Fatalf("续作必须是同一作业行且回到 pending:%+v", prepared)
|
||||
}
|
||||
second, err := store.BeginLibraryAttempt(ctx, jobID, "worker-a#q1#1", 10*time.Minute, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("续作代次失败:%v", err)
|
||||
}
|
||||
if second.Epoch != first.Epoch+1 || second.OperationID != first.OperationID {
|
||||
t.Fatalf("续作必须是同一 operation 的新代次:%+v vs %+v", second, first)
|
||||
}
|
||||
// 新代次可以用原窗口释放(owner 仍是同一个作业)。
|
||||
if err := store.ReleaseLibraryMaintenanceGuarded(ctx, "internal", jobID, second, now.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("续作代次释放原窗口失败:%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 编译期确认维护层级常量可用(避免测试里手写魔法字符串与实现漂移)。
|
||||
var _ = librarymaintenance.LevelInstance
|
||||
|
||||
// 迁移 76 的 down/up 幂等:治理库里这一层只加身份表与 current 指针,
|
||||
// 撤下来不能带走任何历史事件(版本行必须留着)。真实 PG 上逐条执行,
|
||||
// 不靠"迁移文件存在"这种静态证据。
|
||||
func TestMigration76DownAndUpKeepsVersionHistory(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-mig76", "sync", now)
|
||||
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: governance.LibraryOperationID("internal", governance.LibraryJobSync, "legacy-op"),
|
||||
Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "s",
|
||||
RequestDigest: "d", OperationKey: "k", Actor: "t", CreatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: "libver-mig76-keep", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
|
||||
SnapshotID: "snap-mig76", SourceJobID: "libsync-mig76", CreatedBy: "worker",
|
||||
Evidence: map[string]any{"backup_snapshot": "snap-mig76"}, CreatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
all, err := loadMigrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var target migration
|
||||
for _, item := range all {
|
||||
if item.Version == 76 {
|
||||
target = item
|
||||
}
|
||||
}
|
||||
if target.Version == 0 {
|
||||
t.Fatal("migration 76 不存在")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, target.DownSQL); err != nil {
|
||||
t.Fatalf("撤销 migration 76 失败:%v", err)
|
||||
}
|
||||
var versions int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_versions WHERE id='libver-mig76-keep'`).Scan(&versions); err != nil {
|
||||
t.Fatalf("撤销后版本历史不可读:%v", err)
|
||||
}
|
||||
if versions != 1 {
|
||||
t.Fatalf("撤销 identity 层不许删版本事件:count=%d", versions)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, target.UpSQL); err != nil {
|
||||
t.Fatalf("重新应用 migration 76 失败:%v", err)
|
||||
}
|
||||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: "libver-mig76-after", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup,
|
||||
SnapshotID: "snap-mig76-b", SourceJobID: "libsync-mig76", CreatedBy: "worker",
|
||||
Evidence: map[string]any{"backup_snapshot": "snap-mig76-b"}, CreatedAt: now.Add(time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("重新应用后写入失败:%v", err)
|
||||
}
|
||||
}
|
||||
@@ -288,18 +288,35 @@ func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.Lib
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := tx.Exec(ctx, `
|
||||
// 事件身份 = (kind, scope, 快照, operation)。同一身份的重复登记只有两种合法形态:
|
||||
// · 同一执行者补登记**新增**的字段(预备份先登记、写集合随后补上);
|
||||
// · 完全没有新字段的纯重放。
|
||||
// 所以冲突时只做**加法**:已有键永远赢(右操作数是已有 evidence),身份字段
|
||||
// (source_job_id/operation_id/created_at)一格都不许改;换了执行者或换了意图的
|
||||
// 登记在这里匹配不到行,什么也不写(F02/F09 不能靠 upsert 改写归属)。
|
||||
// xmax = 0 才是真的新插入:重放同一份证据(或换了执行者被 WHERE 挡住)都不算新事实,
|
||||
// 否则一次纯重放就能把 current 再"推进"一次,把后来的操作顶掉。
|
||||
var inserted bool
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id,
|
||||
operation_id, attempt_epoch, evidence, created_by, created_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
SET evidence = EXCLUDED.evidence || library_versions.evidence
|
||||
WHERE library_versions.source_job_id = EXCLUDED.source_job_id
|
||||
AND library_versions.operation_id = EXCLUDED.operation_id
|
||||
RETURNING (xmax = 0)`,
|
||||
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
|
||||
version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy,
|
||||
version.CreatedAt.UTC())
|
||||
version.CreatedAt.UTC()).Scan(&inserted)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// 同一身份已经存在、但归属是别的执行者/别的意图:不覆盖、不报错,也不推进 current。
|
||||
inserted = false
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("record library version event: %w", err)
|
||||
}
|
||||
inserted := tag.RowsAffected() == 1
|
||||
|
||||
if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified {
|
||||
if err := advanceLibraryCurrent(ctx, tx, version, inserted); err != nil {
|
||||
@@ -311,10 +328,31 @@ ON CONFLICT (id) DO NOTHING`,
|
||||
|
||||
// advanceLibraryCurrent 用单调次序条件推进 current 指针。
|
||||
//
|
||||
// 次序键 = (事件登记时间|operation|epoch)。它只回答"哪一条事实更晚被登记",
|
||||
// 不回答"哪个意图更重要";旧代次能不能写由 epoch 守卫回答,两件事分开才不会互相冒充。
|
||||
// 次序键 = (意图创建时间 | operation | epoch),**不是**事件登记时间(父的复核项):
|
||||
// 拿登记时间排序,一次迟到的旧 operation 重放(例如重跑几天前的同步)就会因为"我刚登记"
|
||||
// 而把 current 推回历史。意图的创建时间不会因为重放而变新,所以旧意图永远推不动
|
||||
// 比它更新的意图。
|
||||
//
|
||||
// 恢复要不要能把 current 指回更早的**内容**?要,但它是通过**新的意图**(kind=recover,
|
||||
// 创建于恢复时刻)做到的:内容更旧、意图更新,两者不矛盾。旧代次能不能写由 epoch
|
||||
// 守卫回答(见 libraryFence),不由时间回答。
|
||||
func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool) error {
|
||||
orderKey := governance.CurrentOrderKey(version.CreatedAt, version.OperationID, version.AttemptEpoch)
|
||||
orderAt := version.CreatedAt
|
||||
if version.OperationID != "" {
|
||||
var created *time.Time
|
||||
err := tx.QueryRow(ctx, `SELECT created_at FROM library_operations WHERE id=$1`,
|
||||
version.OperationID).Scan(&created)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("事件引用的 operation %s 不存在:拒绝推进 current", version.OperationID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("load library operation for current ordering: %w", err)
|
||||
}
|
||||
if created != nil {
|
||||
orderAt = *created
|
||||
}
|
||||
}
|
||||
orderKey := governance.CurrentOrderKey(orderAt, version.OperationID, version.AttemptEpoch)
|
||||
if !inserted {
|
||||
// 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。
|
||||
var currentID string
|
||||
@@ -404,6 +442,47 @@ ORDER BY lj.created_at`)
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// PrepareLibraryJobResume 把**同一个业务作业**放回 pending,供续作/接管重新执行。
|
||||
//
|
||||
// 为什么是原作业而不是新开一行(R3):维护窗口 owner_job_id、freeze journal 文件名、
|
||||
// 预备份与写集合的身份全部按业务作业 id 落在持久层。新开 `-resume` 作业只能继承
|
||||
// operation_id,其它所有者都还指向旧作业——AcquireInstanceMaintenance 会拒绝它,
|
||||
// journal/预备份也查不到,于是"续作"只是一个漂亮的新行。
|
||||
//
|
||||
// 条件更新:只有 failed/cancelled 才允许置回 pending;终态时间与错误清零,
|
||||
// 旧 attempt 行保留(历史不可变)。代次由 BeginLibraryAttempt 在 worker 起手时递增。
|
||||
func (s *Store) PrepareLibraryJobResume(ctx context.Context, jobID string, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
// 续作是人工发起的接管:作业已经 failed/cancelled,它那一代执行者不可能还在合法地写。
|
||||
// 必须**显式结束**那一代(行保留、标 abandoned),否则它的租约还没过期,
|
||||
// BeginLibraryAttempt 会认为"有活跃执行者"而拒绝新代次——续作就永远开不起来。
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume',
|
||||
lease_until=NULL
|
||||
WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2,
|
||||
lease_owner='', lease_until=NULL
|
||||
WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC())
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||||
}
|
||||
job, loadErr := s.LibraryJob(ctx, jobID)
|
||||
if loadErr != nil {
|
||||
return governance.LibraryJob{}, false, loadErr
|
||||
}
|
||||
return job, tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
|
||||
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
|
||||
//
|
||||
@@ -612,10 +691,21 @@ ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID)
|
||||
if err != nil {
|
||||
return governance.LibraryVersion{}, false, err
|
||||
}
|
||||
// 同一快照被多次意图引用时会有多条事件(每次 operation 一条),这是**正常历史**,
|
||||
// 不是台账损坏:按登记时间取最新一条回答"这个快照最近一次登记成什么"。
|
||||
// 谁当前生效不在这里判断,那是 current 指针的事实。
|
||||
if len(found) >= 1 {
|
||||
// 同一快照可以被多次意图引用,每次 operation 一条事件,这是**正常历史**;
|
||||
// 但"两条同身份的行"是台账被写坏(或迁移前没有 operation 身份的未知证据),
|
||||
// 这时挑选任何一条都是替调用方做决定,必须拒绝(未知证据 fail closed)。
|
||||
if len(found) == 1 {
|
||||
return found[0], true, nil
|
||||
}
|
||||
if len(found) >= 2 {
|
||||
if found[0].OperationID == "" || found[1].OperationID == "" {
|
||||
return governance.LibraryVersion{}, false, fmt.Errorf(
|
||||
"快照 %s 在 %s 下有 %d 条无执行身份的记录:拒绝挑选", snapshotID, kind, len(found))
|
||||
}
|
||||
if found[0].OperationID == found[1].OperationID {
|
||||
return governance.LibraryVersion{}, false, fmt.Errorf(
|
||||
"快照 %s 在 %s 下同一意图登记了多行:拒绝挑选", snapshotID, kind)
|
||||
}
|
||||
return found[0], true, nil
|
||||
}
|
||||
return governance.LibraryVersion{}, false, nil
|
||||
|
||||
@@ -2,6 +2,7 @@ package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
@@ -83,45 +84,92 @@ FROM library_operations WHERE id=$1`, id).Scan(&op.ID, &op.Scope, &kind, &op.Tar
|
||||
|
||||
// OperationAnchors 读不可变首次锚点。
|
||||
func (s *Store) OperationAnchors(ctx context.Context, operationID string) (governance.LibraryAnchors, bool, error) {
|
||||
var a governance.LibraryAnchors
|
||||
// 锚点列允许 NULL:必须用可空类型扫描,否则空锚点会让整条读取失败
|
||||
// (真库验证发现:NULL 扫进 *string 直接报错,而内存 fake 永远不会有这个形状)。
|
||||
var (
|
||||
snapshot sql.NullString
|
||||
backupAt sql.NullTime
|
||||
digest sql.NullString
|
||||
fingerprint sql.NullString
|
||||
recordedAt sql.NullTime
|
||||
)
|
||||
var bytes, rows sql.NullInt64
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows,
|
||||
target_fingerprint, anchors_at
|
||||
FROM library_operations WHERE id=$1`, operationID).Scan(&a.FirstBackupSnapshotID, &a.FirstBackupBytes,
|
||||
&a.FirstBackupAt, &a.WriteSetDigest, &a.WriteSetRows, &a.TargetFingerprint, &a.RecordedAt)
|
||||
FROM library_operations WHERE id=$1`, operationID).Scan(&snapshot, &bytes, &backupAt, &digest, &rows,
|
||||
&fingerprint, &recordedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryAnchors{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return governance.LibraryAnchors{}, false, fmt.Errorf("load library operation anchors: %w", err)
|
||||
}
|
||||
a := scanAnchors(snapshot, bytes, backupAt, digest, rows, fingerprint, recordedAt)
|
||||
return a, a.RecordedAt != nil, nil
|
||||
}
|
||||
|
||||
// scanAnchors 是锚点唯一的解码实现:NULL 与零值必须分清(未知 ≠ 0)。
|
||||
func scanAnchors(snapshot sql.NullString, bytes sql.NullInt64, backupAt sql.NullTime, digest sql.NullString,
|
||||
rows sql.NullInt64, fingerprint sql.NullString, recordedAt sql.NullTime) governance.LibraryAnchors {
|
||||
a := governance.LibraryAnchors{FirstBackupSnapshotID: snapshot.String, WriteSetDigest: digest.String,
|
||||
TargetFingerprint: fingerprint.String}
|
||||
if bytes.Valid {
|
||||
value := bytes.Int64
|
||||
a.FirstBackupBytes = &value
|
||||
}
|
||||
if rows.Valid {
|
||||
value := rows.Int64
|
||||
a.WriteSetRows = &value
|
||||
}
|
||||
if backupAt.Valid {
|
||||
value := backupAt.Time
|
||||
a.FirstBackupAt = &value
|
||||
}
|
||||
if recordedAt.Valid {
|
||||
value := recordedAt.Time
|
||||
a.RecordedAt = &value
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// RecordOperationAnchors 只**填空**,不改已存在的锚点。
|
||||
// 接管/重放会用同一份锚点重复调用;如果新值想覆盖旧值,那说明"首次"被重拍了,
|
||||
// 直接报错让调用方停下来(把半写状态当新基线等于毁掉唯一可回滚的那份)。
|
||||
//
|
||||
// 读-改-写必须在同一个事务里并锁住 operation 行(R2 附带项):两个执行者同时"首次"
|
||||
// 登记时,先读后写的实现会各自读到 NULL、各自写入,最后一个静默覆盖前一个——
|
||||
// 首次锚点就不再是首次。这里 FOR UPDATE 串行化,写入后**逐字段回读核对**:
|
||||
// 任何一个不可变字段与本次声称的值不一致就报错(而不是让调用方以为登记成功)。
|
||||
func (s *Store) RecordOperationAnchors(ctx context.Context, operationID string, anchors governance.LibraryAnchors, now time.Time) error {
|
||||
if operationID == "" {
|
||||
return errors.New("record operation anchors requires an operation")
|
||||
}
|
||||
existing, found, err := s.OperationAnchors(ctx, operationID)
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin operation anchors: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var (
|
||||
exSnapshot, exDigest, exFingerprint sql.NullString
|
||||
exBytes, exRows sql.NullInt64
|
||||
exBackupAt, exRecordedAt sql.NullTime
|
||||
)
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows,
|
||||
target_fingerprint, anchors_at
|
||||
FROM library_operations WHERE id=$1 FOR UPDATE`, operationID).Scan(&exSnapshot, &exBytes, &exBackupAt,
|
||||
&exDigest, &exRows, &exFingerprint, &exRecordedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("operation %s 不存在:拒绝登记首次锚点", operationID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("load library operation anchors: %w", err)
|
||||
}
|
||||
existing := scanAnchors(exSnapshot, exBytes, exBackupAt, exDigest, exRows, exFingerprint, exRecordedAt)
|
||||
// 已存在且不同 ⇒ 有人想把"首次"换掉(把半写状态当新基线),必须报错。
|
||||
if err := compareAnchors(operationID, existing, anchors); err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if existing.FirstBackupSnapshotID != "" && anchors.FirstBackupSnapshotID != "" &&
|
||||
existing.FirstBackupSnapshotID != anchors.FirstBackupSnapshotID {
|
||||
return fmt.Errorf("operation %s 的首次预备份是 %s,不能用 %s 覆盖",
|
||||
operationID, existing.FirstBackupSnapshotID, anchors.FirstBackupSnapshotID)
|
||||
}
|
||||
if existing.WriteSetDigest != "" && anchors.WriteSetDigest != "" &&
|
||||
existing.WriteSetDigest != anchors.WriteSetDigest {
|
||||
return fmt.Errorf("operation %s 的首次写集合是 %s,不能用 %s 覆盖",
|
||||
operationID, existing.WriteSetDigest, anchors.WriteSetDigest)
|
||||
}
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_operations SET
|
||||
first_backup_snapshot_id = COALESCE(first_backup_snapshot_id, $2),
|
||||
first_backup_bytes = COALESCE(first_backup_bytes, $3),
|
||||
@@ -132,17 +180,98 @@ UPDATE library_operations SET
|
||||
anchors_at = COALESCE(anchors_at, $8)
|
||||
WHERE id=$1`, operationID, nullIfEmpty(anchors.FirstBackupSnapshotID), anchors.FirstBackupBytes,
|
||||
anchors.FirstBackupAt, nullIfEmpty(anchors.WriteSetDigest), anchors.WriteSetRows,
|
||||
nullIfEmpty(anchors.TargetFingerprint), now.UTC())
|
||||
if err != nil {
|
||||
nullIfEmpty(anchors.TargetFingerprint), now.UTC()); err != nil {
|
||||
return fmt.Errorf("record library operation anchors: %w", err)
|
||||
}
|
||||
// 回读核对:NULL 与空串、nil 与 0 必须分清(真实 PG 才有这个差别)。
|
||||
var (
|
||||
afSnapshot, afDigest, afFingerprint sql.NullString
|
||||
afBytes, afRows sql.NullInt64
|
||||
afBackupAt, afRecordedAt sql.NullTime
|
||||
)
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows,
|
||||
target_fingerprint, anchors_at
|
||||
FROM library_operations WHERE id=$1`, operationID).Scan(&afSnapshot, &afBytes, &afBackupAt, &afDigest,
|
||||
&afRows, &afFingerprint, &afRecordedAt); err != nil {
|
||||
return fmt.Errorf("read back library operation anchors: %w", err)
|
||||
}
|
||||
after := scanAnchors(afSnapshot, afBytes, afBackupAt, afDigest, afRows, afFingerprint, afRecordedAt)
|
||||
if err := verifyAnchorsPersisted(operationID, anchors, after); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// compareAnchors 拒绝"用新值覆盖已登记的首次锚点"。
|
||||
func compareAnchors(operationID string, existing, incoming governance.LibraryAnchors) error {
|
||||
if existing.FirstBackupSnapshotID != "" && incoming.FirstBackupSnapshotID != "" &&
|
||||
existing.FirstBackupSnapshotID != incoming.FirstBackupSnapshotID {
|
||||
return fmt.Errorf("operation %s 的首次预备份是 %s,不能用 %s 覆盖",
|
||||
operationID, existing.FirstBackupSnapshotID, incoming.FirstBackupSnapshotID)
|
||||
}
|
||||
if existing.WriteSetDigest != "" && incoming.WriteSetDigest != "" &&
|
||||
existing.WriteSetDigest != incoming.WriteSetDigest {
|
||||
return fmt.Errorf("operation %s 的首次写集合是 %s,不能用 %s 覆盖",
|
||||
operationID, existing.WriteSetDigest, incoming.WriteSetDigest)
|
||||
}
|
||||
if existing.FirstBackupBytes != nil && incoming.FirstBackupBytes != nil &&
|
||||
*existing.FirstBackupBytes != *incoming.FirstBackupBytes {
|
||||
return fmt.Errorf("operation %s 的首次预备份字节是 %d,不能用 %d 覆盖",
|
||||
operationID, *existing.FirstBackupBytes, *incoming.FirstBackupBytes)
|
||||
}
|
||||
if existing.TargetFingerprint != "" && incoming.TargetFingerprint != "" &&
|
||||
existing.TargetFingerprint != incoming.TargetFingerprint {
|
||||
return fmt.Errorf("operation %s 的目标指纹是 %s,不能用 %s 覆盖",
|
||||
operationID, existing.TargetFingerprint, incoming.TargetFingerprint)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyAnchorsPersisted 核对"本次声称的值确实落库了":空值必须仍是 NULL,
|
||||
// 有值必须逐字段相同。避免"写成功但被 NULL/0 混淆"的静默丢锚点。
|
||||
func verifyAnchorsPersisted(operationID string, claimed, persisted governance.LibraryAnchors) error {
|
||||
check := func(name, want, got string) error {
|
||||
if want != "" && want != got {
|
||||
return fmt.Errorf("operation %s 的 %s 回读为 %q,与声称的 %q 不一致", operationID, name, got, want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := check("first_backup_snapshot_id", claimed.FirstBackupSnapshotID, persisted.FirstBackupSnapshotID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := check("writeset_digest", claimed.WriteSetDigest, persisted.WriteSetDigest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := check("target_fingerprint", claimed.TargetFingerprint, persisted.TargetFingerprint); err != nil {
|
||||
return err
|
||||
}
|
||||
if claimed.FirstBackupBytes != nil &&
|
||||
(persisted.FirstBackupBytes == nil || *persisted.FirstBackupBytes != *claimed.FirstBackupBytes) {
|
||||
return fmt.Errorf("operation %s 的 first_backup_bytes 没有按声称值落库", operationID)
|
||||
}
|
||||
if claimed.WriteSetRows != nil &&
|
||||
(persisted.WriteSetRows == nil || *persisted.WriteSetRows != *claimed.WriteSetRows) {
|
||||
return fmt.Errorf("operation %s 的 writeset_rows 没有按声称值落库", operationID)
|
||||
}
|
||||
if persisted.RecordedAt == nil {
|
||||
return fmt.Errorf("operation %s 的 anchors_at 仍为空:锚点没被真正登记", operationID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BeginLibraryAttempt 取得这个作业的执行权并激活一个新代次。
|
||||
//
|
||||
// 三件事必须在同一个事务里:判租约(谁能执行)→ 递增 epoch(谁是当前代次)→
|
||||
// 写作业行的租约(守卫读它)。分成三步就会出现"两个 worker 都认为自己是当前代次"。
|
||||
// 顺序与理由(R2):
|
||||
// 1. 锁作业行、再锁 **operation 行**:epoch 是 operation 维度的单调量,两个作业同时
|
||||
// 接管同一个 operation 时,只有锁 operation 才能保证"下一个 epoch"不重号。
|
||||
// 2. 同一次投递重复调用 ⇒ **幂等**:返回同一个活跃代次并续租,不新增谱系。
|
||||
// 3. 有别的活跃代次:
|
||||
// - 它的租约仍未过期 ⇒ 拒绝(ErrLibraryAttemptHeld)—— 不能把还在跑的代次踢掉;
|
||||
// - 租约已过期(或就是本作业的旧代次)⇒ 结束它(abandoned,保留行),再开新代次。
|
||||
// 4. 新代次 = max(epoch)+1,**新增一行**(历史不可变);同时把同 operation 其它作业行
|
||||
// 的租约清空,让它们的守卫立刻失效(旧 job 行上的 epoch/owner 字段还会留着,
|
||||
// 所以守卫必须同时看 attempts 的活跃状态,见 libraryFence)。
|
||||
func (s *Store) BeginLibraryAttempt(ctx context.Context, jobID, owner string, lease time.Duration, now time.Time) (governance.LibraryAttempt, error) {
|
||||
if jobID == "" || owner == "" {
|
||||
return governance.LibraryAttempt{}, errors.New("begin library attempt requires job and owner")
|
||||
@@ -157,63 +286,116 @@ func (s *Store) BeginLibraryAttempt(ctx context.Context, jobID, owner string, le
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
var operationID, leaseOwner, queueJobID string
|
||||
var epoch int
|
||||
var leaseUntil *time.Time
|
||||
var jobEpoch int
|
||||
var jobLeaseUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT operation_id, attempt_epoch, lease_owner, lease_until, queue_job_id
|
||||
FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(&operationID, &epoch, &leaseOwner, &leaseUntil, &queueJobID)
|
||||
FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(&operationID, &jobEpoch, &leaseOwner, &jobLeaseUntil, &queueJobID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("library job %s not found", jobID)
|
||||
}
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("load library job for attempt: %w", err)
|
||||
}
|
||||
// 同一 owner 重入是幂等(重试/队列重投);别人持有未过期租约就必须让路。
|
||||
if leaseOwner != "" && leaseOwner != owner && leaseUntil != nil && leaseUntil.After(now.UTC()) {
|
||||
return governance.LibraryAttempt{}, ErrLibraryAttemptHeld
|
||||
until := now.Add(lease).UTC()
|
||||
|
||||
// 旧作业(迁移前没有 operation):没有代次谱系可建,只做租约互斥。
|
||||
if operationID == "" {
|
||||
if leaseOwner != "" && leaseOwner != owner && jobLeaseUntil != nil && jobLeaseUntil.After(now.UTC()) {
|
||||
return governance.LibraryAttempt{}, ErrLibraryAttemptHeld
|
||||
}
|
||||
next := jobEpoch + 1
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`,
|
||||
jobID, next, owner, until, now.UTC()); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("activate legacy library attempt: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("commit legacy library attempt: %w", err)
|
||||
}
|
||||
return governance.LibraryAttempt{JobID: jobID, Epoch: next, QueueJobID: queueJobID, Owner: owner,
|
||||
State: governance.LibraryAttemptRunning, StartedAt: now.UTC(), LeaseUntil: &until}, nil
|
||||
}
|
||||
nextEpoch := epoch + 1
|
||||
if nextEpoch < 1 {
|
||||
nextEpoch = 1
|
||||
|
||||
// epoch 分配必须串行到 operation 粒度。
|
||||
if _, err := tx.Exec(ctx, `SELECT id FROM library_operations WHERE id=$1 FOR UPDATE`, operationID); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("lock library operation for attempt: %w", err)
|
||||
}
|
||||
// 代次必须按 operation 单调递增,而不是按作业行:同一个意图的重试/接管会**新开作业行**,
|
||||
// 若按作业行各自从 1 数起,旧代次(1)与新代次(1)就分不出先后,fencing 失效。
|
||||
if operationID != "" {
|
||||
var maxEpoch *int
|
||||
if err := tx.QueryRow(ctx, `SELECT MAX(epoch) FROM library_attempts WHERE operation_id=$1`,
|
||||
operationID).Scan(&maxEpoch); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("load library attempt epoch: %w", err)
|
||||
|
||||
// 当前活跃代次(部分唯一索引保证至多一行)。
|
||||
var activeEpoch int
|
||||
var activeJob, activeOwner, activeState string
|
||||
var activeUntil *time.Time
|
||||
activeErr := tx.QueryRow(ctx, `
|
||||
SELECT epoch, job_id, owner, state, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running')`, operationID).Scan(
|
||||
&activeEpoch, &activeJob, &activeOwner, &activeState, &activeUntil)
|
||||
switch {
|
||||
case activeErr != nil && !errors.Is(activeErr, pgx.ErrNoRows):
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("load active library attempt: %w", activeErr)
|
||||
case activeErr == nil && activeOwner == owner && activeJob == jobID:
|
||||
// 同一次投递重复调用:续租,返回同一代次(不新增谱系)。
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5 WHERE operation_id=$1 AND epoch=$2 AND owner=$3`,
|
||||
operationID, activeEpoch, owner, now.UTC(), until); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("renew library attempt: %w", err)
|
||||
}
|
||||
next := 1
|
||||
if maxEpoch != nil && *maxEpoch >= next {
|
||||
next = *maxEpoch + 1
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`,
|
||||
jobID, activeEpoch, owner, until, now.UTC()); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("renew library job lease: %w", err)
|
||||
}
|
||||
if nextEpoch < next {
|
||||
nextEpoch = next
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("commit library attempt: %w", err)
|
||||
}
|
||||
return governance.LibraryAttempt{OperationID: operationID, Epoch: activeEpoch, JobID: jobID,
|
||||
QueueJobID: queueJobID, Owner: owner, State: governance.LibraryAttemptRunning,
|
||||
StartedAt: now.UTC(), LeaseUntil: &until}, nil
|
||||
case activeErr == nil && activeUntil != nil && activeUntil.After(now.UTC()):
|
||||
// 别人(另一个作业或另一个执行者)仍持有未过期租约:让路,不许抢占。
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("%w(operation %s 的 %d 代次由 %s/%s 持有)",
|
||||
ErrLibraryAttemptHeld, operationID, activeEpoch, activeJob, activeOwner)
|
||||
case activeErr == nil:
|
||||
// 租约已过期:结束它(行保留),再开新代次。
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_attempts SET state='abandoned', finished_at=$3, outcome=$4, lease_until=NULL
|
||||
WHERE operation_id=$1 AND epoch=$2`, operationID, activeEpoch, now.UTC(),
|
||||
"lease expired: superseded by a new attempt"); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("abandon stale library attempt: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var maxEpoch *int
|
||||
if err := tx.QueryRow(ctx, `SELECT MAX(epoch) FROM library_attempts WHERE operation_id=$1`,
|
||||
operationID).Scan(&maxEpoch); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("load library attempt epoch: %w", err)
|
||||
}
|
||||
next := 1
|
||||
if maxEpoch != nil && *maxEpoch >= next {
|
||||
next = *maxEpoch + 1
|
||||
}
|
||||
attempt := governance.LibraryAttempt{
|
||||
OperationID: operationID, Epoch: nextEpoch, JobID: jobID, QueueJobID: queueJobID,
|
||||
Owner: owner, State: governance.LibraryAttemptRunning, StartedAt: now.UTC(),
|
||||
OperationID: operationID, Epoch: next, JobID: jobID, QueueJobID: queueJobID,
|
||||
Owner: owner, State: governance.LibraryAttemptRunning, StartedAt: now.UTC(), LeaseUntil: &until,
|
||||
}
|
||||
until := now.Add(lease).UTC()
|
||||
attempt.LeaseUntil = &until
|
||||
if operationID != "" {
|
||||
// 一个代次只对应一个作业行:重复 Begin(同 job)不再新增谱系。
|
||||
if _, err := tx.Exec(ctx, `
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO library_attempts (operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until)
|
||||
VALUES ($1,$2,$3,$4,$5,'running',$6,$6,$7)
|
||||
ON CONFLICT (job_id) DO UPDATE SET
|
||||
owner=EXCLUDED.owner, state='running', lease_until=EXCLUDED.lease_until, heartbeat_at=EXCLUDED.heartbeat_at`,
|
||||
operationID, nextEpoch, jobID, queueJobID, owner, now.UTC(), until); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("record library attempt: %w", err)
|
||||
}
|
||||
VALUES ($1,$2,$3,$4,$5,'running',$6,$6,$7)`,
|
||||
operationID, next, jobID, queueJobID, owner, now.UTC(), until); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("record library attempt: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`,
|
||||
jobID, nextEpoch, owner, until, now.UTC()); err != nil {
|
||||
jobID, next, owner, until, now.UTC()); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("activate library attempt: %w", err)
|
||||
}
|
||||
// 同一 operation 的其它作业行(续作留下/旧作业)立刻失去租约:
|
||||
// 它们的守卫写从这一刻起失败,即使行上的 epoch/owner 字段还没被改写。
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET lease_owner='', lease_until=NULL, updated_at=$3
|
||||
WHERE operation_id=$1 AND id<>$2`, operationID, jobID, now.UTC()); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("revoke sibling library job leases: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryAttempt{}, fmt.Errorf("commit library attempt: %w", err)
|
||||
}
|
||||
@@ -226,9 +408,13 @@ func (s *Store) FinishLibraryAttempt(ctx context.Context, attempt governance.Lib
|
||||
if attempt.OperationID == "" {
|
||||
return nil // 旧作业没有 operation:没有代次可结
|
||||
}
|
||||
// 只允许结束**还没结束**的那一代:已经 finished/abandoned 的代次是历史事实,
|
||||
// 迟到的 finish 不能把它的 outcome 改写成"成功"(否则接管后旧执行者一收尾,
|
||||
// 历史就变成了对自己有利的版本,排障时无从分辨谁真的跑完)。
|
||||
// 终态集合与服务端的 state CHECK 一致;`completed` 只允许从活跃态进来一次。
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_attempts SET state=$4, finished_at=$5, outcome=$6, lease_until=NULL
|
||||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3`,
|
||||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state IN ('claimed','running')`,
|
||||
attempt.OperationID, attempt.Epoch, attempt.Owner, string(state), now.UTC(), outcome)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finish library attempt: %w", err)
|
||||
|
||||
@@ -11,15 +11,40 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 写入栅栏:所有"业务状态 / 维护锁"的写都必须证明调用方仍是当前执行代次。
|
||||
// 写入栅栏:所有"业务状态 / 维护锁"的写都必须证明调用方仍是**当前活跃代次**。
|
||||
//
|
||||
// 为什么单独放一份:F13 的根因不是某一条 SQL 写错了,而是"同一个 job_id 被两个代次
|
||||
// 同时使用"这件事本身没有被表达出来。只要有一处更新不带代次条件,失租的旧执行就能
|
||||
// 用同一个 job_id 写业务终态、释放别人刚拿到的维护锁。所以这里给每个可变入口配一个
|
||||
// 带守卫的版本,worker 只准走这些版本;不带守卫的旧方法留给 API(它代表"当前意图",
|
||||
// 没有执行代次)。
|
||||
// 两条条件缺一不可:
|
||||
// 1. 作业行上的 (attempt_epoch, lease_owner) 就是这次的 (epoch, owner);
|
||||
// 2. library_attempts 里那一代次仍是 active(claimed/running)。
|
||||
//
|
||||
// 为什么第 2 条不能省(R2):接管/续跑会激活新一代次,而**旧作业行上的 epoch/owner
|
||||
// 字段仍原样保留**。只比较旧作业行,旧执行器的守卫会继续通过——它就能继续写业务状态、
|
||||
// 释放别人刚拿到的锁。把"该代次仍活跃"作为条件后,旧代次被结束的那一刻起,
|
||||
// 它的所有守卫写都会失败。
|
||||
//
|
||||
// 参数契约(R1):作业 id 取自**被更新行自己的列**(`library_maintenance.owner_job_id`
|
||||
// 等),不写死 `$1`——维护类更新的 `$1` 是 scope/instance,写死 `$1` 会让守卫永远匹配
|
||||
// 不到作业,正常写与释放全部失败。列表达式由调用方传入,序号漂移不可能再发生。
|
||||
|
||||
const epochFence = `EXISTS (SELECT 1 FROM library_jobs j WHERE j.id=$1 AND j.attempt_epoch=$%d AND j.lease_owner=$%d)`
|
||||
// libraryFence 生成"调用方仍是当前活跃代次"的谓词。jobIDExpr 是**列表达式**。
|
||||
func libraryFence(jobIDExpr string, epochParam, ownerParam int) string {
|
||||
return fmt.Sprintf(`EXISTS (
|
||||
SELECT 1 FROM library_jobs j
|
||||
WHERE j.id = %s
|
||||
AND j.attempt_epoch = $%d
|
||||
AND j.lease_owner = $%d
|
||||
AND (
|
||||
j.operation_id = ''
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM library_attempts a
|
||||
WHERE a.operation_id = j.operation_id
|
||||
AND a.epoch = j.attempt_epoch
|
||||
AND a.owner = j.lease_owner
|
||||
AND a.state IN ('claimed','running')
|
||||
)
|
||||
)
|
||||
)`, jobIDExpr, epochParam, ownerParam)
|
||||
}
|
||||
|
||||
// AdvanceLibraryJobGuarded 只在调用方仍是该作业当前代次时写业务状态。
|
||||
func (s *Store) AdvanceLibraryJobGuarded(ctx context.Context, attempt governance.LibraryAttempt,
|
||||
@@ -38,7 +63,7 @@ UPDATE library_jobs SET
|
||||
target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END,
|
||||
updated_at=$8,
|
||||
finished_at=COALESCE($9, finished_at)
|
||||
WHERE id=$1 AND attempt_epoch=$10 AND lease_owner=$11`,
|
||||
WHERE id=$1 AND `+libraryFence("$1", 10, 11),
|
||||
attempt.JobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished,
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
@@ -62,8 +87,7 @@ UPDATE library_job_stages SET
|
||||
bytes_done = COALESCE($7, bytes_done),
|
||||
detail = CASE WHEN $8='' THEN detail ELSE $8 END,
|
||||
updated_at = $9
|
||||
WHERE job_id=$1 AND stage=$2
|
||||
AND EXISTS (SELECT 1 FROM library_jobs j WHERE j.id=$1 AND j.attempt_epoch=$10 AND j.lease_owner=$11)`,
|
||||
WHERE job_id=$1 AND stage=$2 AND `+libraryFence("$1", 10, 11),
|
||||
attempt.JobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC(),
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
@@ -81,7 +105,8 @@ func (s *Store) ReleaseLibraryMaintenanceGuarded(ctx context.Context, scope, own
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3
|
||||
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 4, 5), scope, ownerJobID, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_maintenance.owner_job_id", 4, 5), scope, ownerJobID, now.UTC(),
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("release library maintenance (guarded): %w", err)
|
||||
}
|
||||
@@ -100,7 +125,8 @@ func (s *Store) ReleaseInstanceMaintenanceGuarded(ctx context.Context, instance,
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 4, 5), instance, ownerJobID, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_instance_maintenance.owner_job_id", 4, 5), instance, ownerJobID, now.UTC(),
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("release instance maintenance (guarded): %w", err)
|
||||
}
|
||||
@@ -120,7 +146,8 @@ func (s *Store) UpdateLibraryMaintenanceStateGuarded(ctx context.Context, scope,
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_maintenance SET state=$3, note=$4, updated_at=$5
|
||||
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 6, 7), scope, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_maintenance.owner_job_id", 6, 7), scope, ownerJobID, state, note, now.UTC(),
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update library maintenance (guarded): %w", err)
|
||||
}
|
||||
@@ -138,7 +165,8 @@ func (s *Store) UpdateInstanceMaintenanceStateGuarded(ctx context.Context, insta
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 6, 7), instance, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_instance_maintenance.owner_job_id", 6, 7), instance, ownerJobID, state, note,
|
||||
now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update instance maintenance (guarded): %w", err)
|
||||
}
|
||||
@@ -154,17 +182,22 @@ WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
// fencedWriteError 区分"作业不存在"与"代次已过期":两者都要停下,但文案不能误导运维。
|
||||
func (s *Store) fencedWriteError(ctx context.Context, attempt governance.LibraryAttempt, action string) error {
|
||||
var epoch int
|
||||
var owner string
|
||||
err := s.db.QueryRow(ctx, `SELECT attempt_epoch, lease_owner FROM library_jobs WHERE id=$1`,
|
||||
attempt.JobID).Scan(&epoch, &owner)
|
||||
var owner, operation string
|
||||
err := s.db.QueryRow(ctx, `SELECT attempt_epoch, lease_owner, operation_id FROM library_jobs WHERE id=$1`,
|
||||
attempt.JobID).Scan(&epoch, &owner, &operation)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("%s:作业 %s 不存在", action, attempt.JobID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s:读不到作业代次:%w", action, err)
|
||||
}
|
||||
return fmt.Errorf("%w(%s:本代次 %d/%s,当前 %d/%s)", governance.ErrLibraryEpochStale,
|
||||
action, attempt.Epoch, attempt.Owner, epoch, owner)
|
||||
var state string
|
||||
if operation != "" {
|
||||
_ = s.db.QueryRow(ctx, `SELECT state FROM library_attempts WHERE operation_id=$1 AND epoch=$2`,
|
||||
operation, attempt.Epoch).Scan(&state)
|
||||
}
|
||||
return fmt.Errorf("%w(%s:本代次 %d/%s,作业行 %d/%s,代次状态 %q)", governance.ErrLibraryEpochStale,
|
||||
action, attempt.Epoch, attempt.Owner, epoch, owner, state)
|
||||
}
|
||||
|
||||
func (s *Store) SetInstanceMaintenanceIngressGuarded(ctx context.Context, instance, ownerJobID, state, note string,
|
||||
@@ -175,7 +208,8 @@ func (s *Store) SetInstanceMaintenanceIngressGuarded(ctx context.Context, instan
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 6, 7), instance, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_instance_maintenance.owner_job_id", 6, 7), instance, ownerJobID, state, note,
|
||||
now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update instance maintenance ingress (guarded): %w", err)
|
||||
}
|
||||
@@ -198,7 +232,8 @@ func (s *Store) SetLibraryMaintenanceIngressGuarded(ctx context.Context, scope,
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_maintenance SET ingress_state=$3, note=$4, updated_at=$5
|
||||
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL
|
||||
AND `+fmt.Sprintf(epochFence, 6, 7), scope, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner)
|
||||
AND `+libraryFence("library_maintenance.owner_job_id", 6, 7), scope, ownerJobID, state, note, now.UTC(),
|
||||
attempt.Epoch, attempt.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update library maintenance ingress (guarded): %w", err)
|
||||
}
|
||||
|
||||
@@ -49,8 +49,10 @@ CREATE TABLE IF NOT EXISTS library_attempts (
|
||||
outcome TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (operation_id, epoch)
|
||||
);
|
||||
-- 一个作业行只对应一个代次:重复 Begin 不新增谱系。
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS library_attempts_job ON library_attempts(job_id);
|
||||
-- 一个作业行可以有多条**不可变**代次:接管/续跑本身就是"同一作业的新执行",
|
||||
-- 若按 (job_id) 唯一,第二次接管只能改写旧行 —— 那就不是不可变历史了(R2)。
|
||||
-- 这里只保留普通索引,供"这个作业的所有代次"查询。
|
||||
CREATE INDEX IF NOT EXISTS library_attempts_job ON library_attempts(job_id);
|
||||
-- 同一 operation 同时只能有一个活跃代次:第二个 resumer 的失败不能释放另一个代次的锁。
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS library_attempts_active
|
||||
ON library_attempts(operation_id) WHERE state IN ('claimed','running');
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -25,6 +27,14 @@ func IsolatedDatabaseURL(t *testing.T, databaseURL string) string {
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err)
|
||||
}
|
||||
// 护栏:这些用例会在目标库里建/删 schema。指到业务库上就会在业务库里反复
|
||||
// 建表删表(哪怕只删自己的 schema,也是在业务库上动 DDL)。所以库名必须明确
|
||||
// 是一次性的(含 test),否则要求显式声明,绝不"顺便"跑在业务库上。
|
||||
if databaseName := strings.TrimPrefix(parsed.Path, "/"); !strings.Contains(strings.ToLower(databaseName), "test") &&
|
||||
os.Getenv("KG_ALLOW_NON_TEST_DATABASE_URL") != "1" {
|
||||
t.Fatalf("TEST_DATABASE_URL 指向的库 %q 看起来不是一次性测试库:拒绝在其中建/删 schema"+
|
||||
"(确属一次性库请设 KG_ALLOW_NON_TEST_DATABASE_URL=1)", databaseName)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(t.Name()))
|
||||
schema := "kgtest_" + hex.EncodeToString(digest[:8])
|
||||
ctx := context.Background()
|
||||
@@ -33,6 +43,11 @@ func IsolatedDatabaseURL(t *testing.T, databaseURL string) string {
|
||||
t.Fatalf("connect integration database: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
// 上一次被 kill 的运行可能留下同名 schema(cleanup 没跑到):先清掉自己的,
|
||||
// 否则紧接着的 CREATE 会撞名,整包用例连"跑起来"都做不到。只清本用例自己的名字。
|
||||
if _, err := admin.Exec(ctx, "DROP SCHEMA IF EXISTS "+schema+" CASCADE"); err != nil {
|
||||
t.Fatalf("drop stale isolated schema: %v", err)
|
||||
}
|
||||
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
|
||||
t.Fatalf("create isolated integration schema: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user