package main import ( "context" "encoding/json" "errors" "os" "path/filepath" "testing" "time" "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/governance" "knowledge-governance/internal/libraryrelease" postgresstore "knowledge-governance/internal/platform/postgres" v071 "knowledge-governance/internal/releaseadapter/v071" ) // 现场(2026-10-30,job librestore-internal-1793321916382871758-retry): // 第一次中断后接管复用了同一份预备份,BackupPrepared 用一份**新造的**证据 // 覆盖了已持久化的 writeset_*,于是 verify_effective 的 writeset_digest 从 // fddd98cf… 变成了空;第二次中断再接管时 loadWriteSetForJob 已经读不回记录。 // 预备份本身没变(快照 id / manifest / 首次 created_at 都不变),丢的只有台账。 // // 这些用例必须跑在真实 PostgreSQL 上:丢证据的机制是 RecordLibraryVersion 的 // `evidence = EXCLUDED.evidence`(整份覆盖),而"同一身份归谁"也是存储层语义 // (id 由 kind+scope+快照决定、upsert 会改写 source_job_id)。假 store 造不出这两件事。 // resumeLedgerFixture 是本组用例的账本夹具:一个作业、一份预备份、一份写集合。 type resumeLedgerFixture struct { pool *pgxpool.Pool scope string jobID string backup string target string params string root string store *postgresstore.Store record libraryStageRecorder writeSet libraryrelease.WriteSet } func newResumeLedgerFixture(t *testing.T, suffix string) *resumeLedgerFixture { t.Helper() store, pool := openLedgerStore(t) ctx := context.Background() now := time.Now().UTC() f := &resumeLedgerFixture{ pool: pool, scope: "internal", jobID: "librestore-internal-ws-" + suffix, backup: "backup-snap-ws-" + suffix, target: "target-snap-ws-" + suffix, params: "params-ws-" + suffix, root: t.TempDir(), store: store, } if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: f.jobID, Scope: f.scope, Kind: governance.LibraryJobRestore, TargetSnapshotID: f.target, RequestDigest: f.params, IdempotencyKey: "library.restore:internal:" + suffix}, governance.SyncStages(), now); err != nil { t.Fatalf("create library job: %v", err) } f.record = libraryStageRecorder{store: store, jobID: f.jobID, scope: f.scope, mappingDigest: "mapping-live"} // 首轮:真实预备份字节(4096)与写集合都按生产顺序落库/落盘。 if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup, SnapshotID: f.backup, Bytes: 4096}); err != nil { t.Fatalf("首轮登记预备份:%v", err) } set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: f.jobID, Target: f.scope, TenantID: 10001, KnowledgeBaseID: "kb-resume", BackupSnapshotID: f.backup, TargetSnapshotID: f.target, CreatedAt: now, Rows: []libraryrelease.WriteSetRow{{Table: "resources", ID: "res-1"}, {Table: "chunks", ID: "chunk-1"}}, Universe: map[string]libraryrelease.TableFingerprint{ "resources": {Count: 1, Digest: "u"}}} if err := set.Seal(); err != nil { t.Fatalf("seal write set: %v", err) } if _, err := v071.PersistWriteSet(f.root, &set); err != nil { t.Fatalf("persist write set: %v", err) } if err := f.record.WriteSetPlanned(ctx, set); err != nil { t.Fatalf("登记写集合:%v", err) } f.writeSet = set return f } // evidenceOf 直接读库,绕过被测代码:证据里到底还剩什么,由这里说了算。 func (f *resumeLedgerFixture) evidenceOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) map[string]any { t.Helper() var raw []byte if err := f.pool.QueryRow(context.Background(), `SELECT evidence FROM library_versions WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`, f.scope, string(kind), f.jobID, snapshot).Scan(&raw); err != nil { t.Fatalf("read %s evidence: %v", kind, err) } var evidence map[string]any if err := json.Unmarshal(raw, &evidence); err != nil { t.Fatalf("decode %s evidence: %v", kind, err) } return evidence } func (f *resumeLedgerFixture) createdAtOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) time.Time { t.Helper() var at time.Time if err := f.pool.QueryRow(context.Background(), `SELECT created_at FROM library_versions WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`, f.scope, string(kind), f.jobID, snapshot).Scan(&at); err != nil { t.Fatalf("read %s created_at: %v", kind, err) } return at.UTC() } // interruptedTarget 模拟"写某一类数据时进程被杀":写入阶段失败,恢复链路照常执行。 type interruptedTarget struct { failAt libraryrelease.Stage applied int } func (t *interruptedTarget) Current(context.Context) (libraryrelease.CurrentState, error) { return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1}, Digests: map[string]string{"resources": "u"}}, nil } func (t *interruptedTarget) Apply(_ context.Context, stage libraryrelease.Stage, _ string) error { t.applied++ if stage == t.failAt { return errors.New("simulated worker kill during apply") } return nil } func (t *interruptedTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil } func (t *interruptedTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil } // completeTarget 是一条能跑完的链路:用来证明最后一次真实恢复仍带着同一份写集合, // 并且生效版本证据里是真实字节(不是复用路径上被抹成 0 的那个)。 type completeTarget struct{ applied int } func (t *completeTarget) Current(context.Context) (libraryrelease.CurrentState, error) { return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1}, Digests: map[string]string{"resources": "u"}}, nil } func (t *completeTarget) Apply(context.Context, libraryrelease.Stage, string) error { t.applied++ return nil } func (t *completeTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil } func (t *completeTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil } // takeoverRequest 是接管时用的发布请求:身份与首轮完全一致。 func (f *resumeLedgerFixture) takeoverRequest(now time.Time) libraryrelease.Request { return libraryrelease.Request{JobID: f.jobID, Target: f.scope, SnapshotID: f.target, ParamsDigest: f.params, Now: now, Grant: libraryrelease.Grant{Target: f.scope, SnapshotID: f.target, ParamsDigest: f.params, ExpiresAt: now.Add(time.Hour), AuthorizedBy: "tester"}} } // takeoverBackups 按生产代码的方式装配接管来源:先按身份取回写集合与真实字节, // 再交给 ledgerBackupStore。缺任何一样都必须失败,不许降级成"按目标现状推断"。 func (f *resumeLedgerFixture) takeoverBackups(t *testing.T, store *postgresstore.Store) libraryrelease.BackupStore { t.Helper() ctx := context.Background() set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup) if err != nil { t.Fatalf("接管读不回写集合(作业 %s,预备份 %s):%v", f.jobID, f.backup, err) } bytes, err := prebackupBytesOf(ctx, store, f.scope, f.jobID, f.backup) if err != nil { t.Fatalf("接管读不回预备份字节:%v", err) } if bytes != 4096 { t.Fatalf("接管拿到的预备份字节 = %d, want 4096(复用不许把真实字节抹成 0)", bytes) } return ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + f.backup, SnapshotID: f.backup, Bytes: bytes}, write: set} } func TestTakeoverKeepsWriteSetEvidenceAcrossRepeatedInterruptions(t *testing.T) { f := newResumeLedgerFixture(t, "repeat") store := f.store ctx := context.Background() firstCreatedAt := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup) // 两次"写到一半被杀 + 接管":每次接管都会复用同一份预备份并重新登记证据。 for round := 1; round <= 2; round++ { backups := f.takeoverBackups(t, store) now := time.Now().UTC() outcome := libraryrelease.Run(ctx, f.takeoverRequest(now), &interruptedTarget{failAt: libraryrelease.Stage("database")}, backups, f.record) if outcome.State != "rolled_back" { t.Fatalf("第 %d 次中断的结局 = %q(%s / %s),want rolled_back", round, outcome.State, outcome.Error, outcome.Rollback) } // 每次中断之后,下一次接管/恢复都必须还能按身份取回写集合。 set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup) if err != nil { t.Fatalf("第 %d 次接管后写集合丢失(正是本缺陷):%v", round, err) } if set.Digest != f.writeSet.Digest { t.Fatalf("第 %d 次接管后写集合摘要 = %s, want %s", round, set.Digest, f.writeSet.Digest) } } // 证据逐项核对:写集合锚点、真实字节、作业/预备份身份都必须还在。 prebackup := f.evidenceOf(t, governance.LibraryVersionProductionPrebackup, f.backup) for key, want := range map[string]any{ "writeset_digest": f.writeSet.Digest, "writeset_rows": float64(2), "job": f.jobID, "backup_snapshot": f.backup, "backup_bytes": float64(4096), "index_mapping_sha256": "mapping-live", "backup_reused": true, } { if got := prebackup[key]; got != want { t.Fatalf("复用后证据 %s = %v, want %v(整份覆盖就是本缺陷)", key, got, want) } } if got := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup); !got.Equal(firstCreatedAt) { t.Fatalf("复用后预备份登记时刻 = %s, want %s(备份什么时候拍的被洗掉了)", got, firstCreatedAt) } // 第三次不中断:完整跑通,证明最后一次真实恢复仍按同一份写集合执行。 final := &completeTarget{} outcome := libraryrelease.Run(ctx, f.takeoverRequest(time.Now().UTC()), final, f.takeoverBackups(t, store), f.record) if outcome.State != "succeeded" { t.Fatalf("最终恢复 state=%q error=%s rollback=%s", outcome.State, outcome.Error, outcome.Rollback) } if final.applied != len(libraryrelease.Stages()) { t.Fatalf("四类写入没有全部执行:applied=%d", final.applied) } // 生效版本证据由处理器用 outcome.Backup 构造(这里不重复实现那一层), // 所以断言的是它拿到的输入:复用路径必须带回真实字节与同一份预备份身份。 if outcome.Backup.Bytes != 4096 { t.Fatalf("复用路径交回的备份字节 = %d, want 4096(处理器会把 0 写进生效版本证据)", outcome.Backup.Bytes) } if !outcome.Backup.Reused || outcome.Backup.SnapshotID != f.backup { t.Fatalf("复用路径交回的预备份身份不对:%+v", outcome.Backup) } effective := effectiveEvidence(outcome.Backup, "restore", "mapping-live") if effective["backup_bytes"] != int64(4096) || effective["backup_snapshot"] != f.backup { t.Fatalf("生效版本证据 = %v, want backup_bytes 4096 + backup_snapshot %s", effective, f.backup) } } // 负例:复用声明与已持久化证据对不上时,必须 fail closed —— 既不许凭空登记, // 也不许借用别的作业/别的预备份的记录。 func TestTakeoverLedgerRefusesForeignOrMissingBackupEvidence(t *testing.T) { f := newResumeLedgerFixture(t, "negatives") store := f.store ctx := context.Background() // 声称复用一份从没登记过的预备份:没有证据可合并 ⇒ 拒绝。 if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-unknown", SnapshotID: "backup-snap-never-registered", Bytes: 1, Reused: true}); err == nil { t.Fatal("复用未登记的预备份必须失败(否则就是把复用洗成新备份)") } // 同一份预备份被别的作业占用:LibraryVersionByRef 必须报身份冲突,而不是当没有。 if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup, SnapshotID: f.backup, Bytes: 4096, Reused: true}); err != nil { t.Fatalf("本方复用同一份预备份应当成功:%v", err) } // 字节数对不上(本次声称 2048,已登记 4096)⇒ 拒绝:宁可保持维护,也不许写错基线。 if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup, SnapshotID: f.backup, Bytes: 2048, Reused: true}); err == nil { t.Fatal("复用预备份字节数不一致时必须失败") } // 写集合被篡改:文件内容改一个字节,摘要就对不上 ⇒ 拒绝恢复。 path := filepath.Join(f.root, "writesets", f.writeSet.Digest+".json") raw, err := os.ReadFile(path) if err != nil { t.Fatalf("read write set file: %v", err) } tampered := []byte(string(raw[:len(raw)-2]) + "X}") // 落盘的写集合是 0444(只读台账):篡改必须先放开权限,这也顺带证明它默认不可写。 if err := os.Chmod(path, 0600); err != nil { t.Fatalf("chmod write set: %v", err) } if err := os.WriteFile(path, tampered, 0600); err != nil { t.Fatalf("tamper write set: %v", err) } if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err == nil { t.Fatal("写集合被篡改后必须拒绝取回") } if err := os.WriteFile(path, raw, 0600); err != nil { t.Fatalf("restore write set: %v", err) } if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err != nil { t.Fatalf("恢复原文件后应能取回:%v", err) } // 换了预备份身份 / 换了作业:都不许借到这份记录。 if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, "backup-snap-other"); err == nil { t.Fatal("不同预备份必须拒绝") } if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, "librestore-internal-ws-other", f.backup); err == nil { t.Fatal("不同作业必须拒绝") } }