Files
knowledge-governance/cmd/worker/library_writeset_test.go
T
姚章浩 d0310761b6 fix(worker): 接管复用预备份不再抹掉写集合证据
BackupPrepared 每轮都新造一份证据,而版本登记是 evidence=EXCLUDED.evidence
整份替换。接管复用时 BackupRef 只带快照 id(没有字节数)、写集合预检又只在
首轮执行,于是已登记的 writeset_digest/writeset_rows 被抹掉、backup_bytes
记成 0;同一作业第二次中断后再接管就取不回写集合,恢复链彻底断开
(2026-10-30T00:58Z 演练 …-1793321916382871758-retry,摘要 fddd98cf70… → 空)。

现在证据按「同一作业 + 同一份预备份」合并进已持久化那一行,身份不符一律失败;
接管路径用 prebackupBytesOf 从首次登记的证据取回真实字节,缺失即拒绝(不再写 0);
生效版本证据统一由 effectiveEvidence 构造;同一作业的写集合只允许落定一次。

验证:真实 PG(一次性库 127.0.0.1:25471)上先复现再修
- 修前红:cmd/worker 两条新集成用例分别停在「第 1 次接管后写集合丢失」
  与「复用未登记的预备份竟然成功」
- 修后绿:两轮中断+接管+第三次完整恢复后 writeset_digest/backup_bytes(4096)/
  job/backup_snapshot/backup_reused 逐项存活,created_at 仍是首次登记时刻
- 反向自检(各自单独变异后复跑,均红):去掉合并、去掉身份核对、
  去掉摘要冲突判定、去掉复用字节硬门禁
- make verify:16 项全部 success(含 tests-go 真跑集成用例)
2026-10-30 09:14:10 +08:00

390 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// 这里验证的是"治理侧的锚点":摘要真的写进版本证据、能按作业+目标+预备份取回,
// 并且任何一步对不上都返回错误(调用方据此不写目标 / 保持维护)。
// 归档文件的读写在 v071 的写集合用例里用真实文件验证。
type fakeVersionStore struct {
versions []governance.LibraryVersion
recorded *governance.LibraryVersion
err error
}
// LibraryVersionByRef 按 scope+kind+作业+快照四个条件精确命中:
// 真实 store 已不再提供"列最近 N 条再筛"这条路,夹具也不能假装还能那样查。
func (f *fakeVersionStore) LibraryVersionByRef(_ context.Context, scope string, kind governance.LibraryVersionKind,
sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) {
if f.err != nil {
return governance.LibraryVersion{}, false, f.err
}
for _, version := range f.versions {
if version.Scope != "" && version.Scope != scope {
continue
}
if version.Kind == kind && version.SnapshotID == snapshotID && version.SourceJobID == sourceJobID {
return version, true, nil
}
}
return governance.LibraryVersion{}, false, nil
}
func (f *fakeVersionStore) RecordLibraryVersion(_ context.Context, version governance.LibraryVersion) error {
f.recorded = &version
return nil
}
func sealedLedger(t *testing.T, digest string) libraryrelease.WriteSet {
t.Helper()
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: "job-1", Target: "internal",
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap",
TargetSnapshotID: "target-snap", CreatedAt: time.Now().UTC(),
Rows: []libraryrelease.WriteSetRow{{Table: "resources", ID: "res-new"}},
Universe: map[string]libraryrelease.TableFingerprint{"resources": {Count: 2, Digest: "d"}}}
if err := set.Seal(); err != nil {
t.Fatal(err)
}
return set
}
func TestWriteSetEvidenceKeepsBackupEvidenceAndAddsAnchor(t *testing.T) {
set := sealedLedger(t, "")
evidence := writeSetEvidence(map[string]any{"backup_bytes": int64(4096), "index_mapping_sha256": "old"}, set, "mapping-1")
if evidence["backup_bytes"] != int64(4096) {
t.Fatalf("原预备份证据必须保留:%v", evidence)
}
if evidence["writeset_digest"] != set.Digest {
t.Fatalf("writeset_digest = %v, want %s", evidence["writeset_digest"], set.Digest)
}
if evidence["writeset_universe_digest"] != set.UniverseDigest() || evidence["writeset_universe_digest"] == "" {
t.Fatalf("整实例证据摘要必须登记:%v", evidence)
}
if evidence["index_mapping_sha256"] != "mapping-1" {
t.Fatalf("当前索引映射必须覆盖旧值:%v", evidence)
}
if evidence["writeset_job"] != nil && evidence["job"] != "job-1" {
t.Fatalf("写集合必须绑定作业:%v", evidence)
}
}
func TestWriteSetDigestLookupRequiresJobTargetAndSnapshotMatch(t *testing.T) {
store := &fakeVersionStore{versions: []governance.LibraryVersion{
{ID: "v1", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: "backup-snap",
SourceJobID: "job-1", Evidence: map[string]any{"writeset_digest": "digest-1"}},
{ID: "v2", Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: "backup-snap",
SourceJobID: "job-2", Evidence: map[string]any{"writeset_digest": "digest-2"}},
}}
digest, err := writeSetDigestOf(context.Background(), store, "internal", "job-1", "backup-snap")
if err != nil || digest != "digest-1" {
t.Fatalf("digest = %q, err = %v", digest, err)
}
// 别的作业的版本不能被拿来用。
if _, err := writeSetDigestOf(context.Background(), store, "internal", "job-3", "backup-snap"); err == nil {
t.Fatal("找不到本作业的版本时必须失败")
}
// 有版本但没登记摘要 → 失败(不许当成"没有写集合"降级)。
missing := &fakeVersionStore{versions: []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionPrebackup,
SnapshotID: "backup-snap", SourceJobID: "job-1", Evidence: map[string]any{}}}}
if _, err := writeSetDigestOf(context.Background(), missing, "internal", "job-1", "backup-snap"); err == nil {
t.Fatal("没有写集合摘要时必须失败")
}
// 读不到版本 → 失败。
if _, err := writeSetDigestOf(context.Background(), &fakeVersionStore{err: errors.New("db down")},
"internal", "job-1", "backup-snap"); err == nil {
t.Fatal("版本查询失败必须向上传递")
}
}
func TestLoadWriteSetForJobChecksBinding(t *testing.T) {
root := t.TempDir()
set := sealedLedger(t, "")
if _, err := v071.PersistWriteSet(root, &set); err != nil {
t.Fatal(err)
}
store := &fakeVersionStore{versions: []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionPrebackup,
SnapshotID: "backup-snap", SourceJobID: "job-1", Evidence: map[string]any{"writeset_digest": set.Digest}}}}
loaded, err := loadWriteSetForJob(context.Background(), store, root, "internal", "job-1", "backup-snap")
if err != nil {
t.Fatalf("load: %v", err)
}
if loaded.Digest != set.Digest {
t.Fatalf("digest = %s", loaded.Digest)
}
// 目标不一致 → 拒绝(同一份记录不能被拿去恢复另一个目标)。
if _, err := loadWriteSetForJob(context.Background(), store, root, "external", "job-1", "backup-snap"); err == nil {
t.Fatal("目标不一致时必须失败")
}
// 作业不一致 → 拒绝。
if _, err := loadWriteSetForJob(context.Background(), store, root, "internal", "job-2", "backup-snap"); err == nil {
t.Fatal("作业不一致时必须失败")
}
// 记录文件被删 → 拒绝。
if err := os.Chmod(filepath.Join(root, "writesets", set.Digest+".json"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(root, "writesets", set.Digest+".json")); err != nil {
t.Fatal(err)
}
if _, err := loadWriteSetForJob(context.Background(), store, root, "internal", "job-1", "backup-snap"); err == nil {
t.Fatal("写集合文件缺失时必须失败")
}
}
// 接管用的预备份来源:没有写集合就不许继续(返回错误,而不是给一个空引用)。
func TestLedgerBackupStoreRefusesWithoutWriteSet(t *testing.T) {
store := ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-1", SnapshotID: "backup-snap"}}
if _, err := store.Prepare(context.Background(), "target-snap", libraryrelease.CurrentState{}); err == nil {
t.Fatal("没有写集合时接管必须失败")
}
set := sealedLedger(t, "")
store.write = &set
// 复用不重做备份:真实字节只能来自首次登记的证据,缺了必须失败(见下一个用例)。
store.ref.Bytes = 4096
ref, err := store.Prepare(context.Background(), "target-snap", libraryrelease.CurrentState{})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if ref.WriteSet == nil || ref.WriteSet.Digest != set.Digest {
t.Fatalf("接管必须带上持久化的写集合:%+v", ref)
}
}
// 包装层必须把"写集合预检"这件事透出去:orchestrator 用的是类型断言,
// 少一个方法就等于静默跳过台账(真实后果:真作业没有 writeset_digest、也没有
// writesets/ 文件,崩溃接管无台账可读)。
type plainTarget struct{ applied int }
func (t *plainTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
return libraryrelease.CurrentState{}, nil
}
func (t *plainTarget) Apply(context.Context, libraryrelease.Stage, string) error {
t.applied++
return nil
}
func (t *plainTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
func (t *plainTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
type planningTarget struct {
plainTarget
planned int
last libraryrelease.WriteSetPlan
}
func (t *planningTarget) PlanWriteSet(_ context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
t.planned++
t.last = plan
return libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: "job-1", Target: "internal"}, nil
}
func TestContentTrackedTargetForwardsWriteSetPlanning(t *testing.T) {
inner := &planningTarget{}
wrapped := &contentTrackedTarget{inner: inner, onApply: func() {}}
planner, ok := libraryrelease.Target(wrapped).(libraryrelease.WriteSetPlanner)
if !ok {
t.Fatal("包装后的目标必须仍然实现 WriteSetPlanner,否则写集合会被静默跳过")
}
if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal",
PriorSnapshots: []libraryrelease.PriorSnapshot{{JobID: "job-0", SnapshotID: "snap-0", Kind: libraryrelease.PriorSnapshotSync}}}); err != nil {
t.Fatalf("plan: %v", err)
}
if inner.planned != 1 {
t.Fatalf("预检必须落到内层目标,planned=%d", inner.planned)
}
// 来源证据也必须透传:丢掉它就会让"恢复保留下来的资源"永远无法证明可复用。
if len(inner.last.PriorSnapshots) != 1 || inner.last.PriorSnapshots[0].SnapshotID != "snap-0" {
t.Fatalf("PriorSyncs 没有透传到内层:%+v", inner.last)
}
// 内层不支持预检 ⇒ 硬错误,不许"没有台账也照写"。
strict := &contentTrackedTarget{inner: &plainTarget{}, onApply: func() {}}
if _, err := strict.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{}); err == nil {
t.Fatal("内层不支持写集合预检时必须报错")
} else if !strings.Contains(err.Error(), "写集合") {
t.Fatalf("错误必须点名写集合:%v", err)
}
}
// 编排路径上的同一个断言:Run 是按 target.(WriteSetPlanner) 决定要不要预检的,
// 所以包装体必须在**真实调用链里**也把预检带上(只测直接调用内层会漏掉这一层)。
type orchestratorTarget struct {
planned int
applied []libraryrelease.Stage
}
func (t *orchestratorTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
return libraryrelease.CurrentState{}, nil
}
func (t *orchestratorTarget) Apply(_ context.Context, stage libraryrelease.Stage, _ string) error {
t.applied = append(t.applied, stage)
return nil
}
func (t *orchestratorTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
func (t *orchestratorTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
func (t *orchestratorTarget) PlanWriteSet(_ context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
t.planned++
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID,
TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(),
Universe: plan.Universe,
Rows: []libraryrelease.WriteSetRow{{Table: "resources", ID: "res-1"}}}
if err := set.Seal(); err != nil {
return libraryrelease.WriteSet{}, err
}
return set, nil
}
type stubBackupStore struct{}
func (stubBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
return libraryrelease.BackupRef{ID: "backup-1", SnapshotID: "backup-snap",
Universe: map[string]libraryrelease.TableFingerprint{"resources": {Count: 1, Digest: "abc"}}}, nil
}
func TestRunPlansWriteSetThroughContentTrackedWrapper(t *testing.T) {
inner := &orchestratorTarget{}
tracked := &contentTrackedTarget{inner: inner, onApply: func() {}}
now := time.Now().UTC()
request := libraryrelease.Request{JobID: "job-1", Target: "internal", SnapshotID: "snap-1",
ParamsDigest: "digest-1", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: "snap-1", ParamsDigest: "digest-1",
ExpiresAt: now.Add(time.Hour), AuthorizedBy: "acceptance"}}
outcome := libraryrelease.Run(context.Background(), request, tracked, stubBackupStore{}, nil)
if outcome.State != "succeeded" {
t.Fatalf("outcome = %+v", outcome)
}
if inner.planned != 1 {
t.Fatalf("编排路径没有经过包装层触发写集合预检,planned=%d", inner.planned)
}
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
t.Fatalf("预检结果必须带回给编排层(恢复要按它清理):%+v", outcome.Backup)
}
if len(inner.applied) != len(libraryrelease.Stages()) {
t.Fatalf("四类写入必须都执行,applied=%v", inner.applied)
}
}
// 复用登记必须把"这一轮"的字段合并进已持久化的证据,而不是换一份新的。
//
// 现场(2026-10-30 演练):BackupPrepared 在接管复用时新造证据,把首次规划写入的
// writeset_* 整段抹掉;verify_effective 的摘要从 fddd98cf… 变成空,第二次中断
// 再接管就读不回写集合。这里把"抹掉"这件事在纯函数层面钉死。
func TestPrebackupEvidenceKeepsWriteSetAnchorOnReuse(t *testing.T) {
existing := map[string]any{"backup_snapshot": "backup-snap", "job": "job-1",
"backup_bytes": float64(4096), "writeset_digest": "digest-1",
"writeset_universe_digest": "universe-1", "writeset_rows": float64(3),
"index_mapping_sha256": "mapping-old"}
evidence, err := prebackupEvidence(existing, libraryrelease.BackupRef{ID: "backup-backup-snap",
SnapshotID: "backup-snap", Bytes: 4096, Reused: true}, "job-1", "mapping-live")
if err != nil {
t.Fatalf("复用合并:%v", err)
}
for key, want := range map[string]any{"writeset_digest": "digest-1", "writeset_universe_digest": "universe-1",
"writeset_rows": float64(3), "backup_bytes": int64(4096), "backup_snapshot": "backup-snap",
"job": "job-1", "backup_reused": true, "index_mapping_sha256": "mapping-live"} {
if got := evidence[key]; got != want {
t.Fatalf("复用后 %s = %v, want %v(整份替换会丢写集合锚点)", key, got, want)
}
}
// 复用但没有 bytes(历史缺陷的调用形状):字节必须从已登记的证据里补回来。
restored, err := prebackupEvidence(existing, libraryrelease.BackupRef{SnapshotID: "backup-snap", Reused: true},
"job-1", "")
if err != nil {
t.Fatalf("复用缺字节时应从证据补齐:%v", err)
}
if restored["backup_bytes"] != int64(4096) {
t.Fatalf("补回来的字节 = %v, want 4096", restored["backup_bytes"])
}
// 负例:身份不符一律失败,不做"看起来对就合"。
foreignJob := map[string]any{"job": "job-2", "writeset_digest": "digest-other"}
if _, err := prebackupEvidence(foreignJob, libraryrelease.BackupRef{SnapshotID: "backup-snap",
Bytes: 4096, Reused: true}, "job-1", ""); err == nil {
t.Fatal("证据属于别的作业时必须失败(否则会把别人的写集合带进来)")
}
foreignSnapshot := map[string]any{"job": "job-1", "backup_snapshot": "backup-other"}
if _, err := prebackupEvidence(foreignSnapshot, libraryrelease.BackupRef{SnapshotID: "backup-snap",
Bytes: 4096, Reused: true}, "job-1", ""); err == nil {
t.Fatal("证据绑定别的预备份时必须失败")
}
// 负例:两份不同的字节数不能被当成同一份备份。
if _, err := prebackupEvidence(existing, libraryrelease.BackupRef{SnapshotID: "backup-snap",
Bytes: 2048, Reused: true}, "job-1", ""); err == nil {
t.Fatal("复用字节数与已登记不一致时必须失败")
}
// 负例:复用却拿不出任何可信字节 → 拒绝(不许写 0)。
if _, err := prebackupEvidence(map[string]any{"job": "job-1"}, libraryrelease.BackupRef{SnapshotID: "backup-snap",
Reused: true}, "job-1", ""); err == nil {
t.Fatal("复用缺少可信字节数时必须失败")
}
}
// 写集合只能落定一次:第二次预检摘要不同 ⇒ 台账不再描述"写了什么" ⇒ 失败。
func TestAnchorWriteSetRefusesConflictingDigestAndForeignEvidence(t *testing.T) {
set := sealedLedger(t, "")
recorded := func(evidence map[string]any) *fakeVersionStore {
return &fakeVersionStore{versions: []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionPrebackup,
SnapshotID: "backup-snap", SourceJobID: "job-1", Evidence: evidence}}}
}
anchor := func(store libraryVersionLedger) error {
return anchorWriteSet(context.Background(), store, "internal", "job-1", "m", set)
}
if err := anchor(recorded(map[string]any{"job": "job-2"})); err == nil {
t.Fatal("版本证据属于别的作业时必须失败")
}
if err := anchor(recorded(map[string]any{"writeset_digest": "another-digest"})); err == nil {
t.Fatal("已登记不同写集合摘要时必须失败")
}
// 正例对照:同一摘要重复登记是幂等的。
ok := recorded(map[string]any{"writeset_digest": set.Digest})
if err := anchor(ok); err != nil {
t.Fatalf("同一份写集合重复登记应当幂等:%v", err)
}
if ok.recorded == nil || ok.recorded.Evidence["writeset_digest"] != set.Digest {
t.Fatalf("登记没有落到证据上:%+v", ok.recorded)
}
}
// 复用没有真实字节时不许继续:0 字节基线在生效版本证据里与"空备份"无法区分。
func TestLedgerBackupStoreRefusesZeroBytes(t *testing.T) {
set := sealedLedger(t, "")
store := ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-1", SnapshotID: "backup-snap"}, write: &set}
if _, err := store.Prepare(context.Background(), "target-snap", libraryrelease.CurrentState{}); err == nil {
t.Fatal("缺少真实字节数时接管必须失败")
}
store.ref.Bytes = 4096
ref, err := store.Prepare(context.Background(), "target-snap", libraryrelease.CurrentState{})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if ref.Bytes != 4096 || !ref.Reused {
t.Fatalf("复用预备份必须带真实字节与 Reused:%+v", ref)
}
}
// 生效版本证据必须带真实字节;没有字节时不许写 0 冒充。
func TestEffectiveEvidenceCarriesRealBytes(t *testing.T) {
withBytes := effectiveEvidence(libraryrelease.BackupRef{SnapshotID: "backup-snap", Bytes: 4096}, "restore", "m")
if withBytes["backup_bytes"] != int64(4096) || withBytes["backup_snapshot"] != "backup-snap" ||
withBytes["action"] != "restore" || withBytes["index_mapping_sha256"] != "m" {
t.Fatalf("生效版本证据 = %v", withBytes)
}
without := effectiveEvidence(libraryrelease.BackupRef{SnapshotID: "backup-snap"}, "restore", "")
if _, ok := without["backup_bytes"]; ok {
t.Fatalf("未知字节数不许写成 0:%v", without)
}
}