package libraryrelease import ( "context" "errors" "strings" "testing" "time" ) type fakeTarget struct { currentErr error applyErr map[Stage]error verifyErr error restoreErr error // 恢复之后校验是否通过:用来区分"恢复跑完"和"恢复已验证"。 verifyPassesAfterRestore bool applied []Stage restored int currentN int verifyN int // 记录最后一次 Verify 是否发生在 Restore 之后(恢复必须被校验兜底)。 verifiedAfterRestore bool } func (f *fakeTarget) Current(context.Context) (CurrentState, error) { f.currentN++ if f.currentErr != nil { return CurrentState{}, f.currentErr } return CurrentState{Revision: "prod-r1", Counts: map[string]int64{"documents": 4}, Digests: map[string]string{"knowledges": "abc"}, CapturedAt: time.Now().UTC()}, nil } func (f *fakeTarget) Apply(_ context.Context, stage Stage, _ string) error { if err, ok := f.applyErr[stage]; ok && err != nil { return err } f.applied = append(f.applied, stage) return nil } func (f *fakeTarget) Verify(context.Context, Expectation) error { f.verifyN++ f.verifiedAfterRestore = f.restored > 0 if f.verifyErr == nil { return nil } if f.verifyPassesAfterRestore && f.restored > 0 { return nil } return f.verifyErr } func (f *fakeTarget) Restore(context.Context, BackupRef) error { f.restored++ return f.restoreErr } type fakeBackups struct { err error ref BackupRef seen int } func (f *fakeBackups) Prepare(context.Context, string, CurrentState) (BackupRef, error) { f.seen++ if f.err != nil { return BackupRef{}, f.err } ref := f.ref if ref.ID == "" { ref = BackupRef{ID: "backup-1", Bytes: 1024, CreatedAt: time.Now().UTC()} } return ref, nil } type stageLog struct{ entries []string } func (s *stageLog) Stage(_ context.Context, stage, state, _ string) { s.entries = append(s.entries, stage+":"+state) } func (s *stageLog) has(entry string) bool { for _, item := range s.entries { if item == entry { return true } } return false } func request(now time.Time) Request { return Request{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1", Now: now, Grant: Grant{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1", ExpiresAt: now.Add(30 * time.Minute), AuthorizedBy: "admin"}} } func TestRunHappyPathWritesAllStagesThenVerifies(t *testing.T) { target := &fakeTarget{} backups := &fakeBackups{} recorder := &stageLog{} now := time.Now() outcome := Run(context.Background(), request(now), target, backups, recorder) if outcome.State != "succeeded" { t.Fatalf("state = %s (%s)", outcome.State, outcome.Error) } if len(target.applied) != 4 || target.restored != 0 { t.Fatalf("applied=%v restored=%d", target.applied, target.restored) } if !recorder.has("verify:succeeded") || recorder.has("rollback:running") { t.Fatalf("stage log = %v", recorder.entries) } } func TestRunRefusesExpiredOrMismatchedGrantBeforeTouchingTarget(t *testing.T) { cases := []struct { name string mutate func(*Request) want error }{ {"expired", func(r *Request) { r.Now = r.Grant.ExpiresAt.Add(time.Minute) }, ErrGrantExpired}, {"other target", func(r *Request) { r.Target = "external-prod" }, ErrGrantTarget}, {"other snapshot", func(r *Request) { r.SnapshotID = "snap-2" }, ErrGrantSnapshot}, {"params changed", func(r *Request) { r.ParamsDigest = "digest-2" }, ErrGrantParams}, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { req := request(time.Now()) testCase.mutate(&req) target := &fakeTarget{} backups := &fakeBackups{} outcome := Run(context.Background(), req, target, backups, nil) if outcome.State != "failed" { t.Fatalf("state = %s, want failed", outcome.State) } if !errors.Is(errors.New(outcome.Error), testCase.want) && !strings.Contains(outcome.Error, testCase.want.Error()) { t.Fatalf("error = %q, want %v", outcome.Error, testCase.want) } if target.currentN != 0 || len(target.applied) != 0 || backups.seen != 0 { t.Fatalf("refused request must not touch production: current=%d applied=%d backups=%d", target.currentN, len(target.applied), backups.seen) } }) } } func TestRunStopsWhenProductionStateUnreadable(t *testing.T) { target := &fakeTarget{currentErr: errors.New("生产不可达")} backups := &fakeBackups{} outcome := Run(context.Background(), request(time.Now()), target, backups, nil) if outcome.State != "failed" || !strings.Contains(outcome.Error, "读取生产当前状态失败") { t.Fatalf("outcome = %+v", outcome) } if backups.seen != 0 || len(target.applied) != 0 { t.Fatal("must not write when production state is unknown") } } func TestRunStopsBeforeWritingWhenBackupFails(t *testing.T) { target := &fakeTarget{} backups := &fakeBackups{err: errors.New("磁盘不足")} recorder := &stageLog{} outcome := Run(context.Background(), request(time.Now()), target, backups, recorder) if outcome.State != "failed" { t.Fatalf("state = %s", outcome.State) } if len(target.applied) != 0 { t.Fatalf("must not write production without a verified backup: %v", target.applied) } if !strings.Contains(outcome.Error, ErrBackupUnavailable.Error()) { t.Fatalf("error must name the backup failure: %q", outcome.Error) } if !recorder.has("prebackup:failed") { t.Fatalf("stage log = %v", recorder.entries) } } func TestRunRollsBackWhenStageFails(t *testing.T) { target := &fakeTarget{applyErr: map[Stage]error{StageIndex: errors.New("索引写入失败")}} recorder := &stageLog{} outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) if outcome.State != "rolled_back" { t.Fatalf("state = %s (%s)", outcome.State, outcome.Error) } if outcome.FailedAt != StageIndex { t.Fatalf("failed_at = %s", outcome.FailedAt) } if target.restored != 1 { t.Fatalf("restore calls = %d, want 1", target.restored) } // wiki 在 index 之后,不应执行 for _, stage := range target.applied { if stage == StageWiki { t.Fatal("later stages must not run after a failure") } } if !recorder.has("rollback:succeeded") { t.Fatalf("stage log = %v", recorder.entries) } } // 恢复跑完但恢复后校验仍失败:不能报 rolled_back。真实事故里页面就是在这种 // 状态下显示"已恢复到预备份并验证",而目标其实还留着 67 行残留。 func TestRunDoesNotClaimRollbackWhenRecoveryVerificationFails(t *testing.T) { target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")}, verifyErr: errors.New("生产分块数不一致")} recorder := &stageLog{} outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) if outcome.State != "rollback_failed" || target.restored != 1 { t.Fatalf("outcome = %+v restored=%d", outcome, target.restored) } if !strings.Contains(outcome.Rollback, "恢复后校验失败") { t.Fatalf("rollback message = %q", outcome.Rollback) } if !recorder.has("rollback:failed") { t.Fatalf("stage log = %v", recorder.entries) } } // 只有"恢复到预备份且恢复后校验通过"才允许报 rolled_back。 func TestRunReportsRolledBackOnlyAfterRecoveryVerifies(t *testing.T) { target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")}, verifyErr: errors.New("半写状态"), verifyPassesAfterRestore: true} recorder := &stageLog{} outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) if outcome.State != "rolled_back" { t.Fatalf("outcome = %+v", outcome) } if !target.verifiedAfterRestore { t.Fatal("restore must be followed by a content verification") } if !recorder.has("rollback:succeeded") { t.Fatalf("stage log = %v", recorder.entries) } } func TestRunReportsRollbackFailureDistinctly(t *testing.T) { target := &fakeTarget{applyErr: map[Stage]error{StageDatabase: errors.New("写库失败")}, restoreErr: errors.New("恢复也失败")} recorder := &stageLog{} outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) if outcome.State != "rollback_failed" { t.Fatalf("state = %s, want rollback_failed", outcome.State) } if !strings.Contains(outcome.Rollback, "恢复失败") { t.Fatalf("rollback message = %q", outcome.Rollback) } if !recorder.has("rollback:failed") { t.Fatalf("stage log = %v", recorder.entries) } } // writeSetPlanTarget 记录"写集合规划"与阶段写入的相对顺序:写集合必须在第一笔之前。 type writeSetPlanTarget struct { *fakeTarget planErr error plans []WriteSetPlan events *[]string ledger *WriteSet restores []BackupRef // adopted 记录"接管时消费了哪份已持久化写集合"(F07):adoptErr 表达"读不懂计划"。 adopted []WriteSet adoptErr error } // AdoptWriteSet 让这个假目标具备"消费既有计划"的能力:真实实现按它把复用决定装回去。 func (p *writeSetPlanTarget) AdoptWriteSet(set WriteSet) error { *p.events = append(*p.events, "adopt") if p.adoptErr != nil { return p.adoptErr } p.adopted = append(p.adopted, set) return nil } func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) { *p.events = append(*p.events, "plan") p.plans = append(p.plans, plan) if p.planErr != nil { return WriteSet{}, p.planErr } set := WriteSet{Version: WriteSetVersion, JobID: plan.JobID, Target: plan.Target, TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(), Universe: plan.Universe, Rows: []WriteSetRow{{Table: "resources", ID: "res-new"}}} if err := set.Seal(); err != nil { return WriteSet{}, err } p.ledger = &set return set, nil } func (p *writeSetPlanTarget) Apply(ctx context.Context, stage Stage, snapshot string) error { *p.events = append(*p.events, "apply:"+string(stage)) return p.fakeTarget.Apply(ctx, stage, snapshot) } func (p *writeSetPlanTarget) Restore(ctx context.Context, backup BackupRef) error { p.restores = append(p.restores, backup) return p.fakeTarget.Restore(ctx, backup) } type writeSetRecorder struct { *stageLog events *[]string err error sets []WriteSet } func (r *writeSetRecorder) WriteSetPlanned(_ context.Context, set WriteSet) error { *r.events = append(*r.events, "observe") r.sets = append(r.sets, set) return r.err } func plannedRun(t *testing.T) (*writeSetPlanTarget, *writeSetRecorder, *[]string, Request, *fakeBackups) { t.Helper() now := time.Now() events := []string{} target := &writeSetPlanTarget{fakeTarget: &fakeTarget{}, events: &events} recorder := &writeSetRecorder{stageLog: &stageLog{}, events: &events} backups := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", Bytes: 512, CreatedAt: now.UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}} request := request(now) return target, recorder, &events, request, backups } func TestRunPlansWriteSetBeforeFirstWrite(t *testing.T) { target, recorder, events, request, backups := plannedRun(t) outcome := Run(context.Background(), request, target, backups, recorder) if outcome.State != "succeeded" { t.Fatalf("outcome = %+v", outcome) } if len(target.plans) != 1 { t.Fatalf("write set must be planned exactly once, got %d", len(target.plans)) } if got := (*events)[0]; got != "plan" { t.Fatalf("write set must be planned before anything touches the target, first event = %s", got) } if len(recorder.sets) != 1 || recorder.sets[0].Digest == "" { t.Fatalf("planner output must be persisted before writing: %+v", recorder.sets) } if target.plans[0].JobID != request.JobID || target.plans[0].TargetSnapshotID != request.SnapshotID || target.plans[0].Target != request.Target { t.Fatalf("plan must bind job/target/snapshot: %+v", target.plans[0]) } if target.plans[0].BackupSnapshotID != "backup-snap" { t.Fatalf("plan must bind the backup snapshot: %+v", target.plans[0]) } if len(*events) < 2 || (*events)[1] != "observe" { t.Fatalf("write set must be persisted right after planning, events=%v", *events) } } func TestRunStopsBeforeWritingWhenWriteSetPlanFails(t *testing.T) { target, recorder, events, request, backups := plannedRun(t) target.planErr = errors.New("资源 ID 冲突:res-x 已属于其它知识库") outcome := Run(context.Background(), request, target, backups, recorder) if outcome.State != "failed" { t.Fatalf("outcome = %+v", outcome) } if !strings.Contains(outcome.Error, "冲突") { t.Fatalf("error must carry the conflict reason: %s", outcome.Error) } if len(target.applied) != 0 { t.Fatalf("no stage may run after a failed preflight: %v", target.applied) } if len(recorder.sets) != 0 { t.Fatal("a failed plan must not be persisted as the write set") } for _, event := range *events { if strings.HasPrefix(event, "apply:") { t.Fatalf("target was written despite a failed preflight: %v", *events) } } } func TestRunStopsBeforeWritingWhenWriteSetCannotBePersisted(t *testing.T) { target, recorder, events, request, backups := plannedRun(t) recorder.err = errors.New("治理库写不进去") outcome := Run(context.Background(), request, target, backups, recorder) if outcome.State != "failed" { t.Fatalf("outcome = %+v", outcome) } if !strings.Contains(outcome.Error, "写集合") { t.Fatalf("error must name the write-set record: %s", outcome.Error) } if len(target.applied) != 0 { t.Fatalf("no stage may run when the write set cannot be persisted: %v", target.applied) } for _, event := range *events { if strings.HasPrefix(event, "apply:") { t.Fatalf("target was written without a persisted write set: %v", *events) } } } // 接管时必须复用持久化的写集合,而不是按当前(已半写的)目标重新规划。 func TestRunResumeUsesPersistedWriteSet(t *testing.T) { target, recorder, events, request, backups := plannedRun(t) resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target, TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap", TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}} backups = resumed target.fakeTarget.applyErr = map[Stage]error{StageResources: errors.New("上传失败")} outcome := Run(context.Background(), request, target, backups, recorder) if outcome.State != "rolled_back" && outcome.State != "rollback_failed" { t.Fatalf("outcome = %+v", outcome) } if len(target.plans) != 0 { t.Fatalf("resume must not re-plan the write set: %+v", target.plans) } // F07:既有计划必须被执行目标消费——否则"复用"只停留在账本上,Apply 仍会重写共享对象。 if len(target.adopted) != 1 { t.Fatalf("接管必须把已持久化的写集合交给执行目标:adopted=%d", len(target.adopted)) } if target.adopted[0].BackupSnapshotID != "backup-snap" { t.Fatalf("消费的写集合绑定了错误的预备份:%+v", target.adopted[0].BackupSnapshotID) } if len(target.restores) != 1 || target.restores[0].WriteSet == nil { t.Fatalf("restore must carry the persisted write set: %+v", target.restores) } for _, event := range *events { if event == "plan" { t.Fatalf("resume re-planned against a half-written target: %v", *events) } } } // F07 反向:会写资源、却读不懂既有计划的执行目标必须**在被写之前**失败。 // 继续写就等于"账本说没动共享对象,实际又 upsert + 重传了一遍"。 func TestRunRefusesPlanBlindTargetOnResume(t *testing.T) { _, recorder, _, request, _ := plannedRun(t) blind := &blindPlanTarget{fakeTarget: &fakeTarget{}} resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target, TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap", TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC()}}} outcome := Run(context.Background(), request, blind, resumed, recorder) if outcome.State != "failed" { t.Fatalf("读不懂计划的执行目标必须失败:%+v", outcome) } if len(blind.applied) != 0 { t.Fatalf("失去复用语义时不许写任何阶段:%v", blind.applied) } } // blindPlanTarget 会规划写集合(因此会写资源),但没有 AdoptWriteSet。 type blindPlanTarget struct{ *fakeTarget } func (b *blindPlanTarget) PlanWriteSet(context.Context, WriteSetPlan) (WriteSet, error) { return WriteSet{Version: WriteSetVersion}, nil }