Files
knowledge-governance/internal/libraryrelease/release_test.go
T
姚章浩 e0768f2caa
build / build (push) Successful in 1m34s
fix(library): 写集合台账与跨范围冲突预检
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它
知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可
恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例
证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。

改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据"
算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份
版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对
资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。
resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。
整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。

本地验证(已跑):
- make verify 15 项全 success(identity 因本地档案 skipped)
- go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES +
  真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/
  历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复;
  跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净
- SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变
- go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、
  接管不重新规划
2026-10-29 23:55:13 +08:00

412 lines
14 KiB
Go

package libraryrelease
import (
"context"
"errors"
"strings"
"testing"
"time"
)
type fakeTarget struct {
currentErr error
applyErr map[Stage]error
verifyErr error
restoreErr error
// 恢复之后校验是否通过:用来区分"恢复跑完"和"恢复已验证"。
verifyPassesAfterRestore bool
applied []Stage
restored int
currentN int
verifyN int
// 记录最后一次 Verify 是否发生在 Restore 之后(恢复必须被校验兜底)。
verifiedAfterRestore bool
}
func (f *fakeTarget) Current(context.Context) (CurrentState, error) {
f.currentN++
if f.currentErr != nil {
return CurrentState{}, f.currentErr
}
return CurrentState{Revision: "prod-r1", Counts: map[string]int64{"documents": 4},
Digests: map[string]string{"knowledges": "abc"}, CapturedAt: time.Now().UTC()}, nil
}
func (f *fakeTarget) Apply(_ context.Context, stage Stage, _ string) error {
if err, ok := f.applyErr[stage]; ok && err != nil {
return err
}
f.applied = append(f.applied, stage)
return nil
}
func (f *fakeTarget) Verify(context.Context, Expectation) error {
f.verifyN++
f.verifiedAfterRestore = f.restored > 0
if f.verifyErr == nil {
return nil
}
if f.verifyPassesAfterRestore && f.restored > 0 {
return nil
}
return f.verifyErr
}
func (f *fakeTarget) Restore(context.Context, BackupRef) error {
f.restored++
return f.restoreErr
}
type fakeBackups struct {
err error
ref BackupRef
seen int
}
func (f *fakeBackups) Prepare(context.Context, string, CurrentState) (BackupRef, error) {
f.seen++
if f.err != nil {
return BackupRef{}, f.err
}
ref := f.ref
if ref.ID == "" {
ref = BackupRef{ID: "backup-1", Bytes: 1024, CreatedAt: time.Now().UTC()}
}
return ref, nil
}
type stageLog struct{ entries []string }
func (s *stageLog) Stage(_ context.Context, stage, state, _ string) {
s.entries = append(s.entries, stage+":"+state)
}
func (s *stageLog) has(entry string) bool {
for _, item := range s.entries {
if item == entry {
return true
}
}
return false
}
func request(now time.Time) Request {
return Request{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1", Now: now,
Grant: Grant{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1",
ExpiresAt: now.Add(30 * time.Minute), AuthorizedBy: "admin"}}
}
func TestRunHappyPathWritesAllStagesThenVerifies(t *testing.T) {
target := &fakeTarget{}
backups := &fakeBackups{}
recorder := &stageLog{}
now := time.Now()
outcome := Run(context.Background(), request(now), target, backups, recorder)
if outcome.State != "succeeded" {
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
}
if len(target.applied) != 4 || target.restored != 0 {
t.Fatalf("applied=%v restored=%d", target.applied, target.restored)
}
if !recorder.has("verify:succeeded") || recorder.has("rollback:running") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunRefusesExpiredOrMismatchedGrantBeforeTouchingTarget(t *testing.T) {
cases := []struct {
name string
mutate func(*Request)
want error
}{
{"expired", func(r *Request) { r.Now = r.Grant.ExpiresAt.Add(time.Minute) }, ErrGrantExpired},
{"other target", func(r *Request) { r.Target = "external-prod" }, ErrGrantTarget},
{"other snapshot", func(r *Request) { r.SnapshotID = "snap-2" }, ErrGrantSnapshot},
{"params changed", func(r *Request) { r.ParamsDigest = "digest-2" }, ErrGrantParams},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
req := request(time.Now())
testCase.mutate(&req)
target := &fakeTarget{}
backups := &fakeBackups{}
outcome := Run(context.Background(), req, target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("state = %s, want failed", outcome.State)
}
if !errors.Is(errors.New(outcome.Error), testCase.want) && !strings.Contains(outcome.Error, testCase.want.Error()) {
t.Fatalf("error = %q, want %v", outcome.Error, testCase.want)
}
if target.currentN != 0 || len(target.applied) != 0 || backups.seen != 0 {
t.Fatalf("refused request must not touch production: current=%d applied=%d backups=%d",
target.currentN, len(target.applied), backups.seen)
}
})
}
}
func TestRunStopsWhenProductionStateUnreadable(t *testing.T) {
target := &fakeTarget{currentErr: errors.New("生产不可达")}
backups := &fakeBackups{}
outcome := Run(context.Background(), request(time.Now()), target, backups, nil)
if outcome.State != "failed" || !strings.Contains(outcome.Error, "读取生产当前状态失败") {
t.Fatalf("outcome = %+v", outcome)
}
if backups.seen != 0 || len(target.applied) != 0 {
t.Fatal("must not write when production state is unknown")
}
}
func TestRunStopsBeforeWritingWhenBackupFails(t *testing.T) {
target := &fakeTarget{}
backups := &fakeBackups{err: errors.New("磁盘不足")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("state = %s", outcome.State)
}
if len(target.applied) != 0 {
t.Fatalf("must not write production without a verified backup: %v", target.applied)
}
if !strings.Contains(outcome.Error, ErrBackupUnavailable.Error()) {
t.Fatalf("error must name the backup failure: %q", outcome.Error)
}
if !recorder.has("prebackup:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunRollsBackWhenStageFails(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageIndex: errors.New("索引写入失败")}}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rolled_back" {
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
}
if outcome.FailedAt != StageIndex {
t.Fatalf("failed_at = %s", outcome.FailedAt)
}
if target.restored != 1 {
t.Fatalf("restore calls = %d, want 1", target.restored)
}
// wiki 在 index 之后,不应执行
for _, stage := range target.applied {
if stage == StageWiki {
t.Fatal("later stages must not run after a failure")
}
}
if !recorder.has("rollback:succeeded") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// 恢复跑完但恢复后校验仍失败:不能报 rolled_back。真实事故里页面就是在这种
// 状态下显示"已恢复到预备份并验证",而目标其实还留着 67 行残留。
func TestRunDoesNotClaimRollbackWhenRecoveryVerificationFails(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
verifyErr: errors.New("生产分块数不一致")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rollback_failed" || target.restored != 1 {
t.Fatalf("outcome = %+v restored=%d", outcome, target.restored)
}
if !strings.Contains(outcome.Rollback, "恢复后校验失败") {
t.Fatalf("rollback message = %q", outcome.Rollback)
}
if !recorder.has("rollback:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// 只有"恢复到预备份且恢复后校验通过"才允许报 rolled_back。
func TestRunReportsRolledBackOnlyAfterRecoveryVerifies(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
verifyErr: errors.New("半写状态"), verifyPassesAfterRestore: true}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if !target.verifiedAfterRestore {
t.Fatal("restore must be followed by a content verification")
}
if !recorder.has("rollback:succeeded") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunReportsRollbackFailureDistinctly(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageDatabase: errors.New("写库失败")},
restoreErr: errors.New("恢复也失败")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rollback_failed" {
t.Fatalf("state = %s, want rollback_failed", outcome.State)
}
if !strings.Contains(outcome.Rollback, "恢复失败") {
t.Fatalf("rollback message = %q", outcome.Rollback)
}
if !recorder.has("rollback:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// writeSetPlanTarget 记录"写集合规划"与阶段写入的相对顺序:写集合必须在第一笔之前。
type writeSetPlanTarget struct {
*fakeTarget
planErr error
plans []WriteSetPlan
events *[]string
ledger *WriteSet
restores []BackupRef
}
func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) {
*p.events = append(*p.events, "plan")
p.plans = append(p.plans, plan)
if p.planErr != nil {
return WriteSet{}, p.planErr
}
set := WriteSet{Version: WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID,
TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(), Universe: plan.Universe,
Rows: []WriteSetRow{{Table: "resources", ID: "res-new"}}}
if err := set.Seal(); err != nil {
return WriteSet{}, err
}
p.ledger = &set
return set, nil
}
func (p *writeSetPlanTarget) Apply(ctx context.Context, stage Stage, snapshot string) error {
*p.events = append(*p.events, "apply:"+string(stage))
return p.fakeTarget.Apply(ctx, stage, snapshot)
}
func (p *writeSetPlanTarget) Restore(ctx context.Context, backup BackupRef) error {
p.restores = append(p.restores, backup)
return p.fakeTarget.Restore(ctx, backup)
}
type writeSetRecorder struct {
*stageLog
events *[]string
err error
sets []WriteSet
}
func (r *writeSetRecorder) WriteSetPlanned(_ context.Context, set WriteSet) error {
*r.events = append(*r.events, "observe")
r.sets = append(r.sets, set)
return r.err
}
func plannedRun(t *testing.T) (*writeSetPlanTarget, *writeSetRecorder, *[]string, Request, *fakeBackups) {
t.Helper()
now := time.Now()
events := []string{}
target := &writeSetPlanTarget{fakeTarget: &fakeTarget{}, events: &events}
recorder := &writeSetRecorder{stageLog: &stageLog{}, events: &events}
backups := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", Bytes: 512,
CreatedAt: now.UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}
request := request(now)
return target, recorder, &events, request, backups
}
func TestRunPlansWriteSetBeforeFirstWrite(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "succeeded" {
t.Fatalf("outcome = %+v", outcome)
}
if len(target.plans) != 1 {
t.Fatalf("write set must be planned exactly once, got %d", len(target.plans))
}
if got := (*events)[0]; got != "plan" {
t.Fatalf("write set must be planned before anything touches the target, first event = %s", got)
}
if len(recorder.sets) != 1 || recorder.sets[0].Digest == "" {
t.Fatalf("planner output must be persisted before writing: %+v", recorder.sets)
}
if target.plans[0].JobID != request.JobID || target.plans[0].TargetSnapshotID != request.SnapshotID ||
target.plans[0].Target != request.Target {
t.Fatalf("plan must bind job/target/snapshot: %+v", target.plans[0])
}
if target.plans[0].BackupSnapshotID != "backup-snap" {
t.Fatalf("plan must bind the backup snapshot: %+v", target.plans[0])
}
if len(*events) < 2 || (*events)[1] != "observe" {
t.Fatalf("write set must be persisted right after planning, events=%v", *events)
}
}
func TestRunStopsBeforeWritingWhenWriteSetPlanFails(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
target.planErr = errors.New("资源 ID 冲突:res-x 已属于其它知识库")
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if !strings.Contains(outcome.Error, "冲突") {
t.Fatalf("error must carry the conflict reason: %s", outcome.Error)
}
if len(target.applied) != 0 {
t.Fatalf("no stage may run after a failed preflight: %v", target.applied)
}
if len(recorder.sets) != 0 {
t.Fatal("a failed plan must not be persisted as the write set")
}
for _, event := range *events {
if strings.HasPrefix(event, "apply:") {
t.Fatalf("target was written despite a failed preflight: %v", *events)
}
}
}
func TestRunStopsBeforeWritingWhenWriteSetCannotBePersisted(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
recorder.err = errors.New("治理库写不进去")
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if !strings.Contains(outcome.Error, "写集合") {
t.Fatalf("error must name the write-set record: %s", outcome.Error)
}
if len(target.applied) != 0 {
t.Fatalf("no stage may run when the write set cannot be persisted: %v", target.applied)
}
for _, event := range *events {
if strings.HasPrefix(event, "apply:") {
t.Fatalf("target was written without a persisted write set: %v", *events)
}
}
}
// 接管时必须复用持久化的写集合,而不是按当前(已半写的)目标重新规划。
func TestRunResumeUsesPersistedWriteSet(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap",
WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target,
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap",
TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC(),
Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}}
backups = resumed
target.fakeTarget.applyErr = map[Stage]error{StageResources: errors.New("上传失败")}
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "rolled_back" && outcome.State != "rollback_failed" {
t.Fatalf("outcome = %+v", outcome)
}
if len(target.plans) != 0 {
t.Fatalf("resume must not re-plan the write set: %+v", target.plans)
}
if len(target.restores) != 1 || target.restores[0].WriteSet == nil {
t.Fatalf("restore must carry the persisted write set: %+v", target.restores)
}
for _, event := range *events {
if event == "plan" {
t.Fatalf("resume re-planned against a half-written target: %v", *events)
}
}
}