Files
knowledge-governance/internal/libraryrelease/release_test.go
T
姚章浩 5defe238e0
build / build (push) Successful in 1m34s
fix(library): 跨范围同步按目标范围改写并校验残留
真实事故:源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时,归档行被原样
插入目标库,落在来源租户/知识库;资源阶段随即报"生产缺少资源 ... 的存储路径"
(资源查询按目标租户过滤,看不见刚写进去的行)。回滚只按目标范围回放,于是页面
报"已恢复到预备份",目标却多出 4 文件/38 分块/21 资源/4 绑定。

改:
- 写入与校验共用 retargetRow:tenant_id / knowledge_base_id / 资源定位符(后端、
  provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id
  统一映射到目标范围;行里的来源身份对不上就失败,宁可不写。
- Verify 增加"本次写集合仍以来源身份残留"判定;Restore 增加整库指纹比对(预备份
  时刻 vs 恢复后)与孤儿资源清理,回滚后用 Verify 复核,恢复工具退出 0 不再等于
  恢复完成。
- 新增 cmd/libraryresidue:按归档身份精确、可备份、可复核地清理已知的范围外残留。

验证:make verify 全绿;真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段
失败",断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份;正向用例断言
行落在目标范围且资源定位符指向目标存储。
2026-10-29 22:11:59 +08:00

253 lines
8.3 KiB
Go

package libraryrelease
import (
"context"
"errors"
"strings"
"testing"
"time"
)
type fakeTarget struct {
currentErr error
applyErr map[Stage]error
verifyErr error
restoreErr error
// 恢复之后校验是否通过:用来区分"恢复跑完"和"恢复已验证"。
verifyPassesAfterRestore bool
applied []Stage
restored int
currentN int
verifyN int
// 记录最后一次 Verify 是否发生在 Restore 之后(恢复必须被校验兜底)。
verifiedAfterRestore bool
}
func (f *fakeTarget) Current(context.Context) (CurrentState, error) {
f.currentN++
if f.currentErr != nil {
return CurrentState{}, f.currentErr
}
return CurrentState{Revision: "prod-r1", Counts: map[string]int64{"documents": 4},
Digests: map[string]string{"knowledges": "abc"}, CapturedAt: time.Now().UTC()}, nil
}
func (f *fakeTarget) Apply(_ context.Context, stage Stage, _ string) error {
if err, ok := f.applyErr[stage]; ok && err != nil {
return err
}
f.applied = append(f.applied, stage)
return nil
}
func (f *fakeTarget) Verify(context.Context, Expectation) error {
f.verifyN++
f.verifiedAfterRestore = f.restored > 0
if f.verifyErr == nil {
return nil
}
if f.verifyPassesAfterRestore && f.restored > 0 {
return nil
}
return f.verifyErr
}
func (f *fakeTarget) Restore(context.Context, BackupRef) error {
f.restored++
return f.restoreErr
}
type fakeBackups struct {
err error
ref BackupRef
seen int
}
func (f *fakeBackups) Prepare(context.Context, string, CurrentState) (BackupRef, error) {
f.seen++
if f.err != nil {
return BackupRef{}, f.err
}
ref := f.ref
if ref.ID == "" {
ref = BackupRef{ID: "backup-1", Bytes: 1024, CreatedAt: time.Now().UTC()}
}
return ref, nil
}
type stageLog struct{ entries []string }
func (s *stageLog) Stage(_ context.Context, stage, state, _ string) {
s.entries = append(s.entries, stage+":"+state)
}
func (s *stageLog) has(entry string) bool {
for _, item := range s.entries {
if item == entry {
return true
}
}
return false
}
func request(now time.Time) Request {
return Request{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1", Now: now,
Grant: Grant{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1",
ExpiresAt: now.Add(30 * time.Minute), AuthorizedBy: "admin"}}
}
func TestRunHappyPathWritesAllStagesThenVerifies(t *testing.T) {
target := &fakeTarget{}
backups := &fakeBackups{}
recorder := &stageLog{}
now := time.Now()
outcome := Run(context.Background(), request(now), target, backups, recorder)
if outcome.State != "succeeded" {
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
}
if len(target.applied) != 4 || target.restored != 0 {
t.Fatalf("applied=%v restored=%d", target.applied, target.restored)
}
if !recorder.has("verify:succeeded") || recorder.has("rollback:running") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunRefusesExpiredOrMismatchedGrantBeforeTouchingTarget(t *testing.T) {
cases := []struct {
name string
mutate func(*Request)
want error
}{
{"expired", func(r *Request) { r.Now = r.Grant.ExpiresAt.Add(time.Minute) }, ErrGrantExpired},
{"other target", func(r *Request) { r.Target = "external-prod" }, ErrGrantTarget},
{"other snapshot", func(r *Request) { r.SnapshotID = "snap-2" }, ErrGrantSnapshot},
{"params changed", func(r *Request) { r.ParamsDigest = "digest-2" }, ErrGrantParams},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
req := request(time.Now())
testCase.mutate(&req)
target := &fakeTarget{}
backups := &fakeBackups{}
outcome := Run(context.Background(), req, target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("state = %s, want failed", outcome.State)
}
if !errors.Is(errors.New(outcome.Error), testCase.want) && !strings.Contains(outcome.Error, testCase.want.Error()) {
t.Fatalf("error = %q, want %v", outcome.Error, testCase.want)
}
if target.currentN != 0 || len(target.applied) != 0 || backups.seen != 0 {
t.Fatalf("refused request must not touch production: current=%d applied=%d backups=%d",
target.currentN, len(target.applied), backups.seen)
}
})
}
}
func TestRunStopsWhenProductionStateUnreadable(t *testing.T) {
target := &fakeTarget{currentErr: errors.New("生产不可达")}
backups := &fakeBackups{}
outcome := Run(context.Background(), request(time.Now()), target, backups, nil)
if outcome.State != "failed" || !strings.Contains(outcome.Error, "读取生产当前状态失败") {
t.Fatalf("outcome = %+v", outcome)
}
if backups.seen != 0 || len(target.applied) != 0 {
t.Fatal("must not write when production state is unknown")
}
}
func TestRunStopsBeforeWritingWhenBackupFails(t *testing.T) {
target := &fakeTarget{}
backups := &fakeBackups{err: errors.New("磁盘不足")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("state = %s", outcome.State)
}
if len(target.applied) != 0 {
t.Fatalf("must not write production without a verified backup: %v", target.applied)
}
if !strings.Contains(outcome.Error, ErrBackupUnavailable.Error()) {
t.Fatalf("error must name the backup failure: %q", outcome.Error)
}
if !recorder.has("prebackup:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunRollsBackWhenStageFails(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageIndex: errors.New("索引写入失败")}}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rolled_back" {
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
}
if outcome.FailedAt != StageIndex {
t.Fatalf("failed_at = %s", outcome.FailedAt)
}
if target.restored != 1 {
t.Fatalf("restore calls = %d, want 1", target.restored)
}
// wiki 在 index 之后,不应执行
for _, stage := range target.applied {
if stage == StageWiki {
t.Fatal("later stages must not run after a failure")
}
}
if !recorder.has("rollback:succeeded") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// 恢复跑完但恢复后校验仍失败:不能报 rolled_back。真实事故里页面就是在这种
// 状态下显示"已恢复到预备份并验证",而目标其实还留着 67 行残留。
func TestRunDoesNotClaimRollbackWhenRecoveryVerificationFails(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
verifyErr: errors.New("生产分块数不一致")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rollback_failed" || target.restored != 1 {
t.Fatalf("outcome = %+v restored=%d", outcome, target.restored)
}
if !strings.Contains(outcome.Rollback, "恢复后校验失败") {
t.Fatalf("rollback message = %q", outcome.Rollback)
}
if !recorder.has("rollback:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// 只有"恢复到预备份且恢复后校验通过"才允许报 rolled_back。
func TestRunReportsRolledBackOnlyAfterRecoveryVerifies(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
verifyErr: errors.New("半写状态"), verifyPassesAfterRestore: true}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if !target.verifiedAfterRestore {
t.Fatal("restore must be followed by a content verification")
}
if !recorder.has("rollback:succeeded") {
t.Fatalf("stage log = %v", recorder.entries)
}
}
func TestRunReportsRollbackFailureDistinctly(t *testing.T) {
target := &fakeTarget{applyErr: map[Stage]error{StageDatabase: errors.New("写库失败")},
restoreErr: errors.New("恢复也失败")}
recorder := &stageLog{}
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
if outcome.State != "rollback_failed" {
t.Fatalf("state = %s, want rollback_failed", outcome.State)
}
if !strings.Contains(outcome.Rollback, "恢复失败") {
t.Fatalf("rollback message = %q", outcome.Rollback)
}
if !recorder.has("rollback:failed") {
t.Fatalf("stage log = %v", recorder.entries)
}
}