5defe238e0
build / build (push) Successful in 1m34s
真实事故:源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时,归档行被原样 插入目标库,落在来源租户/知识库;资源阶段随即报"生产缺少资源 ... 的存储路径" (资源查询按目标租户过滤,看不见刚写进去的行)。回滚只按目标范围回放,于是页面 报"已恢复到预备份",目标却多出 4 文件/38 分块/21 资源/4 绑定。 改: - 写入与校验共用 retargetRow:tenant_id / knowledge_base_id / 资源定位符(后端、 provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id 统一映射到目标范围;行里的来源身份对不上就失败,宁可不写。 - Verify 增加"本次写集合仍以来源身份残留"判定;Restore 增加整库指纹比对(预备份 时刻 vs 恢复后)与孤儿资源清理,回滚后用 Verify 复核,恢复工具退出 0 不再等于 恢复完成。 - 新增 cmd/libraryresidue:按归档身份精确、可备份、可复核地清理已知的范围外残留。 验证:make verify 全绿;真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段 失败",断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份;正向用例断言 行落在目标范围且资源定位符指向目标存储。
253 lines
8.3 KiB
Go
253 lines
8.3 KiB
Go
package libraryrelease
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type fakeTarget struct {
|
|
currentErr error
|
|
applyErr map[Stage]error
|
|
verifyErr error
|
|
restoreErr error
|
|
// 恢复之后校验是否通过:用来区分"恢复跑完"和"恢复已验证"。
|
|
verifyPassesAfterRestore bool
|
|
applied []Stage
|
|
restored int
|
|
currentN int
|
|
verifyN int
|
|
// 记录最后一次 Verify 是否发生在 Restore 之后(恢复必须被校验兜底)。
|
|
verifiedAfterRestore bool
|
|
}
|
|
|
|
func (f *fakeTarget) Current(context.Context) (CurrentState, error) {
|
|
f.currentN++
|
|
if f.currentErr != nil {
|
|
return CurrentState{}, f.currentErr
|
|
}
|
|
return CurrentState{Revision: "prod-r1", Counts: map[string]int64{"documents": 4},
|
|
Digests: map[string]string{"knowledges": "abc"}, CapturedAt: time.Now().UTC()}, nil
|
|
}
|
|
|
|
func (f *fakeTarget) Apply(_ context.Context, stage Stage, _ string) error {
|
|
if err, ok := f.applyErr[stage]; ok && err != nil {
|
|
return err
|
|
}
|
|
f.applied = append(f.applied, stage)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeTarget) Verify(context.Context, Expectation) error {
|
|
f.verifyN++
|
|
f.verifiedAfterRestore = f.restored > 0
|
|
if f.verifyErr == nil {
|
|
return nil
|
|
}
|
|
if f.verifyPassesAfterRestore && f.restored > 0 {
|
|
return nil
|
|
}
|
|
return f.verifyErr
|
|
}
|
|
|
|
func (f *fakeTarget) Restore(context.Context, BackupRef) error {
|
|
f.restored++
|
|
return f.restoreErr
|
|
}
|
|
|
|
type fakeBackups struct {
|
|
err error
|
|
ref BackupRef
|
|
seen int
|
|
}
|
|
|
|
func (f *fakeBackups) Prepare(context.Context, string, CurrentState) (BackupRef, error) {
|
|
f.seen++
|
|
if f.err != nil {
|
|
return BackupRef{}, f.err
|
|
}
|
|
ref := f.ref
|
|
if ref.ID == "" {
|
|
ref = BackupRef{ID: "backup-1", Bytes: 1024, CreatedAt: time.Now().UTC()}
|
|
}
|
|
return ref, nil
|
|
}
|
|
|
|
type stageLog struct{ entries []string }
|
|
|
|
func (s *stageLog) Stage(_ context.Context, stage, state, _ string) {
|
|
s.entries = append(s.entries, stage+":"+state)
|
|
}
|
|
|
|
func (s *stageLog) has(entry string) bool {
|
|
for _, item := range s.entries {
|
|
if item == entry {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func request(now time.Time) Request {
|
|
return Request{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1", Now: now,
|
|
Grant: Grant{Target: "internal-prod", SnapshotID: "snap-1", ParamsDigest: "digest-1",
|
|
ExpiresAt: now.Add(30 * time.Minute), AuthorizedBy: "admin"}}
|
|
}
|
|
|
|
func TestRunHappyPathWritesAllStagesThenVerifies(t *testing.T) {
|
|
target := &fakeTarget{}
|
|
backups := &fakeBackups{}
|
|
recorder := &stageLog{}
|
|
now := time.Now()
|
|
outcome := Run(context.Background(), request(now), target, backups, recorder)
|
|
if outcome.State != "succeeded" {
|
|
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
|
|
}
|
|
if len(target.applied) != 4 || target.restored != 0 {
|
|
t.Fatalf("applied=%v restored=%d", target.applied, target.restored)
|
|
}
|
|
if !recorder.has("verify:succeeded") || recorder.has("rollback:running") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|
|
|
|
func TestRunRefusesExpiredOrMismatchedGrantBeforeTouchingTarget(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
mutate func(*Request)
|
|
want error
|
|
}{
|
|
{"expired", func(r *Request) { r.Now = r.Grant.ExpiresAt.Add(time.Minute) }, ErrGrantExpired},
|
|
{"other target", func(r *Request) { r.Target = "external-prod" }, ErrGrantTarget},
|
|
{"other snapshot", func(r *Request) { r.SnapshotID = "snap-2" }, ErrGrantSnapshot},
|
|
{"params changed", func(r *Request) { r.ParamsDigest = "digest-2" }, ErrGrantParams},
|
|
}
|
|
for _, testCase := range cases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
req := request(time.Now())
|
|
testCase.mutate(&req)
|
|
target := &fakeTarget{}
|
|
backups := &fakeBackups{}
|
|
outcome := Run(context.Background(), req, target, backups, nil)
|
|
if outcome.State != "failed" {
|
|
t.Fatalf("state = %s, want failed", outcome.State)
|
|
}
|
|
if !errors.Is(errors.New(outcome.Error), testCase.want) && !strings.Contains(outcome.Error, testCase.want.Error()) {
|
|
t.Fatalf("error = %q, want %v", outcome.Error, testCase.want)
|
|
}
|
|
if target.currentN != 0 || len(target.applied) != 0 || backups.seen != 0 {
|
|
t.Fatalf("refused request must not touch production: current=%d applied=%d backups=%d",
|
|
target.currentN, len(target.applied), backups.seen)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunStopsWhenProductionStateUnreadable(t *testing.T) {
|
|
target := &fakeTarget{currentErr: errors.New("生产不可达")}
|
|
backups := &fakeBackups{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, backups, nil)
|
|
if outcome.State != "failed" || !strings.Contains(outcome.Error, "读取生产当前状态失败") {
|
|
t.Fatalf("outcome = %+v", outcome)
|
|
}
|
|
if backups.seen != 0 || len(target.applied) != 0 {
|
|
t.Fatal("must not write when production state is unknown")
|
|
}
|
|
}
|
|
|
|
func TestRunStopsBeforeWritingWhenBackupFails(t *testing.T) {
|
|
target := &fakeTarget{}
|
|
backups := &fakeBackups{err: errors.New("磁盘不足")}
|
|
recorder := &stageLog{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, backups, recorder)
|
|
if outcome.State != "failed" {
|
|
t.Fatalf("state = %s", outcome.State)
|
|
}
|
|
if len(target.applied) != 0 {
|
|
t.Fatalf("must not write production without a verified backup: %v", target.applied)
|
|
}
|
|
if !strings.Contains(outcome.Error, ErrBackupUnavailable.Error()) {
|
|
t.Fatalf("error must name the backup failure: %q", outcome.Error)
|
|
}
|
|
if !recorder.has("prebackup:failed") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|
|
|
|
func TestRunRollsBackWhenStageFails(t *testing.T) {
|
|
target := &fakeTarget{applyErr: map[Stage]error{StageIndex: errors.New("索引写入失败")}}
|
|
recorder := &stageLog{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
|
|
if outcome.State != "rolled_back" {
|
|
t.Fatalf("state = %s (%s)", outcome.State, outcome.Error)
|
|
}
|
|
if outcome.FailedAt != StageIndex {
|
|
t.Fatalf("failed_at = %s", outcome.FailedAt)
|
|
}
|
|
if target.restored != 1 {
|
|
t.Fatalf("restore calls = %d, want 1", target.restored)
|
|
}
|
|
// wiki 在 index 之后,不应执行
|
|
for _, stage := range target.applied {
|
|
if stage == StageWiki {
|
|
t.Fatal("later stages must not run after a failure")
|
|
}
|
|
}
|
|
if !recorder.has("rollback:succeeded") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|
|
|
|
// 恢复跑完但恢复后校验仍失败:不能报 rolled_back。真实事故里页面就是在这种
|
|
// 状态下显示"已恢复到预备份并验证",而目标其实还留着 67 行残留。
|
|
func TestRunDoesNotClaimRollbackWhenRecoveryVerificationFails(t *testing.T) {
|
|
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
|
|
verifyErr: errors.New("生产分块数不一致")}
|
|
recorder := &stageLog{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
|
|
if outcome.State != "rollback_failed" || target.restored != 1 {
|
|
t.Fatalf("outcome = %+v restored=%d", outcome, target.restored)
|
|
}
|
|
if !strings.Contains(outcome.Rollback, "恢复后校验失败") {
|
|
t.Fatalf("rollback message = %q", outcome.Rollback)
|
|
}
|
|
if !recorder.has("rollback:failed") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|
|
|
|
// 只有"恢复到预备份且恢复后校验通过"才允许报 rolled_back。
|
|
func TestRunReportsRolledBackOnlyAfterRecoveryVerifies(t *testing.T) {
|
|
target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")},
|
|
verifyErr: errors.New("半写状态"), verifyPassesAfterRestore: true}
|
|
recorder := &stageLog{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
|
|
if outcome.State != "rolled_back" {
|
|
t.Fatalf("outcome = %+v", outcome)
|
|
}
|
|
if !target.verifiedAfterRestore {
|
|
t.Fatal("restore must be followed by a content verification")
|
|
}
|
|
if !recorder.has("rollback:succeeded") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|
|
|
|
func TestRunReportsRollbackFailureDistinctly(t *testing.T) {
|
|
target := &fakeTarget{applyErr: map[Stage]error{StageDatabase: errors.New("写库失败")},
|
|
restoreErr: errors.New("恢复也失败")}
|
|
recorder := &stageLog{}
|
|
outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder)
|
|
if outcome.State != "rollback_failed" {
|
|
t.Fatalf("state = %s, want rollback_failed", outcome.State)
|
|
}
|
|
if !strings.Contains(outcome.Rollback, "恢复失败") {
|
|
t.Fatalf("rollback message = %q", outcome.Rollback)
|
|
}
|
|
if !recorder.has("rollback:failed") {
|
|
t.Fatalf("stage log = %v", recorder.entries)
|
|
}
|
|
}
|