Files
knowledge-governance/internal/libraryfreeze/recover_window_test.go
T
姚章浩 3643e9bf82
build / build (push) Successful in 1m20s
fix(library): 维护收尾必须按预备份恢复并校验
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
2026-10-29 00:50:32 +08:00

176 lines
6.0 KiB
Go

package libraryfreeze
import (
"context"
"errors"
"strings"
"testing"
"time"
)
func heldJournal(job string, fingerprint, backup string) journal {
j := journal{Version: 1, Source: "target-instance", Job: job, Level: "instance",
Ingress: "http://127.0.0.1:1/health", TargetFingerprint: fingerprint,
ContentRisk: true, RestoreBlocked: true, BackupSnapshotID: backup,
Result: Result{State: "held"}, StartedAt: time.Now().UTC(),
Gate: &GateEvidence{Closed: true, Drained: true}}
j.Services = []serviceRecord{
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
{Service: Service{ID: "store", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
}
return j
}
func stopAllServices(c *exitControl) {
for i := range c.services {
c.services[i].Running, c.services[i].Healthy = false, false
}
}
// 收尾只认"真的跑过恢复 + 校验",不认调用方自称已恢复。
func TestRecoverWindowRestoresBeforeOpening(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.save(heldJournal("job-rec", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
gate := &fakeGate{calls: &c.calls, drain: true}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
var restoredWith, verifiedWith string
var order []string
restorer := restorerFunc(func(_ context.Context, id string) error {
restoredWith = id
order = append(order, "restore")
return nil
})
verifier := verifierFunc(func(_ context.Context, id string) (string, error) {
verifiedWith = id
order = append(order, "verify")
return "digest-ok", nil
})
if _, err := e.RecoverWindow(context.Background(), "job-rec", restorer, verifier); err != nil {
t.Fatal(err)
}
if restoredWith != strings.Repeat("b", 64) || verifiedWith != restoredWith {
t.Fatalf("restore=%q verify=%q", restoredWith, verifiedWith)
}
if strings.Join(order, ",") != "restore,verify" {
t.Fatalf("order=%v", order)
}
for _, s := range c.services {
if !s.Running || !s.Healthy {
t.Fatalf("service %s not restored after verified recovery", s.ID)
}
}
after, err := e.load("job-rec")
if err != nil || after.State != "recovered" || after.RecoveredBackup != restoredWith ||
after.RecoveryDigest != "digest-ok" || after.RecoveredAt == nil {
t.Fatalf("journal=%+v err=%v", after, err)
}
}
// 校验不通过 = 恢复未成功:服务保持停止、journal 非终态。
func TestRecoverWindowKeepsTargetClosedWhenVerificationFails(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-bad", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if _, err := e.RecoverWindow(context.Background(), "job-bad",
restorerFunc(func(context.Context, string) error { return nil }),
verifierFunc(func(context.Context, string) (string, error) {
return "", errors.New("chunks 内容与预备份不一致")
})); err == nil {
t.Fatal("verification failure reported success")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("service %s was started although content verification failed", s.ID)
}
}
after, err := e.load("job-bad")
if err != nil || terminal(after.State) || !after.RestoreBlocked || after.RecoveredAt != nil {
t.Fatalf("journal=%+v err=%v", after, err)
}
}
// 没有落盘预备份身份的窗口不许"恢复":那等于凭空开放。
func TestRecoverWindowRequiresPersistedBackup(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
j := heldJournal("job-nobak", strings.Repeat("a", 64), "")
if err := e.save(j); err != nil {
t.Fatal(err)
}
called := false
if _, err := e.RecoverWindow(context.Background(), "job-nobak",
restorerFunc(func(context.Context, string) error { called = true; return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil || called {
t.Fatalf("recovery ran without a persisted backup: err=%v called=%v", err, called)
}
for _, s := range c.services {
if s.Running {
t.Fatal("writer reopened without a backup")
}
}
}
// 目标换了(指纹不一致)就不许拿旧窗口收尾。
func TestRecoverWindowRejectsDifferentTarget(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("c", 64)); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-other", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if _, err := e.RecoverWindow(context.Background(), "job-other",
restorerFunc(func(context.Context, string) error { return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
t.Fatal("recovery accepted another target's journal")
}
}
// 后台任务还在跑时不许开窗口(HTTP 排空覆盖不到它们)。
func TestWindowRefusesWhenBackgroundBusy(t *testing.T) {
_, c, e := instanceFixture(t)
if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceTimeout(100 * time.Millisecond); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-busy", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were stopped although the window never opened")
}
}
}
type restorerFunc func(context.Context, string) error
func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(ctx, id) }
type verifierFunc func(context.Context, string) (string, error)
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }