3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
176 lines
6.0 KiB
Go
176 lines
6.0 KiB
Go
package libraryfreeze
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func heldJournal(job string, fingerprint, backup string) journal {
|
|
j := journal{Version: 1, Source: "target-instance", Job: job, Level: "instance",
|
|
Ingress: "http://127.0.0.1:1/health", TargetFingerprint: fingerprint,
|
|
ContentRisk: true, RestoreBlocked: true, BackupSnapshotID: backup,
|
|
Result: Result{State: "held"}, StartedAt: time.Now().UTC(),
|
|
Gate: &GateEvidence{Closed: true, Drained: true}}
|
|
j.Services = []serviceRecord{
|
|
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
|
|
{Service: Service{ID: "store", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
|
|
}
|
|
return j
|
|
}
|
|
|
|
func stopAllServices(c *exitControl) {
|
|
for i := range c.services {
|
|
c.services[i].Running, c.services[i].Healthy = false, false
|
|
}
|
|
}
|
|
|
|
// 收尾只认"真的跑过恢复 + 校验",不认调用方自称已恢复。
|
|
func TestRecoverWindowRestoresBeforeOpening(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
if err := e.save(heldJournal("job-rec", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gate := &fakeGate{calls: &c.calls, drain: true}
|
|
if err := e.SetGate(gate); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var restoredWith, verifiedWith string
|
|
var order []string
|
|
restorer := restorerFunc(func(_ context.Context, id string) error {
|
|
restoredWith = id
|
|
order = append(order, "restore")
|
|
return nil
|
|
})
|
|
verifier := verifierFunc(func(_ context.Context, id string) (string, error) {
|
|
verifiedWith = id
|
|
order = append(order, "verify")
|
|
return "digest-ok", nil
|
|
})
|
|
|
|
if _, err := e.RecoverWindow(context.Background(), "job-rec", restorer, verifier); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if restoredWith != strings.Repeat("b", 64) || verifiedWith != restoredWith {
|
|
t.Fatalf("restore=%q verify=%q", restoredWith, verifiedWith)
|
|
}
|
|
if strings.Join(order, ",") != "restore,verify" {
|
|
t.Fatalf("order=%v", order)
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running || !s.Healthy {
|
|
t.Fatalf("service %s not restored after verified recovery", s.ID)
|
|
}
|
|
}
|
|
after, err := e.load("job-rec")
|
|
if err != nil || after.State != "recovered" || after.RecoveredBackup != restoredWith ||
|
|
after.RecoveryDigest != "digest-ok" || after.RecoveredAt == nil {
|
|
t.Fatalf("journal=%+v err=%v", after, err)
|
|
}
|
|
}
|
|
|
|
// 校验不通过 = 恢复未成功:服务保持停止、journal 非终态。
|
|
func TestRecoverWindowKeepsTargetClosedWhenVerificationFails(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.save(heldJournal("job-bad", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := e.RecoverWindow(context.Background(), "job-bad",
|
|
restorerFunc(func(context.Context, string) error { return nil }),
|
|
verifierFunc(func(context.Context, string) (string, error) {
|
|
return "", errors.New("chunks 内容与预备份不一致")
|
|
})); err == nil {
|
|
t.Fatal("verification failure reported success")
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatalf("service %s was started although content verification failed", s.ID)
|
|
}
|
|
}
|
|
after, err := e.load("job-bad")
|
|
if err != nil || terminal(after.State) || !after.RestoreBlocked || after.RecoveredAt != nil {
|
|
t.Fatalf("journal=%+v err=%v", after, err)
|
|
}
|
|
}
|
|
|
|
// 没有落盘预备份身份的窗口不许"恢复":那等于凭空开放。
|
|
func TestRecoverWindowRequiresPersistedBackup(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
j := heldJournal("job-nobak", strings.Repeat("a", 64), "")
|
|
if err := e.save(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
called := false
|
|
if _, err := e.RecoverWindow(context.Background(), "job-nobak",
|
|
restorerFunc(func(context.Context, string) error { called = true; return nil }),
|
|
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil || called {
|
|
t.Fatalf("recovery ran without a persisted backup: err=%v called=%v", err, called)
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatal("writer reopened without a backup")
|
|
}
|
|
}
|
|
}
|
|
|
|
// 目标换了(指纹不一致)就不许拿旧窗口收尾。
|
|
func TestRecoverWindowRejectsDifferentTarget(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
if err := e.SetTargetFingerprint(strings.Repeat("c", 64)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.save(heldJournal("job-other", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := e.RecoverWindow(context.Background(), "job-other",
|
|
restorerFunc(func(context.Context, string) error { return nil }),
|
|
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
|
|
t.Fatal("recovery accepted another target's journal")
|
|
}
|
|
}
|
|
|
|
// 后台任务还在跑时不许开窗口(HTTP 排空覆盖不到它们)。
|
|
func TestWindowRefusesWhenBackgroundBusy(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceTimeout(100 * time.Millisecond); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-busy", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || work != 0 {
|
|
t.Fatalf("err=%v work=%d", err, work)
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatal("writers were stopped although the window never opened")
|
|
}
|
|
}
|
|
}
|
|
|
|
type restorerFunc func(context.Context, string) error
|
|
|
|
func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(ctx, id) }
|
|
|
|
type verifierFunc func(context.Context, string) (string, error)
|
|
|
|
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }
|