3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
328 lines
11 KiB
Go
328 lines
11 KiB
Go
package libraryfreeze
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// exitControl 模拟真实 Docker 控制器:停止时报告退出码(0 = 自己优雅退出)。
|
|
type exitControl struct {
|
|
*fixtureControl
|
|
exit map[string]int
|
|
}
|
|
|
|
func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) {
|
|
items, err := c.fixtureControl.Inventory(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range items {
|
|
if code, ok := c.exit[items[i].ID]; ok && !items[i].Running {
|
|
value := code
|
|
items[i].ExitCode = &value
|
|
}
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。
|
|
type idleQuiescer struct{ busy bool }
|
|
|
|
func (q *idleQuiescer) Idle(context.Context) (bool, string, error) {
|
|
if q.busy {
|
|
return false, "asynq pending=1", nil
|
|
}
|
|
return true, "asynq=0 pending_ops=0", nil
|
|
}
|
|
|
|
func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
if err := os.Chmod(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := &exitControl{fixtureControl: &fixtureControl{services: []Service{
|
|
{ID: "app", Running: true, Healthy: true}, {ID: "store", Running: true, Healthy: true}}}, exit: map[string]int{"app": 0, "store": 0}}
|
|
engine, err := Open(root, "target-instance", c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { engine.Close() })
|
|
if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := engine.SetQuiescer(&idleQuiescer{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return root, c, engine
|
|
}
|
|
|
|
// 维护窗口必须真的挡住写入:work 只能在所有写入者确认停下、入口确认关闭之后才开始。
|
|
func TestWindowStopsWritersBeforeWorkAndRestoresAfter(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
var sawStopped, sawClosed bool
|
|
e.ingressClosed = func(context.Context) error { sawClosed = true; return nil }
|
|
result, err := e.RunWindow(context.Background(), "job-1", func(context.Context) error { return nil },
|
|
func(ctx context.Context, w *Window) (string, error) {
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatal("work started while a writer was still running")
|
|
}
|
|
}
|
|
if err := w.MarkContentRisk(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sawStopped = true
|
|
return strings.Repeat("b", 64), nil
|
|
}, func(error) bool { return true })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !sawStopped || !sawClosed {
|
|
t.Fatalf("stopped=%v closed=%v", sawStopped, sawClosed)
|
|
}
|
|
if result.State != "succeeded" {
|
|
t.Fatalf("result=%+v", result)
|
|
}
|
|
j, err := e.load("job-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !j.DrainVerified || !j.IngressClosed || !j.ContentRisk || j.Level != "instance" {
|
|
t.Fatalf("journal evidence missing: %+v", j)
|
|
}
|
|
if strings.Join(c.calls, ",") != "stop:app,stop:store,start:app,start:store" {
|
|
t.Fatalf("calls=%v", c.calls)
|
|
}
|
|
}
|
|
|
|
// 没有优雅退出证据(退出码非 0)时不许开始写目标:在途操作可能被切断。
|
|
func TestWindowRefusesWorkWhenDrainUnverified(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
c.exit["app"] = 143
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-drain", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || !strings.Contains(err.Error(), "排空") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if work != 0 {
|
|
t.Fatal("work ran without drain evidence")
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatal("writers were not restored after drain failure")
|
|
}
|
|
}
|
|
}
|
|
|
|
// 入口停不下来(还在对外服务)就不是维护窗口。
|
|
func TestWindowRefusesWorkWhenIngressStillServing(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
e.ingressClosed = func(context.Context) error { return errors.New("ingress still serving") }
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-ingress", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || work != 0 {
|
|
t.Fatalf("err=%v work=%d", err, work)
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatal("writers were not restored after ingress failure")
|
|
}
|
|
}
|
|
}
|
|
|
|
// 内容回滚失败:服务必须保持停止,且崩溃恢复不许自动把它们起来。
|
|
func TestWindowKeepsWritersStoppedWhenRestoreBlocked(t *testing.T) {
|
|
root, c, e := instanceFixture(t)
|
|
_, err := e.RunWindow(context.Background(), "job-half", func(context.Context) error { return nil },
|
|
func(ctx context.Context, w *Window) (string, error) {
|
|
if err := w.MarkContentRisk(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return "", errors.New("目标写到一半,回滚也失败")
|
|
}, func(error) bool { return false })
|
|
if err == nil {
|
|
t.Fatal("blocked restore reported success")
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatalf("writer %s was restarted on a half-written target", s.ID)
|
|
}
|
|
}
|
|
j, err := e.load("job-half")
|
|
if err != nil || !j.RestoreBlocked || !j.ContentRisk {
|
|
t.Fatalf("journal=%+v err=%v", j, err)
|
|
}
|
|
// 进程重启后:恢复器不能自作主张把半写目标对外开放。
|
|
if err = e.Recover(context.Background()); err == nil {
|
|
t.Fatal("recover reopened a target with unverified content")
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatalf("recover restarted %s", s.ID)
|
|
}
|
|
}
|
|
// 原作业接管:不允许重新 stop(写入者本来就没起来),也不从零重来。
|
|
adopted := 0
|
|
_, err = e.RunWindow(context.Background(), "job-half", func(context.Context) error { t.Fatal("prepare ran on adoption"); return nil },
|
|
func(ctx context.Context, w *Window) (string, error) { adopted++; return "", nil }, func(error) bool { return true })
|
|
if err != nil || adopted != 1 {
|
|
t.Fatalf("adopt err=%v adopted=%d", err, adopted)
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatalf("writer %s was not restored after adoption", s.ID)
|
|
}
|
|
}
|
|
if _, statErr := os.Stat(root + "/job-half.json"); statErr != nil {
|
|
t.Fatal(statErr)
|
|
}
|
|
}
|
|
|
|
// 写到一半进程被杀:journal 必须记住"内容有风险",重启恢复不得放行。
|
|
func TestWindowCrashDuringWriteKeepsTargetHeld(t *testing.T) {
|
|
root, c, e := instanceFixture(t)
|
|
e.ingressClosed = func(context.Context) error { return nil }
|
|
_, err := e.RunWindow(context.Background(), "job-kill", func(context.Context) error { return nil },
|
|
func(ctx context.Context, w *Window) (string, error) {
|
|
if err := w.MarkContentRisk(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return "", errors.New("worker 被 kill")
|
|
}, func(error) bool { return false })
|
|
if err == nil {
|
|
t.Fatal("kill scenario reported success")
|
|
}
|
|
j, err := e.load("job-kill")
|
|
if err != nil || !j.ContentRisk || j.State != "held" {
|
|
t.Fatalf("journal=%+v err=%v", j, err)
|
|
}
|
|
// 重新打开(模拟进程重启):磁盘上的证据必须足以阻止自动开放。
|
|
reopened, err := Open(root, "target-instance", c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
if err = reopened.Recover(context.Background()); err == nil {
|
|
t.Fatal("recovery released a target with unverified content")
|
|
}
|
|
if len(c.calls) != 2 {
|
|
t.Fatalf("calls=%v", c.calls)
|
|
}
|
|
}
|
|
|
|
// 停止途中崩溃:journal 处于 stopping(内容还没风险),接管方可以继续停,
|
|
// 但不需要"重新准备",也不许把没停完的服务当成已停。
|
|
func TestWindowAdoptsStoppingJournal(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
// 手工构造一个"停到一半"的现场:app 已停、store 还在跑。
|
|
e.control.(*exitControl).services[0].Running = false
|
|
j := journal{Version: 1, Source: "target-instance", Job: "job-mid", Level: "instance",
|
|
Ingress: "http://127.0.0.1:1/health", Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
|
|
j.Services = []serviceRecord{
|
|
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true},
|
|
{Service: Service{ID: "store", Running: true, Healthy: true}},
|
|
}
|
|
if err := e.save(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err := e.RunWindow(context.Background(), "job-mid", func(context.Context) error { t.Fatal("prepare replayed"); return nil },
|
|
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Join(c.calls, ",") != "stop:store,start:app,start:store" {
|
|
t.Fatalf("calls=%v", c.calls)
|
|
}
|
|
}
|
|
|
|
// 只锁库的层级不许带入口:那会把"没挡入口"说成"挡了"。
|
|
func TestLibraryLevelRejectsIngressClaim(t *testing.T) {
|
|
root := t.TempDir()
|
|
if err := os.Chmod(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := &fixtureControl{services: []Service{{ID: "app", Running: true, Healthy: true}}}
|
|
e, err := Open(root, "target-instance", c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer e.Close()
|
|
if err = e.SetTarget("library", "http://127.0.0.1:1/health", nil, nil); err == nil {
|
|
t.Fatal("library level accepted an ingress claim")
|
|
}
|
|
if err = e.SetTarget("instance", "", nil, nil); err == nil {
|
|
t.Fatal("instance level accepted an empty ingress")
|
|
}
|
|
if err = e.SetTarget("", "", nil, nil); err == nil {
|
|
t.Fatal("empty level accepted")
|
|
}
|
|
}
|
|
|
|
// fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。
|
|
type fakeGate struct {
|
|
calls *[]string
|
|
drain bool
|
|
active int
|
|
}
|
|
|
|
func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) {
|
|
*g.calls = append(*g.calls, "gate:close")
|
|
return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil
|
|
}
|
|
func (g *fakeGate) OpenIngress(context.Context) error {
|
|
*g.calls = append(*g.calls, "gate:open")
|
|
return nil
|
|
}
|
|
|
|
func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
e.ingressClosed = func(context.Context) error { return nil }
|
|
gate := &fakeGate{calls: &c.calls, drain: true, active: 1}
|
|
if err := e.SetGate(gate); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open"
|
|
if strings.Join(c.calls, ",") != want {
|
|
t.Fatalf("calls=%v want=%s", c.calls, want)
|
|
}
|
|
// 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。
|
|
j, err := e.load("job-gate")
|
|
if err != nil || !j.DrainVerified {
|
|
t.Fatalf("journal=%+v err=%v", j, err)
|
|
}
|
|
// 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。
|
|
if j.Gate == nil || !j.GateReopened {
|
|
t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened)
|
|
}
|
|
}
|
|
|
|
// 入口没排空(还有在途请求)时不许停写者、更不许写目标。
|
|
func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
gate := &fakeGate{calls: &c.calls, drain: false, active: 2}
|
|
if err := e.SetGate(gate); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || work != 0 {
|
|
t.Fatalf("err=%v work=%d", err, work)
|
|
}
|
|
if strings.Join(c.calls, ",") != "gate:close" {
|
|
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
|
|
}
|
|
}
|