package libraryfreeze import ( "context" "errors" "os" "strings" "testing" "time" ) // exitControl 模拟真实 Docker 控制器:停止时报告退出码(0 = 自己优雅退出)。 type exitControl struct { *fixtureControl exit map[string]int } func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) { items, err := c.fixtureControl.Inventory(ctx) if err != nil { return nil, err } for i := range items { if code, ok := c.exit[items[i].ID]; ok && !items[i].Running { value := code items[i].ExitCode = &value } } return items, nil } // idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。 // 它同时实现 StableQuiescer:只有能给出任务集合指纹的检查器才谈得上证明切点。 type idleQuiescer struct{ busy bool } func (q *idleQuiescer) Idle(context.Context) (bool, string, error) { if q.busy { return false, "asynq pending=1", nil } return true, "asynq=0 pending_ops=0", nil } func (q *idleQuiescer) Snapshot(context.Context) (QuiesceSnapshot, error) { if q.busy { return QuiesceSnapshot{Depth: 1, Digest: strings.Repeat("b", 64), Detail: "asynq pending=1"}, nil } return QuiesceSnapshot{Depth: 0, Digest: strings.Repeat("e", 64), Detail: "asynq=0 pending_ops=0"}, nil } func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) { t.Helper() root := t.TempDir() if err := os.Chmod(root, 0700); err != nil { t.Fatal(err) } c := &exitControl{fixtureControl: &fixtureControl{services: []Service{ {ID: "app", Running: true, Healthy: true}, {ID: "store", Running: true, Healthy: true}}}, exit: map[string]int{"app": 0, "store": 0}} engine, err := Open(root, "target-instance", c) if err != nil { t.Fatal(err) } t.Cleanup(func() { engine.Close() }) if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil { t.Fatal(err) } if err := engine.SetQuiescer(&idleQuiescer{}); err != nil { t.Fatal(err) } // 实例级窗口必须有入口闸门:默认装一个会正常排空的假闸门。 if err := engine.SetGate(&fakeGate{calls: &c.calls, drain: true, active: 1}); err != nil { t.Fatal(err) } if err := engine.SetQuiesceStability(5*time.Millisecond, 200*time.Millisecond); err != nil { t.Fatal(err) } return root, c, engine } // 维护窗口必须真的挡住写入:work 只能在所有写入者确认停下、入口确认关闭之后才开始。 func TestWindowStopsWritersBeforeWorkAndRestoresAfter(t *testing.T) { _, c, e := instanceFixture(t) var sawStopped, sawClosed bool e.ingressClosed = func(context.Context) error { sawClosed = true; return nil } result, err := e.RunWindow(context.Background(), "job-1", func(context.Context) error { return nil }, func(ctx context.Context, w *Window) (string, error) { for _, s := range c.services { if s.Running { t.Fatal("work started while a writer was still running") } } if err := w.MarkContentRisk(ctx); err != nil { t.Fatal(err) } sawStopped = true return strings.Repeat("b", 64), nil }, func(error) bool { return true }) if err != nil { t.Fatal(err) } if !sawStopped || !sawClosed { t.Fatalf("stopped=%v closed=%v", sawStopped, sawClosed) } if result.State != "succeeded" { t.Fatalf("result=%+v", result) } j, err := e.load("job-1") if err != nil { t.Fatal(err) } if !j.DrainVerified || !j.IngressClosed || !j.ContentRisk || j.Level != "instance" { t.Fatalf("journal evidence missing: %+v", j) } if strings.Join(c.calls, ",") != "gate:close,stop:app,stop:store,start:app,start:store,gate:open" { t.Fatalf("calls=%v", c.calls) } } // 实例级窗口必须有入口闸门:没有闸门就无法证明"入口已关",后台静默也就没有前提。 // 拒绝发生在动任何容器之前。 func TestInstanceWindowRequiresGate(t *testing.T) { _, c, e := instanceFixture(t) e.gate = nil work := 0 _, err := e.RunWindow(context.Background(), "job-nogate", func(context.Context) error { return nil }, func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) if err == nil || !strings.Contains(err.Error(), "入口闸门") { t.Fatalf("err=%v", err) } if work != 0 { t.Fatal("work ran without an ingress gate") } if len(c.calls) != 0 { t.Fatalf("containers touched without a gate: %v", c.calls) } for _, s := range c.services { if !s.Running { t.Fatal("writers were stopped without a gate") } } } // 入口停不下来(还在对外服务)就不是维护窗口。 func TestWindowRefusesWorkWhenIngressStillServing(t *testing.T) { _, c, e := instanceFixture(t) e.ingressClosed = func(context.Context) error { return errors.New("ingress still serving") } work := 0 _, err := e.RunWindow(context.Background(), "job-ingress", func(context.Context) error { return nil }, func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) if err == nil || work != 0 { t.Fatalf("err=%v work=%d", err, work) } for _, s := range c.services { if !s.Running { t.Fatal("writers were not restored after ingress failure") } } } // 内容回滚失败:服务必须保持停止,且崩溃恢复不许自动把它们起来。 func TestWindowKeepsWritersStoppedWhenRestoreBlocked(t *testing.T) { root, c, e := instanceFixture(t) _, err := e.RunWindow(context.Background(), "job-half", func(context.Context) error { return nil }, func(ctx context.Context, w *Window) (string, error) { if err := w.MarkContentRisk(ctx); err != nil { t.Fatal(err) } return "", errors.New("目标写到一半,回滚也失败") }, func(error) bool { return false }) if err == nil { t.Fatal("blocked restore reported success") } for _, s := range c.services { if s.Running { t.Fatalf("writer %s was restarted on a half-written target", s.ID) } } j, err := e.load("job-half") if err != nil || !j.RestoreBlocked || !j.ContentRisk { t.Fatalf("journal=%+v err=%v", j, err) } // 进程重启后:恢复器不能自作主张把半写目标对外开放。 if err = e.Recover(context.Background()); err == nil { t.Fatal("recover reopened a target with unverified content") } for _, s := range c.services { if s.Running { t.Fatalf("recover restarted %s", s.ID) } } // 原作业接管:不允许重新 stop(写入者本来就没起来),也不从零重来。 adopted := 0 _, err = e.RunWindow(context.Background(), "job-half", func(context.Context) error { t.Fatal("prepare ran on adoption"); return nil }, func(ctx context.Context, w *Window) (string, error) { adopted++; return "", nil }, func(error) bool { return true }) if err != nil || adopted != 1 { t.Fatalf("adopt err=%v adopted=%d", err, adopted) } for _, s := range c.services { if !s.Running { t.Fatalf("writer %s was not restored after adoption", s.ID) } } if _, statErr := os.Stat(root + "/job-half.json"); statErr != nil { t.Fatal(statErr) } } // 写到一半进程被杀:journal 必须记住"内容有风险",重启恢复不得放行。 func TestWindowCrashDuringWriteKeepsTargetHeld(t *testing.T) { root, c, e := instanceFixture(t) e.ingressClosed = func(context.Context) error { return nil } _, err := e.RunWindow(context.Background(), "job-kill", func(context.Context) error { return nil }, func(ctx context.Context, w *Window) (string, error) { if err := w.MarkContentRisk(ctx); err != nil { t.Fatal(err) } return "", errors.New("worker 被 kill") }, func(error) bool { return false }) if err == nil { t.Fatal("kill scenario reported success") } j, err := e.load("job-kill") if err != nil || !j.ContentRisk || j.State != "held" { t.Fatalf("journal=%+v err=%v", j, err) } // 重新打开(模拟进程重启):磁盘上的证据必须足以阻止自动开放。 reopened, err := Open(root, "target-instance", c) if err != nil { t.Fatal(err) } defer reopened.Close() if err = reopened.Recover(context.Background()); err == nil { t.Fatal("recovery released a target with unverified content") } if len(c.calls) != 3 || c.calls[0] != "gate:close" { t.Fatalf("calls=%v", c.calls) } } // 停止途中崩溃:journal 处于 stopping(内容还没风险),接管方可以继续停, // 但不需要"重新准备",也不许把没停完的服务当成已停。 func TestWindowAdoptsStoppingJournal(t *testing.T) { _, c, e := instanceFixture(t) // 手工构造一个"停到一半"的现场:app 已停、store 还在跑。 e.control.(*exitControl).services[0].Running = false j := journal{Version: 1, Source: "target-instance", Job: "job-mid", Level: "instance", Ingress: "http://127.0.0.1:1/health", Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} j.Services = []serviceRecord{ {Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true}, {Service: Service{ID: "store", Running: true, Healthy: true}}, } if err := e.save(j); err != nil { t.Fatal(err) } _, err := e.RunWindow(context.Background(), "job-mid", func(context.Context) error { t.Fatal("prepare replayed"); return nil }, func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }) if err != nil { t.Fatal(err) } if strings.Join(c.calls, ",") != "gate:close,stop:store,start:app,start:store,gate:open" { t.Fatalf("calls=%v", c.calls) } } // 只锁库的层级不许带入口:那会把"没挡入口"说成"挡了"。 func TestLibraryLevelRejectsIngressClaim(t *testing.T) { root := t.TempDir() if err := os.Chmod(root, 0700); err != nil { t.Fatal(err) } c := &fixtureControl{services: []Service{{ID: "app", Running: true, Healthy: true}}} e, err := Open(root, "target-instance", c) if err != nil { t.Fatal(err) } defer e.Close() if err = e.SetTarget("library", "http://127.0.0.1:1/health", nil, nil); err == nil { t.Fatal("library level accepted an ingress claim") } if err = e.SetTarget("instance", "", nil, nil); err == nil { t.Fatal("instance level accepted an empty ingress") } if err = e.SetTarget("", "", nil, nil); err == nil { t.Fatal("empty level accepted") } } // fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。 type fakeGate struct { calls *[]string drain bool active int } func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) { *g.calls = append(*g.calls, "gate:close") return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil } func (g *fakeGate) OpenIngress(context.Context) error { *g.calls = append(*g.calls, "gate:open") return nil } func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) { _, c, e := instanceFixture(t) e.ingressClosed = func(context.Context) error { return nil } gate := &fakeGate{calls: &c.calls, drain: true, active: 1} if err := e.SetGate(gate); err != nil { t.Fatal(err) } _, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil }, func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }) if err != nil { t.Fatal(err) } want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open" if strings.Join(c.calls, ",") != want { t.Fatalf("calls=%v want=%s", c.calls, want) } // 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。 j, err := e.load("job-gate") if err != nil || !j.DrainVerified { t.Fatalf("journal=%+v err=%v", j, err) } // 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。 if j.Gate == nil || !j.GateReopened { t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened) } } // 入口没排空(还有在途请求)时不许停写者、更不许写目标。 func TestWindowRefusesWhenGateNotDrained(t *testing.T) { _, c, e := instanceFixture(t) gate := &fakeGate{calls: &c.calls, drain: false, active: 2} if err := e.SetGate(gate); err != nil { t.Fatal(err) } work := 0 _, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil }, func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) if err == nil || work != 0 { t.Fatalf("err=%v work=%d", err, work) } // 写入者一个都不许停;但入口已经被关过一次,放弃窗口时必须把它重新打开, // 否则会出现"维护已释放、服务在跑、入口却是关的"。 if strings.Join(c.calls, ",") != "gate:close,gate:open" { t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls) } for _, s := range c.services { if !s.Running { t.Fatalf("service %s was stopped although the ingress never drained", s.ID) } } } // 窗口开过之后被人工拉起来的写入者,恢复时必须再停一次: // 上一次窗口写下的 StopRequested 不能当成"它现在还停着"。 func TestStopAllStopsWriterRestartedAfterWindowOpened(t *testing.T) { _, c, engine := instanceFixture(t) j := journal{Version: 1, Source: "target-instance", Job: "job-restart", Result: Result{State: "held"}, Level: "instance", Services: []serviceRecord{{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true}}} if err := engine.stopAll(context.Background(), &j); err != nil { t.Fatalf("stopAll: %v", err) } items, err := c.Inventory(context.Background()) if err != nil { t.Fatal(err) } for _, s := range items { if s.ID == "app" && s.Running { t.Fatal("写入者被人工拉起后没有被重新停住") } } found := false for _, call := range c.calls { if call == "stop:app" { found = true } } if !found { t.Fatalf("没有发出停止指令:%v", c.calls) } }