Files
knowledge-governance/internal/libraryfreeze/window_test.go
T
姚章浩 0ef79846e8
build / build (push) Successful in 1m19s
fix(library): 后台静默必须在关入口之后证明,并在停写者后复核
原顺序先证明后台空、再关入口排空:排空期间还在服务的在途请求会派生新任务,
那一刻的空到停写者时已经失效,而 app 收到 SIGTERM 会 exit(1) 把任务腰斩。

现在固定为:关入口并排空 → 证明后台静默(连续为空 + 稳定窗口 + 任务指纹)→
停写入者 → 停完再复核(队列非空/指纹变化即判 background_truncated)。
scheduled/retry 自行产生任务或任务卡住时 fail closed;旧 journal 的
background_idle 布尔不再当证明;入口关到一半失败时恢复入口并保持维护。
2026-10-29 04:02:59 +08:00

384 lines
14 KiB
Go

package libraryfreeze
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
)
// exitControl 模拟真实 Docker 控制器:停止时报告退出码(0 = 自己优雅退出)。
type exitControl struct {
*fixtureControl
exit map[string]int
}
func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) {
items, err := c.fixtureControl.Inventory(ctx)
if err != nil {
return nil, err
}
for i := range items {
if code, ok := c.exit[items[i].ID]; ok && !items[i].Running {
value := code
items[i].ExitCode = &value
}
}
return items, nil
}
// idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。
// 它同时实现 StableQuiescer:只有能给出任务集合指纹的检查器才谈得上证明切点。
type idleQuiescer struct{ busy bool }
func (q *idleQuiescer) Idle(context.Context) (bool, string, error) {
if q.busy {
return false, "asynq pending=1", nil
}
return true, "asynq=0 pending_ops=0", nil
}
func (q *idleQuiescer) Snapshot(context.Context) (QuiesceSnapshot, error) {
if q.busy {
return QuiesceSnapshot{Depth: 1, Digest: strings.Repeat("b", 64), Detail: "asynq pending=1"}, nil
}
return QuiesceSnapshot{Depth: 0, Digest: strings.Repeat("e", 64), Detail: "asynq=0 pending_ops=0"}, nil
}
func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
t.Helper()
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
c := &exitControl{fixtureControl: &fixtureControl{services: []Service{
{ID: "app", Running: true, Healthy: true}, {ID: "store", Running: true, Healthy: true}}}, exit: map[string]int{"app": 0, "store": 0}}
engine, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { engine.Close() })
if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil {
t.Fatal(err)
}
if err := engine.SetQuiescer(&idleQuiescer{}); err != nil {
t.Fatal(err)
}
// 实例级窗口必须有入口闸门:默认装一个会正常排空的假闸门。
if err := engine.SetGate(&fakeGate{calls: &c.calls, drain: true, active: 1}); err != nil {
t.Fatal(err)
}
if err := engine.SetQuiesceStability(5*time.Millisecond, 200*time.Millisecond); err != nil {
t.Fatal(err)
}
return root, c, engine
}
// 维护窗口必须真的挡住写入:work 只能在所有写入者确认停下、入口确认关闭之后才开始。
func TestWindowStopsWritersBeforeWorkAndRestoresAfter(t *testing.T) {
_, c, e := instanceFixture(t)
var sawStopped, sawClosed bool
e.ingressClosed = func(context.Context) error { sawClosed = true; return nil }
result, err := e.RunWindow(context.Background(), "job-1", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
for _, s := range c.services {
if s.Running {
t.Fatal("work started while a writer was still running")
}
}
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
sawStopped = true
return strings.Repeat("b", 64), nil
}, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
if !sawStopped || !sawClosed {
t.Fatalf("stopped=%v closed=%v", sawStopped, sawClosed)
}
if result.State != "succeeded" {
t.Fatalf("result=%+v", result)
}
j, err := e.load("job-1")
if err != nil {
t.Fatal(err)
}
if !j.DrainVerified || !j.IngressClosed || !j.ContentRisk || j.Level != "instance" {
t.Fatalf("journal evidence missing: %+v", j)
}
if strings.Join(c.calls, ",") != "gate:close,stop:app,stop:store,start:app,start:store,gate:open" {
t.Fatalf("calls=%v", c.calls)
}
}
// 实例级窗口必须有入口闸门:没有闸门就无法证明"入口已关",后台静默也就没有前提。
// 拒绝发生在动任何容器之前。
func TestInstanceWindowRequiresGate(t *testing.T) {
_, c, e := instanceFixture(t)
e.gate = nil
work := 0
_, err := e.RunWindow(context.Background(), "job-nogate", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || !strings.Contains(err.Error(), "入口闸门") {
t.Fatalf("err=%v", err)
}
if work != 0 {
t.Fatal("work ran without an ingress gate")
}
if len(c.calls) != 0 {
t.Fatalf("containers touched without a gate: %v", c.calls)
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were stopped without a gate")
}
}
}
// 入口停不下来(还在对外服务)就不是维护窗口。
func TestWindowRefusesWorkWhenIngressStillServing(t *testing.T) {
_, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return errors.New("ingress still serving") }
work := 0
_, err := e.RunWindow(context.Background(), "job-ingress", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were not restored after ingress failure")
}
}
}
// 内容回滚失败:服务必须保持停止,且崩溃恢复不许自动把它们起来。
func TestWindowKeepsWritersStoppedWhenRestoreBlocked(t *testing.T) {
root, c, e := instanceFixture(t)
_, err := e.RunWindow(context.Background(), "job-half", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
return "", errors.New("目标写到一半,回滚也失败")
}, func(error) bool { return false })
if err == nil {
t.Fatal("blocked restore reported success")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("writer %s was restarted on a half-written target", s.ID)
}
}
j, err := e.load("job-half")
if err != nil || !j.RestoreBlocked || !j.ContentRisk {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 进程重启后:恢复器不能自作主张把半写目标对外开放。
if err = e.Recover(context.Background()); err == nil {
t.Fatal("recover reopened a target with unverified content")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("recover restarted %s", s.ID)
}
}
// 原作业接管:不允许重新 stop(写入者本来就没起来),也不从零重来。
adopted := 0
_, err = e.RunWindow(context.Background(), "job-half", func(context.Context) error { t.Fatal("prepare ran on adoption"); return nil },
func(ctx context.Context, w *Window) (string, error) { adopted++; return "", nil }, func(error) bool { return true })
if err != nil || adopted != 1 {
t.Fatalf("adopt err=%v adopted=%d", err, adopted)
}
for _, s := range c.services {
if !s.Running {
t.Fatalf("writer %s was not restored after adoption", s.ID)
}
}
if _, statErr := os.Stat(root + "/job-half.json"); statErr != nil {
t.Fatal(statErr)
}
}
// 写到一半进程被杀:journal 必须记住"内容有风险",重启恢复不得放行。
func TestWindowCrashDuringWriteKeepsTargetHeld(t *testing.T) {
root, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return nil }
_, err := e.RunWindow(context.Background(), "job-kill", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
return "", errors.New("worker 被 kill")
}, func(error) bool { return false })
if err == nil {
t.Fatal("kill scenario reported success")
}
j, err := e.load("job-kill")
if err != nil || !j.ContentRisk || j.State != "held" {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 重新打开(模拟进程重启):磁盘上的证据必须足以阻止自动开放。
reopened, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
if err = reopened.Recover(context.Background()); err == nil {
t.Fatal("recovery released a target with unverified content")
}
if len(c.calls) != 3 || c.calls[0] != "gate:close" {
t.Fatalf("calls=%v", c.calls)
}
}
// 停止途中崩溃:journal 处于 stopping(内容还没风险),接管方可以继续停,
// 但不需要"重新准备",也不许把没停完的服务当成已停。
func TestWindowAdoptsStoppingJournal(t *testing.T) {
_, c, e := instanceFixture(t)
// 手工构造一个"停到一半"的现场:app 已停、store 还在跑。
e.control.(*exitControl).services[0].Running = false
j := journal{Version: 1, Source: "target-instance", Job: "job-mid", Level: "instance",
Ingress: "http://127.0.0.1:1/health", Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
j.Services = []serviceRecord{
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true},
{Service: Service{ID: "store", Running: true, Healthy: true}},
}
if err := e.save(j); err != nil {
t.Fatal(err)
}
_, err := e.RunWindow(context.Background(), "job-mid", func(context.Context) error { t.Fatal("prepare replayed"); return nil },
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
if strings.Join(c.calls, ",") != "gate:close,stop:store,start:app,start:store,gate:open" {
t.Fatalf("calls=%v", c.calls)
}
}
// 只锁库的层级不许带入口:那会把"没挡入口"说成"挡了"。
func TestLibraryLevelRejectsIngressClaim(t *testing.T) {
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
c := &fixtureControl{services: []Service{{ID: "app", Running: true, Healthy: true}}}
e, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
defer e.Close()
if err = e.SetTarget("library", "http://127.0.0.1:1/health", nil, nil); err == nil {
t.Fatal("library level accepted an ingress claim")
}
if err = e.SetTarget("instance", "", nil, nil); err == nil {
t.Fatal("instance level accepted an empty ingress")
}
if err = e.SetTarget("", "", nil, nil); err == nil {
t.Fatal("empty level accepted")
}
}
// fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。
type fakeGate struct {
calls *[]string
drain bool
active int
}
func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) {
*g.calls = append(*g.calls, "gate:close")
return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil
}
func (g *fakeGate) OpenIngress(context.Context) error {
*g.calls = append(*g.calls, "gate:open")
return nil
}
func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) {
_, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return nil }
gate := &fakeGate{calls: &c.calls, drain: true, active: 1}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
_, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open"
if strings.Join(c.calls, ",") != want {
t.Fatalf("calls=%v want=%s", c.calls, want)
}
// 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。
j, err := e.load("job-gate")
if err != nil || !j.DrainVerified {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。
if j.Gate == nil || !j.GateReopened {
t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened)
}
}
// 入口没排空(还有在途请求)时不许停写者、更不许写目标。
func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
_, c, e := instanceFixture(t)
gate := &fakeGate{calls: &c.calls, drain: false, active: 2}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
// 写入者一个都不许停;但入口已经被关过一次,放弃窗口时必须把它重新打开,
// 否则会出现"维护已释放、服务在跑、入口却是关的"。
if strings.Join(c.calls, ",") != "gate:close,gate:open" {
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
}
for _, s := range c.services {
if !s.Running {
t.Fatalf("service %s was stopped although the ingress never drained", s.ID)
}
}
}
// 窗口开过之后被人工拉起来的写入者,恢复时必须再停一次:
// 上一次窗口写下的 StopRequested 不能当成"它现在还停着"。
func TestStopAllStopsWriterRestartedAfterWindowOpened(t *testing.T) {
_, c, engine := instanceFixture(t)
j := journal{Version: 1, Source: "target-instance", Job: "job-restart",
Result: Result{State: "held"}, Level: "instance",
Services: []serviceRecord{{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true}}}
if err := engine.stopAll(context.Background(), &j); err != nil {
t.Fatalf("stopAll: %v", err)
}
items, err := c.Inventory(context.Background())
if err != nil {
t.Fatal(err)
}
for _, s := range items {
if s.ID == "app" && s.Running {
t.Fatal("写入者被人工拉起后没有被重新停住")
}
}
found := false
for _, call := range c.calls {
if call == "stop:app" {
found = true
}
}
if !found {
t.Fatalf("没有发出停止指令:%v", c.calls)
}
}