Files
knowledge-governance/internal/libraryfreeze/quiesce_order_test.go
T
姚章浩 0ef79846e8
build / build (push) Successful in 1m19s
fix(library): 后台静默必须在关入口之后证明,并在停写者后复核
原顺序先证明后台空、再关入口排空:排空期间还在服务的在途请求会派生新任务,
那一刻的空到停写者时已经失效,而 app 收到 SIGTERM 会 exit(1) 把任务腰斩。

现在固定为:关入口并排空 → 证明后台静默(连续为空 + 稳定窗口 + 任务指纹)→
停写入者 → 停完再复核(队列非空/指纹变化即判 background_truncated)。
scheduled/retry 自行产生任务或任务卡住时 fail closed;旧 journal 的
background_idle 布尔不再当证明;入口关到一半失败时恢复入口并保持维护。
2026-10-29 04:02:59 +08:00

341 lines
12 KiB
Go

package libraryfreeze
import (
"context"
"strings"
"testing"
"time"
)
// scriptedQuiescer 按脚本回答取样,并把每次取样记进共享调用序列:
// 顺序断言必须能看见"取样发生在关入口之后、停写者之前"。
type scriptedQuiescer struct {
calls *[]string
samples int
next func(n int) QuiesceSnapshot
}
func (q *scriptedQuiescer) Idle(ctx context.Context) (bool, string, error) {
snapshot, err := q.Snapshot(ctx)
if err != nil {
return false, "", err
}
idle := snapshot.Depth == 0 && snapshot.PendingOps == 0 && snapshot.Unsettled == 0
return idle, snapshot.Detail, nil
}
func (q *scriptedQuiescer) Snapshot(context.Context) (QuiesceSnapshot, error) {
q.samples++
*q.calls = append(*q.calls, "quiesce")
return q.next(q.samples), nil
}
func busySample(digest string) QuiesceSnapshot {
return QuiesceSnapshot{Depth: 1, Digest: strings.Repeat(digest, 64), Detail: "asynq pending=1"}
}
func quietSample() QuiesceSnapshot {
return QuiesceSnapshot{Depth: 0, Digest: strings.Repeat("e", 64), Detail: "asynq=0 pending_ops=0"}
}
func firstIndex(calls []string, want string) int {
for i, call := range calls {
if call == want {
return i
}
}
return -1
}
// 取样必须发生在入口确认关闭之后:关之前拿到的"空"会被排空期间的新任务推翻。
func TestQuiesceProofStartsOnlyAfterIngressClosed(t *testing.T) {
_, c, e := instanceFixture(t)
q := &scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot { return quietSample() }}
if err := e.SetQuiescer(q); err != nil {
t.Fatal(err)
}
if _, err := e.RunWindow(context.Background(), "job-order", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }); err != nil {
t.Fatal(err)
}
gateAt := firstIndex(c.calls, "gate:close")
stopAt := firstIndex(c.calls, "stop:app")
if gateAt < 0 || stopAt < 0 {
t.Fatalf("calls=%v", c.calls)
}
for i, call := range c.calls {
if call == "quiesce" && i < gateAt {
t.Fatalf("quiesce sampled before the ingress was closed: %v", c.calls)
}
}
proofProbes := 0
for i, call := range c.calls {
if call == "quiesce" && i < stopAt {
proofProbes++
}
}
if proofProbes < 2 {
t.Fatalf("only %d quiet samples before the writers were stopped: %v", proofProbes, c.calls)
}
afterStop := 0
for i, call := range c.calls {
if call == "quiesce" && i > stopAt {
afterStop++
}
}
if afterStop == 0 {
t.Fatalf("no post-stop verification sample: %v", c.calls)
}
journalAfter, err := e.load("job-order")
if err != nil || journalAfter.Quiesce == nil || journalAfter.Quiesce.Samples < 2 {
t.Fatalf("journal=%+v err=%v", journalAfter, err)
}
if journalAfter.Quiesce.PostStopAt.IsZero() {
t.Fatal("停写者之后没有复核后台任务集合")
}
}
// 排空期间通过真实入口派生出来的任务:窗口必须等它结束,不能直接停写者。
func TestWindowWaitsForTaskEnqueuedDuringDrain(t *testing.T) {
_, c, e := instanceFixture(t)
q := &scriptedQuiescer{calls: &c.calls, next: func(n int) QuiesceSnapshot {
if n <= 3 {
if firstIndex(c.calls, "stop:app") >= 0 {
t.Fatalf("writer stopped while a task was still running: %v", c.calls)
}
return busySample("b")
}
return quietSample()
}}
if err := e.SetQuiescer(q); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
t.Fatal(err)
}
work := 0
if _, err := e.RunWindow(context.Background(), "job-drain-task", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }); err != nil {
t.Fatal(err)
}
if work != 1 {
t.Fatalf("work=%d", work)
}
journalAfter, err := e.load("job-drain-task")
if err != nil {
t.Fatal(err)
}
proof := journalAfter.Quiesce
if proof == nil || proof.Depth != 0 || proof.PostStopDepth != 0 || proof.PostStopDigest != proof.Digest {
t.Fatalf("proof=%+v", proof)
}
if q.samples < 4 {
t.Fatalf("sampled %d times, expected to keep sampling while the task ran", q.samples)
}
}
// 队列一直不空(scheduled/retry 自己冒任务):宁可不进窗口,也不把任务留给 SIGTERM 腰斩。
func TestWindowFailsClosedWhenBackgroundKeepsProducing(t *testing.T) {
_, c, e := instanceFixture(t)
q := &scriptedQuiescer{calls: &c.calls, next: func(n int) QuiesceSnapshot {
return busySample([]string{"a", "b", "c"}[n%3])
}}
if err := e.SetQuiescer(q); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceTimeout(400 * time.Millisecond); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-producer", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || !strings.Contains(err.Error(), "后台任务") {
t.Fatalf("err=%v", err)
}
if work != 0 {
t.Fatal("work ran although a producer kept the queue busy")
}
if firstIndex(c.calls, "stop:app") >= 0 {
t.Fatalf("writers were stopped while tasks kept arriving: %v", c.calls)
}
for _, s := range c.services {
if !s.Running {
t.Fatalf("service %s stayed stopped after the window was abandoned", s.ID)
}
}
journalAfter, loadErr := e.load("job-producer")
if loadErr != nil || journalAfter.State != "failed" || journalAfter.Code != "background_busy" {
t.Fatalf("journal=%+v err=%v", journalAfter, loadErr)
}
if journalAfter.Quiesce != nil {
t.Fatalf("failed window carries a fake quiet proof: %+v", journalAfter.Quiesce)
}
}
// 任务在"证明静默"和"停写者"之间冒出来:停完的复核必须发现它,按截断处理。
func TestWindowFailsClosedWhenTaskAppearsAfterWriterStopped(t *testing.T) {
_, c, e := instanceFixture(t)
q := &scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
if firstIndex(c.calls, "stop:app") >= 0 {
return busySample("c")
}
return quietSample()
}}
if err := e.SetQuiescer(q); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-late-task", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || !strings.Contains(err.Error(), "切点") {
t.Fatalf("err=%v", err)
}
if work != 0 {
t.Fatal("work ran with an inconsistent cut point")
}
journalAfter, loadErr := e.load("job-late-task")
if loadErr != nil || journalAfter.Code != "background_truncated" || journalAfter.State != "failed" {
t.Fatalf("journal=%+v err=%v", journalAfter, loadErr)
}
// 还没写目标:可以把服务恢复,但绝不能再往下写。
for _, s := range c.services {
if !s.Running {
t.Fatalf("service %s stayed stopped after an abandoned window", s.ID)
}
}
}
// 旧 journal 里的 background_idle 布尔仍然能读,但绝不当作"后台已静默"的证据。
func TestLegacyBackgroundIdleIsNotProof(t *testing.T) {
root, _, e := instanceFixture(t)
j := heldJournal("job-old-idle", strings.Repeat("a", 64), strings.Repeat("b", 64))
j.State = "stopping"
j.ContentRisk = false
j.RestoreBlocked = false
j.BackgroundIdle = true
if err := e.save(j); err != nil {
t.Fatal(err)
}
reopened, err := Open(root, "target-instance", e.control)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
loaded, err := reopened.load("job-old-idle")
if err != nil {
t.Fatalf("legacy journal became unreadable: %v", err)
}
if loaded.Quiesce != nil || loaded.BackgroundIdle {
t.Fatalf("legacy idle flag was promoted to a proof: %+v", loaded)
}
}
// 恢复要自己重新证明后台静默:journal 里的旧证明不能拿来复用。
func TestRecoverWindowReProvesBackgroundQuiesce(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
j := heldJournal("job-stale-proof", strings.Repeat("a", 64), strings.Repeat("b", 64))
now := time.Now().UTC()
j.BackgroundIdle = true
j.Quiesce = &QuiesceProof{AfterIngress: now.Add(-time.Minute), Samples: 3, StableMS: 1500,
Digest: strings.Repeat("e", 64), Detail: "asynq=0", ProvenAt: now.Add(-30 * time.Second)}
if err := e.save(j); err != nil {
t.Fatal(err)
}
if _, err := e.load("job-stale-proof"); err != nil {
t.Fatalf("fixture journal is invalid: %v", err)
}
if err := e.SetQuiescer(&scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
return busySample("d")
}}); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceTimeout(300 * time.Millisecond); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
t.Fatal(err)
}
called := false
if _, err := e.RecoverWindow(context.Background(), "job-stale-proof",
restorerFunc(func(context.Context, string) error { called = true; return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
t.Fatal("recovery reused the journal's stale quiet proof")
}
if called {
t.Fatal("restore ran although the background was busy")
}
after, loadErr := e.load("job-stale-proof")
if loadErr != nil || terminal(after.State) {
t.Fatalf("journal=%+v err=%v", after, loadErr)
}
for _, s := range c.services {
if s.Running {
t.Fatal("service was started although recovery could not prove background quiet")
}
}
}
// 写入者已停但队列里还有任务:这些任务被截断过,服务不起来就永远排不掉——直接报出来。
func TestRecoverWindowRefusesWithTruncatedTasks(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-trunc", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if err := e.SetQuiescer(&scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
return busySample("f")
}}); err != nil {
t.Fatal(err)
}
called := false
_, err := e.RecoverWindow(context.Background(), "job-trunc",
restorerFunc(func(context.Context, string) error { called = true; return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil }))
if err == nil || !strings.Contains(err.Error(), "截断") {
t.Fatalf("err=%v", err)
}
if called {
t.Fatal("restore ran with truncated tasks in the queue")
}
after, loadErr := e.load("job-trunc")
if loadErr != nil || after.State != "held" || after.Code != "background_truncated" || !after.RestoreBlocked {
t.Fatalf("journal=%+v err=%v", after, loadErr)
}
for _, s := range c.services {
if s.Running {
t.Fatal("service was started although truncated tasks remain")
}
}
}
// 恢复时拿不到任务指纹(只会回答 Idle 的检查器)就不许收尾:那是"瞬时 0",不是切点。
func TestInstanceWindowRejectsQuiescerWithoutSnapshots(t *testing.T) {
_, _, e := instanceFixture(t)
if err := e.SetQuiescer(idleOnlyQuiescer{}); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-idle-only", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || !strings.Contains(err.Error(), "指纹") {
t.Fatalf("err=%v", err)
}
if work != 0 {
t.Fatal("window opened without a provable cut point")
}
}
type idleOnlyQuiescer struct{}
func (idleOnlyQuiescer) Idle(context.Context) (bool, string, error) { return true, "asynq=0", nil }