0ef79846e8
build / build (push) Successful in 1m19s
原顺序先证明后台空、再关入口排空:排空期间还在服务的在途请求会派生新任务, 那一刻的空到停写者时已经失效,而 app 收到 SIGTERM 会 exit(1) 把任务腰斩。 现在固定为:关入口并排空 → 证明后台静默(连续为空 + 稳定窗口 + 任务指纹)→ 停写入者 → 停完再复核(队列非空/指纹变化即判 background_truncated)。 scheduled/retry 自行产生任务或任务卡住时 fail closed;旧 journal 的 background_idle 布尔不再当证明;入口关到一半失败时恢复入口并保持维护。
341 lines
12 KiB
Go
341 lines
12 KiB
Go
package libraryfreeze
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// scriptedQuiescer 按脚本回答取样,并把每次取样记进共享调用序列:
|
|
// 顺序断言必须能看见"取样发生在关入口之后、停写者之前"。
|
|
type scriptedQuiescer struct {
|
|
calls *[]string
|
|
samples int
|
|
next func(n int) QuiesceSnapshot
|
|
}
|
|
|
|
func (q *scriptedQuiescer) Idle(ctx context.Context) (bool, string, error) {
|
|
snapshot, err := q.Snapshot(ctx)
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
idle := snapshot.Depth == 0 && snapshot.PendingOps == 0 && snapshot.Unsettled == 0
|
|
return idle, snapshot.Detail, nil
|
|
}
|
|
|
|
func (q *scriptedQuiescer) Snapshot(context.Context) (QuiesceSnapshot, error) {
|
|
q.samples++
|
|
*q.calls = append(*q.calls, "quiesce")
|
|
return q.next(q.samples), nil
|
|
}
|
|
|
|
func busySample(digest string) QuiesceSnapshot {
|
|
return QuiesceSnapshot{Depth: 1, Digest: strings.Repeat(digest, 64), Detail: "asynq pending=1"}
|
|
}
|
|
|
|
func quietSample() QuiesceSnapshot {
|
|
return QuiesceSnapshot{Depth: 0, Digest: strings.Repeat("e", 64), Detail: "asynq=0 pending_ops=0"}
|
|
}
|
|
|
|
func firstIndex(calls []string, want string) int {
|
|
for i, call := range calls {
|
|
if call == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// 取样必须发生在入口确认关闭之后:关之前拿到的"空"会被排空期间的新任务推翻。
|
|
func TestQuiesceProofStartsOnlyAfterIngressClosed(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
q := &scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot { return quietSample() }}
|
|
if err := e.SetQuiescer(q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := e.RunWindow(context.Background(), "job-order", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateAt := firstIndex(c.calls, "gate:close")
|
|
stopAt := firstIndex(c.calls, "stop:app")
|
|
if gateAt < 0 || stopAt < 0 {
|
|
t.Fatalf("calls=%v", c.calls)
|
|
}
|
|
for i, call := range c.calls {
|
|
if call == "quiesce" && i < gateAt {
|
|
t.Fatalf("quiesce sampled before the ingress was closed: %v", c.calls)
|
|
}
|
|
}
|
|
proofProbes := 0
|
|
for i, call := range c.calls {
|
|
if call == "quiesce" && i < stopAt {
|
|
proofProbes++
|
|
}
|
|
}
|
|
if proofProbes < 2 {
|
|
t.Fatalf("only %d quiet samples before the writers were stopped: %v", proofProbes, c.calls)
|
|
}
|
|
afterStop := 0
|
|
for i, call := range c.calls {
|
|
if call == "quiesce" && i > stopAt {
|
|
afterStop++
|
|
}
|
|
}
|
|
if afterStop == 0 {
|
|
t.Fatalf("no post-stop verification sample: %v", c.calls)
|
|
}
|
|
journalAfter, err := e.load("job-order")
|
|
if err != nil || journalAfter.Quiesce == nil || journalAfter.Quiesce.Samples < 2 {
|
|
t.Fatalf("journal=%+v err=%v", journalAfter, err)
|
|
}
|
|
if journalAfter.Quiesce.PostStopAt.IsZero() {
|
|
t.Fatal("停写者之后没有复核后台任务集合")
|
|
}
|
|
}
|
|
|
|
// 排空期间通过真实入口派生出来的任务:窗口必须等它结束,不能直接停写者。
|
|
func TestWindowWaitsForTaskEnqueuedDuringDrain(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
q := &scriptedQuiescer{calls: &c.calls, next: func(n int) QuiesceSnapshot {
|
|
if n <= 3 {
|
|
if firstIndex(c.calls, "stop:app") >= 0 {
|
|
t.Fatalf("writer stopped while a task was still running: %v", c.calls)
|
|
}
|
|
return busySample("b")
|
|
}
|
|
return quietSample()
|
|
}}
|
|
if err := e.SetQuiescer(q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
if _, err := e.RunWindow(context.Background(), "job-drain-task", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if work != 1 {
|
|
t.Fatalf("work=%d", work)
|
|
}
|
|
journalAfter, err := e.load("job-drain-task")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proof := journalAfter.Quiesce
|
|
if proof == nil || proof.Depth != 0 || proof.PostStopDepth != 0 || proof.PostStopDigest != proof.Digest {
|
|
t.Fatalf("proof=%+v", proof)
|
|
}
|
|
if q.samples < 4 {
|
|
t.Fatalf("sampled %d times, expected to keep sampling while the task ran", q.samples)
|
|
}
|
|
}
|
|
|
|
// 队列一直不空(scheduled/retry 自己冒任务):宁可不进窗口,也不把任务留给 SIGTERM 腰斩。
|
|
func TestWindowFailsClosedWhenBackgroundKeepsProducing(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
q := &scriptedQuiescer{calls: &c.calls, next: func(n int) QuiesceSnapshot {
|
|
return busySample([]string{"a", "b", "c"}[n%3])
|
|
}}
|
|
if err := e.SetQuiescer(q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceTimeout(400 * time.Millisecond); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-producer", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || !strings.Contains(err.Error(), "后台任务") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if work != 0 {
|
|
t.Fatal("work ran although a producer kept the queue busy")
|
|
}
|
|
if firstIndex(c.calls, "stop:app") >= 0 {
|
|
t.Fatalf("writers were stopped while tasks kept arriving: %v", c.calls)
|
|
}
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatalf("service %s stayed stopped after the window was abandoned", s.ID)
|
|
}
|
|
}
|
|
journalAfter, loadErr := e.load("job-producer")
|
|
if loadErr != nil || journalAfter.State != "failed" || journalAfter.Code != "background_busy" {
|
|
t.Fatalf("journal=%+v err=%v", journalAfter, loadErr)
|
|
}
|
|
if journalAfter.Quiesce != nil {
|
|
t.Fatalf("failed window carries a fake quiet proof: %+v", journalAfter.Quiesce)
|
|
}
|
|
}
|
|
|
|
// 任务在"证明静默"和"停写者"之间冒出来:停完的复核必须发现它,按截断处理。
|
|
func TestWindowFailsClosedWhenTaskAppearsAfterWriterStopped(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
q := &scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
|
|
if firstIndex(c.calls, "stop:app") >= 0 {
|
|
return busySample("c")
|
|
}
|
|
return quietSample()
|
|
}}
|
|
if err := e.SetQuiescer(q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-late-task", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || !strings.Contains(err.Error(), "切点") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if work != 0 {
|
|
t.Fatal("work ran with an inconsistent cut point")
|
|
}
|
|
journalAfter, loadErr := e.load("job-late-task")
|
|
if loadErr != nil || journalAfter.Code != "background_truncated" || journalAfter.State != "failed" {
|
|
t.Fatalf("journal=%+v err=%v", journalAfter, loadErr)
|
|
}
|
|
// 还没写目标:可以把服务恢复,但绝不能再往下写。
|
|
for _, s := range c.services {
|
|
if !s.Running {
|
|
t.Fatalf("service %s stayed stopped after an abandoned window", s.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 旧 journal 里的 background_idle 布尔仍然能读,但绝不当作"后台已静默"的证据。
|
|
func TestLegacyBackgroundIdleIsNotProof(t *testing.T) {
|
|
root, _, e := instanceFixture(t)
|
|
j := heldJournal("job-old-idle", strings.Repeat("a", 64), strings.Repeat("b", 64))
|
|
j.State = "stopping"
|
|
j.ContentRisk = false
|
|
j.RestoreBlocked = false
|
|
j.BackgroundIdle = true
|
|
if err := e.save(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := Open(root, "target-instance", e.control)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
loaded, err := reopened.load("job-old-idle")
|
|
if err != nil {
|
|
t.Fatalf("legacy journal became unreadable: %v", err)
|
|
}
|
|
if loaded.Quiesce != nil || loaded.BackgroundIdle {
|
|
t.Fatalf("legacy idle flag was promoted to a proof: %+v", loaded)
|
|
}
|
|
}
|
|
|
|
// 恢复要自己重新证明后台静默:journal 里的旧证明不能拿来复用。
|
|
func TestRecoverWindowReProvesBackgroundQuiesce(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j := heldJournal("job-stale-proof", strings.Repeat("a", 64), strings.Repeat("b", 64))
|
|
now := time.Now().UTC()
|
|
j.BackgroundIdle = true
|
|
j.Quiesce = &QuiesceProof{AfterIngress: now.Add(-time.Minute), Samples: 3, StableMS: 1500,
|
|
Digest: strings.Repeat("e", 64), Detail: "asynq=0", ProvenAt: now.Add(-30 * time.Second)}
|
|
if err := e.save(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := e.load("job-stale-proof"); err != nil {
|
|
t.Fatalf("fixture journal is invalid: %v", err)
|
|
}
|
|
if err := e.SetQuiescer(&scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
|
|
return busySample("d")
|
|
}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceTimeout(300 * time.Millisecond); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiesceStability(5*time.Millisecond, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
called := false
|
|
if _, err := e.RecoverWindow(context.Background(), "job-stale-proof",
|
|
restorerFunc(func(context.Context, string) error { called = true; return nil }),
|
|
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
|
|
t.Fatal("recovery reused the journal's stale quiet proof")
|
|
}
|
|
if called {
|
|
t.Fatal("restore ran although the background was busy")
|
|
}
|
|
after, loadErr := e.load("job-stale-proof")
|
|
if loadErr != nil || terminal(after.State) {
|
|
t.Fatalf("journal=%+v err=%v", after, loadErr)
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatal("service was started although recovery could not prove background quiet")
|
|
}
|
|
}
|
|
}
|
|
|
|
// 写入者已停但队列里还有任务:这些任务被截断过,服务不起来就永远排不掉——直接报出来。
|
|
func TestRecoverWindowRefusesWithTruncatedTasks(t *testing.T) {
|
|
_, c, e := instanceFixture(t)
|
|
stopAllServices(c)
|
|
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.save(heldJournal("job-trunc", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.SetQuiescer(&scriptedQuiescer{calls: &c.calls, next: func(int) QuiesceSnapshot {
|
|
return busySample("f")
|
|
}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
called := false
|
|
_, err := e.RecoverWindow(context.Background(), "job-trunc",
|
|
restorerFunc(func(context.Context, string) error { called = true; return nil }),
|
|
verifierFunc(func(context.Context, string) (string, error) { return "x", nil }))
|
|
if err == nil || !strings.Contains(err.Error(), "截断") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if called {
|
|
t.Fatal("restore ran with truncated tasks in the queue")
|
|
}
|
|
after, loadErr := e.load("job-trunc")
|
|
if loadErr != nil || after.State != "held" || after.Code != "background_truncated" || !after.RestoreBlocked {
|
|
t.Fatalf("journal=%+v err=%v", after, loadErr)
|
|
}
|
|
for _, s := range c.services {
|
|
if s.Running {
|
|
t.Fatal("service was started although truncated tasks remain")
|
|
}
|
|
}
|
|
}
|
|
|
|
// 恢复时拿不到任务指纹(只会回答 Idle 的检查器)就不许收尾:那是"瞬时 0",不是切点。
|
|
func TestInstanceWindowRejectsQuiescerWithoutSnapshots(t *testing.T) {
|
|
_, _, e := instanceFixture(t)
|
|
if err := e.SetQuiescer(idleOnlyQuiescer{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := 0
|
|
_, err := e.RunWindow(context.Background(), "job-idle-only", func(context.Context) error { return nil },
|
|
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
|
if err == nil || !strings.Contains(err.Error(), "指纹") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if work != 0 {
|
|
t.Fatal("window opened without a provable cut point")
|
|
}
|
|
}
|
|
|
|
type idleOnlyQuiescer struct{}
|
|
|
|
func (idleOnlyQuiescer) Idle(context.Context) (bool, string, error) { return true, "asynq=0", nil }
|