姚章浩
6bdc17474c
2026-10-29T19:56Z 的 after:database 演练里，worker 只留下汇总行 `library job takeover sweep: 1 resumed, 0 failed without backup`：既不说哪个 作业，也不说它是否真被本进程认领（被别的 worker 抢走、attach 失败都只影响 计数）。演练脚本要的老文案只在"新建队列行"那条路径打印，真实走的是"队列行 已存在"那条 ⇒ 判据与实现各说一套，演练 exit 5。 - 新增 `library job <id> takeover claimed at=… kind=… scope=… target=… backup=… queue=… path=…`：claim + attach 都成功之后才写，两条认领路径共用 （path=enqueue / existing-queue）； - 新增 `library job <id> takeover decision=…`：分类后、动手之前写，含 finalize_only。只留阶段表的话，同快照收尾能把它伪造成完整续作； - resume 的 store 参数抽成接口，单测才能证明"没认领/attach 失败时一行都不写" （变异验证：把认领行提到 attach 之前 ⇒ 红；删掉 existing-queue 那行 ⇒ 红）； - 接管复用的预备份在证据里写 `backup_reused=true`：分不清"复用"与"崩溃后重做 备份"，后者会把半写状态当成可回滚基线。 verify: go test ./cmd/worker -run 'TestTakeover|TestLedgerBackupStore' -count=1 全绿； make verify overall success（16 项必需全绿）。
251 lines
8.7 KiB
Go
251 lines
8.7 KiB
Go
package main
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"errors"
|
||
"log"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
)
|
||
|
||
// 这些用例守的是"接管证据"的契约，不是实现细节：
|
||
// 只有真的把作业认领到自己名下（claim + attach 都成功）才允许写认领行；
|
||
// 队列行已存在这条路径也必须写（上一轮只加了计数，外部只看到汇总，判不了是哪个作业）。
|
||
type fakeResumeStore struct {
|
||
interrupted []governance.LibraryJob
|
||
claimOK bool
|
||
claimErr error
|
||
enqueueJob *jobs.Job
|
||
enqueueErr error
|
||
attachErr error
|
||
calls []string
|
||
logAtAttach string // attach 那一刻的日志内容：用来断言"先认领、后写日志"
|
||
}
|
||
|
||
func (f *fakeResumeStore) InterruptedLibraryJobs(context.Context) ([]governance.LibraryJob, error) {
|
||
f.calls = append(f.calls, "interrupted")
|
||
return f.interrupted, nil
|
||
}
|
||
|
||
func (f *fakeResumeStore) AdvanceLibraryJob(context.Context, string, governance.LibraryJobState,
|
||
string, string, string, string, string, time.Time) error {
|
||
f.calls = append(f.calls, "advance")
|
||
return nil
|
||
}
|
||
|
||
func (f *fakeResumeStore) ClaimLibraryJobForResume(context.Context, string, time.Time) (bool, error) {
|
||
f.calls = append(f.calls, "claim")
|
||
return f.claimOK, f.claimErr
|
||
}
|
||
|
||
func (f *fakeResumeStore) Enqueue(context.Context, jobs.EnqueueRequest, time.Time) (*jobs.Job, error) {
|
||
f.calls = append(f.calls, "enqueue")
|
||
return f.enqueueJob, f.enqueueErr
|
||
}
|
||
|
||
func (f *fakeResumeStore) ResetQueueJobForResume(context.Context, string, time.Time) error {
|
||
f.calls = append(f.calls, "reset")
|
||
return nil
|
||
}
|
||
|
||
func (f *fakeResumeStore) AttachLibraryQueueJob(context.Context, string, string, time.Time) error {
|
||
f.calls = append(f.calls, "attach")
|
||
f.logAtAttach = captureLogAtAttach()
|
||
return f.attachErr
|
||
}
|
||
|
||
// captureLogAtAttach 由用例在运行时替换（见 newCapture）。
|
||
var captureLogAtAttach = func() string { return "" }
|
||
|
||
func takeoverTestJob() governance.LibraryJob {
|
||
return governance.LibraryJob{
|
||
ID: "librestore-internal-1793303791259342087", Scope: "internal",
|
||
Kind: governance.LibraryJobRestore,
|
||
State: governance.LibraryJobRunning,
|
||
SnapshotID: "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510",
|
||
TargetSnapshotID: "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7",
|
||
}
|
||
}
|
||
|
||
// newCapture 把标准日志接到缓冲区，并在 attach 回调里取当时的快照。
|
||
func newCapture(t *testing.T) *bytes.Buffer {
|
||
t.Helper()
|
||
buf := &bytes.Buffer{}
|
||
old := log.Writer()
|
||
log.SetOutput(buf)
|
||
captureLogAtAttach = func() string { return buf.String() }
|
||
t.Cleanup(func() {
|
||
log.SetOutput(old)
|
||
captureLogAtAttach = func() string { return "" }
|
||
})
|
||
return buf
|
||
}
|
||
|
||
func claimLineOf(t *testing.T, logs string) string {
|
||
t.Helper()
|
||
for _, line := range strings.Split(logs, "\n") {
|
||
if strings.Contains(line, "takeover claimed") {
|
||
return line
|
||
}
|
||
}
|
||
t.Fatalf("日志里没有本作业的认领行：\n%s", logs)
|
||
return ""
|
||
}
|
||
|
||
func kvOf(t *testing.T, line string) map[string]string {
|
||
t.Helper()
|
||
fields := strings.Fields(line)
|
||
out := map[string]string{}
|
||
for _, f := range fields {
|
||
if i := strings.Index(f, "="); i > 0 {
|
||
out[f[:i]] = f[i+1:]
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func TestTakeoverClaimLoggedAfterSuccessfulAttach(t *testing.T) {
|
||
buf := newCapture(t)
|
||
job := takeoverTestJob()
|
||
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
|
||
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}}
|
||
|
||
resumed, failed, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
|
||
if err != nil || failed != 0 {
|
||
t.Fatalf("resume 失败：resumed=%d failed=%d err=%v", resumed, failed, err)
|
||
}
|
||
if resumed != 1 {
|
||
t.Fatalf("resumed=%d, want 1", resumed)
|
||
}
|
||
if strings.Contains(store.logAtAttach, "takeover claimed") {
|
||
t.Fatalf("认领行在 attach 之前就写了（未认领就宣称认领）：\n%s", store.logAtAttach)
|
||
}
|
||
line := claimLineOf(t, buf.String())
|
||
kv := kvOf(t, line)
|
||
for key, want := range map[string]string{
|
||
"kind": string(governance.LibraryJobRestore),
|
||
"scope": string(job.Scope),
|
||
"target": job.TargetSnapshotID,
|
||
"backup": job.SnapshotID,
|
||
"queue": governance.QueueJobIDFor(job.ID),
|
||
"path": takeoverPathEnqueue,
|
||
} {
|
||
if kv[key] != want {
|
||
t.Fatalf("认领行 %s=%q, want %q（整行：%s）", key, kv[key], want, line)
|
||
}
|
||
}
|
||
if kv["at"] == "" {
|
||
t.Fatalf("认领行没有 at= 时刻：%s", line)
|
||
}
|
||
}
|
||
|
||
func TestTakeoverClaimLoggedWhenQueueJobAlreadyExists(t *testing.T) {
|
||
buf := newCapture(t)
|
||
job := takeoverTestJob()
|
||
// 上一次崩溃前队列行就在：Enqueue 撞幂等键（duplicate key），走"已存在"分支。
|
||
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
|
||
enqueueErr: errors.New(`duplicate key value violates unique constraint "jobs_pkey"`)}
|
||
|
||
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
|
||
if err != nil {
|
||
t.Fatalf("resume 出错：%v", err)
|
||
}
|
||
if resumed != 1 {
|
||
t.Fatalf("resumed=%d, want 1（队列行已存在也算真认领）", resumed)
|
||
}
|
||
line := claimLineOf(t, buf.String())
|
||
if got := kvOf(t, line)["path"]; got != takeoverPathExistingQueue {
|
||
t.Fatalf("path=%q, want %q（整行：%s）", got, takeoverPathExistingQueue, line)
|
||
}
|
||
if !strings.Contains(line, job.ID) {
|
||
t.Fatalf("认领行没写作业 id：%s", line)
|
||
}
|
||
}
|
||
|
||
func TestTakeoverClaimNotLoggedWhenNotClaimed(t *testing.T) {
|
||
buf := newCapture(t)
|
||
job := takeoverTestJob()
|
||
// 另一个 worker 已经抢先改了 running→pending：本 worker 什么都没认领到。
|
||
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: false,
|
||
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}}
|
||
|
||
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
|
||
if err != nil {
|
||
t.Fatalf("resume 出错：%v", err)
|
||
}
|
||
if resumed != 0 {
|
||
t.Fatalf("resumed=%d, want 0", resumed)
|
||
}
|
||
if strings.Contains(buf.String(), "takeover claimed") {
|
||
t.Fatalf("没认领到却写了认领行：\n%s", buf.String())
|
||
}
|
||
}
|
||
|
||
func TestTakeoverClaimNotLoggedWhenAttachFails(t *testing.T) {
|
||
buf := newCapture(t)
|
||
job := takeoverTestJob()
|
||
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
|
||
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)},
|
||
attachErr: errors.New("attach failed")}
|
||
|
||
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
|
||
if err != nil {
|
||
t.Fatalf("resume 出错：%v", err)
|
||
}
|
||
if resumed != 0 {
|
||
t.Fatalf("resumed=%d, want 0（attach 失败=没认领成功）", resumed)
|
||
}
|
||
if strings.Contains(buf.String(), "takeover claimed") {
|
||
t.Fatalf("attach 失败却写了认领行：\n%s", buf.String())
|
||
}
|
||
}
|
||
|
||
func TestTakeoverDecisionLineCarriesDecisionAndIdentity(t *testing.T) {
|
||
at := time.Date(2026, 10, 29, 19, 57, 22, 0, time.UTC)
|
||
line := takeoverDecisionLine("librestore-internal-1", takeoverFresh, governance.LibraryJobRestore,
|
||
"internal", "target-snap", "backup-snap", false, at)
|
||
kv := kvOf(t, line)
|
||
for key, want := range map[string]string{
|
||
"decision": string(takeoverFresh),
|
||
"kind": string(governance.LibraryJobRestore),
|
||
"scope": "internal",
|
||
"target": "target-snap",
|
||
"backup": "backup-snap",
|
||
"finalize_only": "false",
|
||
"at": "2026-10-29T19:57:22Z",
|
||
} {
|
||
if kv[key] != want {
|
||
t.Fatalf("%s=%q, want %q（整行：%s）", key, kv[key], want, line)
|
||
}
|
||
}
|
||
if !strings.HasPrefix(line, "library job librestore-internal-1 takeover decision=") {
|
||
t.Fatalf("决策行前缀不对：%s", line)
|
||
}
|
||
if takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", true, at) ==
|
||
takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", false, at) {
|
||
t.Fatal("finalize_only 没有进决策行：两种决策文本一样")
|
||
}
|
||
}
|
||
|
||
func TestTakeoverClaimLineIsNotTheAggregateCounter(t *testing.T) {
|
||
job := takeoverTestJob()
|
||
line := takeoverClaimLine(job, "queue-1", takeoverPathEnqueue, time.Now())
|
||
// 汇总行 "library job takeover sweep: 1 resumed" 这类没有作业 id 的行，
|
||
// 判定脚本必须认不出来；反过来本行必须带着本作业 id 与目标快照。
|
||
if strings.Contains(line, "takeover sweep") {
|
||
t.Fatalf("认领行长得像汇总行：%s", line)
|
||
}
|
||
if !strings.Contains(line, job.ID) || !strings.Contains(line, "target="+job.TargetSnapshotID) {
|
||
t.Fatalf("认领行缺作业身份：%s", line)
|
||
}
|
||
}
|
||
|
||
// resume 的 config 参数当前未使用（接管决策不依赖配置），这里给零值即可。
|
||
func configForResumeTest() appconfig.Config { return appconfig.Config{} }
|