feat(worker): 接管认领与决策落结构化日志

2026-10-29T19:56Z 的 after:database 演练里,worker 只留下汇总行
`library job takeover sweep: 1 resumed, 0 failed without backup`:既不说哪个
作业,也不说它是否真被本进程认领(被别的 worker 抢走、attach 失败都只影响
计数)。演练脚本要的老文案只在"新建队列行"那条路径打印,真实走的是"队列行
已存在"那条 ⇒ 判据与实现各说一套,演练 exit 5。

- 新增 `library job <id> takeover claimed at=… kind=… scope=… target=…
  backup=… queue=… path=…`:claim + attach 都成功之后才写,两条认领路径共用
  (path=enqueue / existing-queue);
- 新增 `library job <id> takeover decision=…`:分类后、动手之前写,含
  finalize_only。只留阶段表的话,同快照收尾能把它伪造成完整续作;
- resume 的 store 参数抽成接口,单测才能证明"没认领/attach 失败时一行都不写"
  (变异验证:把认领行提到 attach 之前 ⇒ 红;删掉 existing-queue 那行 ⇒ 红);
- 接管复用的预备份在证据里写 `backup_reused=true`:分不清"复用"与"崩溃后重做
  备份",后者会把半写状态当成可回滚基线。

verify: go test ./cmd/worker -run 'TestTakeover|TestLedgerBackupStore' -count=1 全绿;
make verify overall success(16 项必需全绿)。
This commit is contained in:
姚章浩
2026-10-30 07:56:07 +08:00
parent d544728001
commit 6bdc17474c
5 changed files with 342 additions and 6 deletions
+21 -6
View File
@@ -130,7 +130,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
resumed := err == nil && previous.SnapshotID != ""
if resumed {
facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID)
switch classifyTakeover(previous, facts) {
decision := classifyTakeover(previous, facts)
// 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份:
// 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。
logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope,
payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC())
switch decision {
case takeoverUnknown:
// 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
@@ -484,7 +489,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) {
func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) {
interrupted, err := store.InterruptedLibraryJobs(ctx)
if err != nil {
return 0, 0, err
@@ -538,6 +543,7 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
} else {
resumed++
logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now)
}
continue
}
@@ -553,14 +559,19 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor
continue
}
resumed++
log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID)
// 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑",
// 外部用它判接管来源(汇总计数证明不了是哪个作业)。
logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now)
}
return resumed, failed, nil
}
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
//
// 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把
// "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行,
// 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if r.onBackup != nil {
@@ -572,12 +583,16 @@ func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup library
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
}
evidence := map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
"job": r.jobID, "index_mapping_sha256": r.mappingDigest}
if backup.Reused {
evidence["backup_reused"] = true
}
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
"job": r.jobID, "index_mapping_sha256": r.mappingDigest},
Evidence: evidence,
CreatedAt: now,
}); err != nil {
return fmt.Errorf("登记预备份版本失败: %w", err)
+63
View File
@@ -0,0 +1,63 @@
package main
import (
"context"
"fmt"
"log"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// libraryResumeStore 是"启动接管"用到的存储动作。
//
// 为什么抽成接口:日志必须证明"这个作业真的被本 worker 认领了",而不是
// "扫描循环数了一下"。只有单测能控制 claim/attach 的返回时,才验证得了
// "认领失败时一行都不许写"(2026-10-29 的演练现场就是只有汇总行
// `library job takeover sweep: 1 resumed`,外部无法判断认领的是哪个作业)。
type libraryResumeStore interface {
InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error)
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState,
stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error)
Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error)
ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error
AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error
}
// 认领路径:两条都是"真认领",差别只在队列行是新建的、还是上一次崩溃前就存在的。
// 必须写进日志:上一轮只有新建那一支打日志,"队列行已存在"那一支只把计数加一,
// 于是外部只看到汇总,无法证明是哪个作业、也无从证明它被认领过。
const (
takeoverPathEnqueue = "enqueue"
takeoverPathExistingQueue = "existing-queue"
)
// takeoverClaimLine 是"作业被本 worker 认领"的唯一证据行(一行、字段全 key=value)。
// 判据脚本按字段名取值,不按位置猜;改字段名会让演练直接红,而不是静默放过。
func takeoverClaimLine(job governance.LibraryJob, queueJobID, path string, at time.Time) string {
return fmt.Sprintf(
"library job %s takeover claimed at=%s kind=%s scope=%s target=%s backup=%s queue=%s path=%s",
job.ID, at.UTC().Format(time.RFC3339Nano), job.Kind, job.Scope,
job.TargetSnapshotID, job.SnapshotID, queueJobID, path)
}
// takeoverDecisionLine 说明"接管时判定要怎么做":fresh=整条写入链路重跑;
// pending_final=只复核与收尾;superseded / foreign_effective / unknown=拒绝接管。
// 没有这一行,事后只能从阶段表倒推决策 —— 而阶段表正是同快照收尾能伪造的地方。
func takeoverDecisionLine(jobID string, decision takeoverKind, kind governance.LibraryJobKind,
scope, target, backup string, finalizeOnly bool, at time.Time) string {
return fmt.Sprintf(
"library job %s takeover decision=%s at=%s kind=%s scope=%s target=%s backup=%s finalize_only=%t",
jobID, decision, at.UTC().Format(time.RFC3339Nano), kind, scope, target, backup, finalizeOnly)
}
func logTakeoverClaim(job governance.LibraryJob, queueJobID, path string, at time.Time) {
log.Print(takeoverClaimLine(job, queueJobID, path, at))
}
func logTakeoverDecision(jobID string, decision takeoverKind, kind governance.LibraryJobKind,
scope, target, backup string, finalizeOnly bool, at time.Time) {
log.Print(takeoverDecisionLine(jobID, decision, kind, scope, target, backup, finalizeOnly, at))
}
+250
View File
@@ -0,0 +1,250 @@
package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"testing"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// 这些用例守的是"接管证据"的契约,不是实现细节:
// 只有真的把作业认领到自己名下(claim + attach 都成功)才允许写认领行;
// 队列行已存在这条路径也必须写(上一轮只加了计数,外部只看到汇总,判不了是哪个作业)。
type fakeResumeStore struct {
interrupted []governance.LibraryJob
claimOK bool
claimErr error
enqueueJob *jobs.Job
enqueueErr error
attachErr error
calls []string
logAtAttach string // attach 那一刻的日志内容:用来断言"先认领、后写日志"
}
func (f *fakeResumeStore) InterruptedLibraryJobs(context.Context) ([]governance.LibraryJob, error) {
f.calls = append(f.calls, "interrupted")
return f.interrupted, nil
}
func (f *fakeResumeStore) AdvanceLibraryJob(context.Context, string, governance.LibraryJobState,
string, string, string, string, string, time.Time) error {
f.calls = append(f.calls, "advance")
return nil
}
func (f *fakeResumeStore) ClaimLibraryJobForResume(context.Context, string, time.Time) (bool, error) {
f.calls = append(f.calls, "claim")
return f.claimOK, f.claimErr
}
func (f *fakeResumeStore) Enqueue(context.Context, jobs.EnqueueRequest, time.Time) (*jobs.Job, error) {
f.calls = append(f.calls, "enqueue")
return f.enqueueJob, f.enqueueErr
}
func (f *fakeResumeStore) ResetQueueJobForResume(context.Context, string, time.Time) error {
f.calls = append(f.calls, "reset")
return nil
}
func (f *fakeResumeStore) AttachLibraryQueueJob(context.Context, string, string, time.Time) error {
f.calls = append(f.calls, "attach")
f.logAtAttach = captureLogAtAttach()
return f.attachErr
}
// captureLogAtAttach 由用例在运行时替换(见 newCapture)。
var captureLogAtAttach = func() string { return "" }
func takeoverTestJob() governance.LibraryJob {
return governance.LibraryJob{
ID: "librestore-internal-1793303791259342087", Scope: "internal",
Kind: governance.LibraryJobRestore,
State: governance.LibraryJobRunning,
SnapshotID: "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510",
TargetSnapshotID: "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7",
}
}
// newCapture 把标准日志接到缓冲区,并在 attach 回调里取当时的快照。
func newCapture(t *testing.T) *bytes.Buffer {
t.Helper()
buf := &bytes.Buffer{}
old := log.Writer()
log.SetOutput(buf)
captureLogAtAttach = func() string { return buf.String() }
t.Cleanup(func() {
log.SetOutput(old)
captureLogAtAttach = func() string { return "" }
})
return buf
}
func claimLineOf(t *testing.T, logs string) string {
t.Helper()
for _, line := range strings.Split(logs, "\n") {
if strings.Contains(line, "takeover claimed") {
return line
}
}
t.Fatalf("日志里没有本作业的认领行:\n%s", logs)
return ""
}
func kvOf(t *testing.T, line string) map[string]string {
t.Helper()
fields := strings.Fields(line)
out := map[string]string{}
for _, f := range fields {
if i := strings.Index(f, "="); i > 0 {
out[f[:i]] = f[i+1:]
}
}
return out
}
func TestTakeoverClaimLoggedAfterSuccessfulAttach(t *testing.T) {
buf := newCapture(t)
job := takeoverTestJob()
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}}
resumed, failed, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
if err != nil || failed != 0 {
t.Fatalf("resume 失败:resumed=%d failed=%d err=%v", resumed, failed, err)
}
if resumed != 1 {
t.Fatalf("resumed=%d, want 1", resumed)
}
if strings.Contains(store.logAtAttach, "takeover claimed") {
t.Fatalf("认领行在 attach 之前就写了(未认领就宣称认领):\n%s", store.logAtAttach)
}
line := claimLineOf(t, buf.String())
kv := kvOf(t, line)
for key, want := range map[string]string{
"kind": string(governance.LibraryJobRestore),
"scope": string(job.Scope),
"target": job.TargetSnapshotID,
"backup": job.SnapshotID,
"queue": governance.QueueJobIDFor(job.ID),
"path": takeoverPathEnqueue,
} {
if kv[key] != want {
t.Fatalf("认领行 %s=%q, want %q(整行:%s)", key, kv[key], want, line)
}
}
if kv["at"] == "" {
t.Fatalf("认领行没有 at= 时刻:%s", line)
}
}
func TestTakeoverClaimLoggedWhenQueueJobAlreadyExists(t *testing.T) {
buf := newCapture(t)
job := takeoverTestJob()
// 上一次崩溃前队列行就在:Enqueue 撞幂等键(duplicate key),走"已存在"分支。
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
enqueueErr: errors.New(`duplicate key value violates unique constraint "jobs_pkey"`)}
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
if err != nil {
t.Fatalf("resume 出错:%v", err)
}
if resumed != 1 {
t.Fatalf("resumed=%d, want 1(队列行已存在也算真认领)", resumed)
}
line := claimLineOf(t, buf.String())
if got := kvOf(t, line)["path"]; got != takeoverPathExistingQueue {
t.Fatalf("path=%q, want %q(整行:%s)", got, takeoverPathExistingQueue, line)
}
if !strings.Contains(line, job.ID) {
t.Fatalf("认领行没写作业 id:%s", line)
}
}
func TestTakeoverClaimNotLoggedWhenNotClaimed(t *testing.T) {
buf := newCapture(t)
job := takeoverTestJob()
// 另一个 worker 已经抢先改了 running→pending:本 worker 什么都没认领到。
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: false,
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}}
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
if err != nil {
t.Fatalf("resume 出错:%v", err)
}
if resumed != 0 {
t.Fatalf("resumed=%d, want 0", resumed)
}
if strings.Contains(buf.String(), "takeover claimed") {
t.Fatalf("没认领到却写了认领行:\n%s", buf.String())
}
}
func TestTakeoverClaimNotLoggedWhenAttachFails(t *testing.T) {
buf := newCapture(t)
job := takeoverTestJob()
store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true,
enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)},
attachErr: errors.New("attach failed")}
resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest())
if err != nil {
t.Fatalf("resume 出错:%v", err)
}
if resumed != 0 {
t.Fatalf("resumed=%d, want 0(attach 失败=没认领成功)", resumed)
}
if strings.Contains(buf.String(), "takeover claimed") {
t.Fatalf("attach 失败却写了认领行:\n%s", buf.String())
}
}
func TestTakeoverDecisionLineCarriesDecisionAndIdentity(t *testing.T) {
at := time.Date(2026, 10, 29, 19, 57, 22, 0, time.UTC)
line := takeoverDecisionLine("librestore-internal-1", takeoverFresh, governance.LibraryJobRestore,
"internal", "target-snap", "backup-snap", false, at)
kv := kvOf(t, line)
for key, want := range map[string]string{
"decision": string(takeoverFresh),
"kind": string(governance.LibraryJobRestore),
"scope": "internal",
"target": "target-snap",
"backup": "backup-snap",
"finalize_only": "false",
"at": "2026-10-29T19:57:22Z",
} {
if kv[key] != want {
t.Fatalf("%s=%q, want %q(整行:%s)", key, kv[key], want, line)
}
}
if !strings.HasPrefix(line, "library job librestore-internal-1 takeover decision=") {
t.Fatalf("决策行前缀不对:%s", line)
}
if takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", true, at) ==
takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", false, at) {
t.Fatal("finalize_only 没有进决策行:两种决策文本一样")
}
}
func TestTakeoverClaimLineIsNotTheAggregateCounter(t *testing.T) {
job := takeoverTestJob()
line := takeoverClaimLine(job, "queue-1", takeoverPathEnqueue, time.Now())
// 汇总行 "library job takeover sweep: 1 resumed" 这类没有作业 id 的行,
// 判定脚本必须认不出来;反过来本行必须带着本作业 id 与目标快照。
if strings.Contains(line, "takeover sweep") {
t.Fatalf("认领行长得像汇总行:%s", line)
}
if !strings.Contains(line, job.ID) || !strings.Contains(line, "target="+job.TargetSnapshotID) {
t.Fatalf("认领行缺作业身份:%s", line)
}
}
// resume 的 config 参数当前未使用(接管决策不依赖配置),这里给零值即可。
func configForResumeTest() appconfig.Config { return appconfig.Config{} }
+3
View File
@@ -128,6 +128,9 @@ func (s ledgerBackupStore) Prepare(context.Context, string, libraryrelease.Curre
}
ref := s.ref
ref.WriteSet = s.write
// 明确的"复用":调用方据此把 backup_reused 落进版本证据,而不是让外部
// 从 created_at 变了没变去猜(那正是被同快照收尾伪造过的地方)。
ref.Reused = true
return ref, nil
}
+5
View File
@@ -87,6 +87,11 @@ type BackupRef struct {
// WriteSet 是本次发布的写集合与整实例内容证据。它在写第一笔之前就落盘并
// 登记摘要;恢复必须按它清理,缺了它恢复要失败(不许降级成"跳过比对")。
WriteSet *WriteSet `json:"write_set,omitempty"`
// Reused 表示"这是接管时复用上一次的预备份,没有重新备份"。
// 崩溃接管绝不允许重新备份:那一刻目标已被写过,新备份会把半写状态当成
// 可回滚基线,覆盖真正干净的那一份。外部要能区分"复用"与"重做",
// 所以复用必须显式落到版本证据里。
Reused bool `json:"reused,omitempty"`
}
type Target interface {