From 6bdc17474c344fd8f343363515fa6ee67bd936c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Fri, 30 Oct 2026 07:56:07 +0800 Subject: [PATCH] =?UTF-8?q?feat(worker):=20=E6=8E=A5=E7=AE=A1=E8=AE=A4?= =?UTF-8?q?=E9=A2=86=E4=B8=8E=E5=86=B3=E7=AD=96=E8=90=BD=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=8C=96=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-10-29T19:56Z 的 after:database 演练里,worker 只留下汇总行 `library job takeover sweep: 1 resumed, 0 failed without backup`:既不说哪个 作业,也不说它是否真被本进程认领(被别的 worker 抢走、attach 失败都只影响 计数)。演练脚本要的老文案只在"新建队列行"那条路径打印,真实走的是"队列行 已存在"那条 ⇒ 判据与实现各说一套,演练 exit 5。 - 新增 `library job takeover claimed at=… kind=… scope=… target=… backup=… queue=… path=…`:claim + attach 都成功之后才写,两条认领路径共用 (path=enqueue / existing-queue); - 新增 `library job takeover decision=…`:分类后、动手之前写,含 finalize_only。只留阶段表的话,同快照收尾能把它伪造成完整续作; - resume 的 store 参数抽成接口,单测才能证明"没认领/attach 失败时一行都不写" (变异验证:把认领行提到 attach 之前 ⇒ 红;删掉 existing-queue 那行 ⇒ 红); - 接管复用的预备份在证据里写 `backup_reused=true`:分不清"复用"与"崩溃后重做 备份",后者会把半写状态当成可回滚基线。 verify: go test ./cmd/worker -run 'TestTakeover|TestLedgerBackupStore' -count=1 全绿; make verify overall success(16 项必需全绿)。 --- cmd/worker/library_handlers.go | 27 ++- cmd/worker/library_takeover_log.go | 63 ++++++ cmd/worker/library_takeover_log_test.go | 250 ++++++++++++++++++++++++ cmd/worker/library_writeset.go | 3 + internal/libraryrelease/release.go | 5 + 5 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 cmd/worker/library_takeover_log.go create mode 100644 cmd/worker/library_takeover_log_test.go diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 095bfb5..2701402 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -130,7 +130,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a resumed := err == nil && previous.SnapshotID != "" if resumed { facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID) - switch classifyTakeover(previous, facts) { + decision := classifyTakeover(previous, facts) + // 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份: + // 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。 + logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope, + payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC()) + switch decision { case takeoverUnknown: // 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。 return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, @@ -484,7 +489,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 -func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) { +func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) { interrupted, err := store.InterruptedLibraryJobs(ctx) if err != nil { return 0, 0, err @@ -538,6 +543,7 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr) } else { resumed++ + logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now) } continue } @@ -553,14 +559,19 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor continue } resumed++ - log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID) + // 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑", + // 外部用它判接管来源(汇总计数证明不了是哪个作业)。 + logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now) } return resumed, failed, nil } -// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。 // BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误: // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 +// +// 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把 +// "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行, +// 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { now := time.Now().UTC() if r.onBackup != nil { @@ -572,12 +583,16 @@ func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup library backup.SnapshotID, "", now); err != nil { return fmt.Errorf("持久化预备份引用失败: %w", err) } + evidence := map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, + "job": r.jobID, "index_mapping_sha256": r.mappingDigest} + if backup.Reused { + evidence["backup_reused"] = true + } if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID), Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: r.jobID, CreatedBy: "worker", - Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, - "job": r.jobID, "index_mapping_sha256": r.mappingDigest}, + Evidence: evidence, CreatedAt: now, }); err != nil { return fmt.Errorf("登记预备份版本失败: %w", err) diff --git a/cmd/worker/library_takeover_log.go b/cmd/worker/library_takeover_log.go new file mode 100644 index 0000000..478b4a8 --- /dev/null +++ b/cmd/worker/library_takeover_log.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// libraryResumeStore 是"启动接管"用到的存储动作。 +// +// 为什么抽成接口:日志必须证明"这个作业真的被本 worker 认领了",而不是 +// "扫描循环数了一下"。只有单测能控制 claim/attach 的返回时,才验证得了 +// "认领失败时一行都不许写"(2026-10-29 的演练现场就是只有汇总行 +// `library job takeover sweep: 1 resumed`,外部无法判断认领的是哪个作业)。 +type libraryResumeStore interface { + InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) + AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, + stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error + ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) + Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error) + ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error + AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error +} + +// 认领路径:两条都是"真认领",差别只在队列行是新建的、还是上一次崩溃前就存在的。 +// 必须写进日志:上一轮只有新建那一支打日志,"队列行已存在"那一支只把计数加一, +// 于是外部只看到汇总,无法证明是哪个作业、也无从证明它被认领过。 +const ( + takeoverPathEnqueue = "enqueue" + takeoverPathExistingQueue = "existing-queue" +) + +// takeoverClaimLine 是"作业被本 worker 认领"的唯一证据行(一行、字段全 key=value)。 +// 判据脚本按字段名取值,不按位置猜;改字段名会让演练直接红,而不是静默放过。 +func takeoverClaimLine(job governance.LibraryJob, queueJobID, path string, at time.Time) string { + return fmt.Sprintf( + "library job %s takeover claimed at=%s kind=%s scope=%s target=%s backup=%s queue=%s path=%s", + job.ID, at.UTC().Format(time.RFC3339Nano), job.Kind, job.Scope, + job.TargetSnapshotID, job.SnapshotID, queueJobID, path) +} + +// takeoverDecisionLine 说明"接管时判定要怎么做":fresh=整条写入链路重跑; +// pending_final=只复核与收尾;superseded / foreign_effective / unknown=拒绝接管。 +// 没有这一行,事后只能从阶段表倒推决策 —— 而阶段表正是同快照收尾能伪造的地方。 +func takeoverDecisionLine(jobID string, decision takeoverKind, kind governance.LibraryJobKind, + scope, target, backup string, finalizeOnly bool, at time.Time) string { + return fmt.Sprintf( + "library job %s takeover decision=%s at=%s kind=%s scope=%s target=%s backup=%s finalize_only=%t", + jobID, decision, at.UTC().Format(time.RFC3339Nano), kind, scope, target, backup, finalizeOnly) +} + +func logTakeoverClaim(job governance.LibraryJob, queueJobID, path string, at time.Time) { + log.Print(takeoverClaimLine(job, queueJobID, path, at)) +} + +func logTakeoverDecision(jobID string, decision takeoverKind, kind governance.LibraryJobKind, + scope, target, backup string, finalizeOnly bool, at time.Time) { + log.Print(takeoverDecisionLine(jobID, decision, kind, scope, target, backup, finalizeOnly, at)) +} diff --git a/cmd/worker/library_takeover_log_test.go b/cmd/worker/library_takeover_log_test.go new file mode 100644 index 0000000..2aac82d --- /dev/null +++ b/cmd/worker/library_takeover_log_test.go @@ -0,0 +1,250 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// 这些用例守的是"接管证据"的契约,不是实现细节: +// 只有真的把作业认领到自己名下(claim + attach 都成功)才允许写认领行; +// 队列行已存在这条路径也必须写(上一轮只加了计数,外部只看到汇总,判不了是哪个作业)。 +type fakeResumeStore struct { + interrupted []governance.LibraryJob + claimOK bool + claimErr error + enqueueJob *jobs.Job + enqueueErr error + attachErr error + calls []string + logAtAttach string // attach 那一刻的日志内容:用来断言"先认领、后写日志" +} + +func (f *fakeResumeStore) InterruptedLibraryJobs(context.Context) ([]governance.LibraryJob, error) { + f.calls = append(f.calls, "interrupted") + return f.interrupted, nil +} + +func (f *fakeResumeStore) AdvanceLibraryJob(context.Context, string, governance.LibraryJobState, + string, string, string, string, string, time.Time) error { + f.calls = append(f.calls, "advance") + return nil +} + +func (f *fakeResumeStore) ClaimLibraryJobForResume(context.Context, string, time.Time) (bool, error) { + f.calls = append(f.calls, "claim") + return f.claimOK, f.claimErr +} + +func (f *fakeResumeStore) Enqueue(context.Context, jobs.EnqueueRequest, time.Time) (*jobs.Job, error) { + f.calls = append(f.calls, "enqueue") + return f.enqueueJob, f.enqueueErr +} + +func (f *fakeResumeStore) ResetQueueJobForResume(context.Context, string, time.Time) error { + f.calls = append(f.calls, "reset") + return nil +} + +func (f *fakeResumeStore) AttachLibraryQueueJob(context.Context, string, string, time.Time) error { + f.calls = append(f.calls, "attach") + f.logAtAttach = captureLogAtAttach() + return f.attachErr +} + +// captureLogAtAttach 由用例在运行时替换(见 newCapture)。 +var captureLogAtAttach = func() string { return "" } + +func takeoverTestJob() governance.LibraryJob { + return governance.LibraryJob{ + ID: "librestore-internal-1793303791259342087", Scope: "internal", + Kind: governance.LibraryJobRestore, + State: governance.LibraryJobRunning, + SnapshotID: "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510", + TargetSnapshotID: "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7", + } +} + +// newCapture 把标准日志接到缓冲区,并在 attach 回调里取当时的快照。 +func newCapture(t *testing.T) *bytes.Buffer { + t.Helper() + buf := &bytes.Buffer{} + old := log.Writer() + log.SetOutput(buf) + captureLogAtAttach = func() string { return buf.String() } + t.Cleanup(func() { + log.SetOutput(old) + captureLogAtAttach = func() string { return "" } + }) + return buf +} + +func claimLineOf(t *testing.T, logs string) string { + t.Helper() + for _, line := range strings.Split(logs, "\n") { + if strings.Contains(line, "takeover claimed") { + return line + } + } + t.Fatalf("日志里没有本作业的认领行:\n%s", logs) + return "" +} + +func kvOf(t *testing.T, line string) map[string]string { + t.Helper() + fields := strings.Fields(line) + out := map[string]string{} + for _, f := range fields { + if i := strings.Index(f, "="); i > 0 { + out[f[:i]] = f[i+1:] + } + } + return out +} + +func TestTakeoverClaimLoggedAfterSuccessfulAttach(t *testing.T) { + buf := newCapture(t) + job := takeoverTestJob() + store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true, + enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}} + + resumed, failed, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest()) + if err != nil || failed != 0 { + t.Fatalf("resume 失败:resumed=%d failed=%d err=%v", resumed, failed, err) + } + if resumed != 1 { + t.Fatalf("resumed=%d, want 1", resumed) + } + if strings.Contains(store.logAtAttach, "takeover claimed") { + t.Fatalf("认领行在 attach 之前就写了(未认领就宣称认领):\n%s", store.logAtAttach) + } + line := claimLineOf(t, buf.String()) + kv := kvOf(t, line) + for key, want := range map[string]string{ + "kind": string(governance.LibraryJobRestore), + "scope": string(job.Scope), + "target": job.TargetSnapshotID, + "backup": job.SnapshotID, + "queue": governance.QueueJobIDFor(job.ID), + "path": takeoverPathEnqueue, + } { + if kv[key] != want { + t.Fatalf("认领行 %s=%q, want %q(整行:%s)", key, kv[key], want, line) + } + } + if kv["at"] == "" { + t.Fatalf("认领行没有 at= 时刻:%s", line) + } +} + +func TestTakeoverClaimLoggedWhenQueueJobAlreadyExists(t *testing.T) { + buf := newCapture(t) + job := takeoverTestJob() + // 上一次崩溃前队列行就在:Enqueue 撞幂等键(duplicate key),走"已存在"分支。 + store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true, + enqueueErr: errors.New(`duplicate key value violates unique constraint "jobs_pkey"`)} + + resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest()) + if err != nil { + t.Fatalf("resume 出错:%v", err) + } + if resumed != 1 { + t.Fatalf("resumed=%d, want 1(队列行已存在也算真认领)", resumed) + } + line := claimLineOf(t, buf.String()) + if got := kvOf(t, line)["path"]; got != takeoverPathExistingQueue { + t.Fatalf("path=%q, want %q(整行:%s)", got, takeoverPathExistingQueue, line) + } + if !strings.Contains(line, job.ID) { + t.Fatalf("认领行没写作业 id:%s", line) + } +} + +func TestTakeoverClaimNotLoggedWhenNotClaimed(t *testing.T) { + buf := newCapture(t) + job := takeoverTestJob() + // 另一个 worker 已经抢先改了 running→pending:本 worker 什么都没认领到。 + store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: false, + enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}} + + resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest()) + if err != nil { + t.Fatalf("resume 出错:%v", err) + } + if resumed != 0 { + t.Fatalf("resumed=%d, want 0", resumed) + } + if strings.Contains(buf.String(), "takeover claimed") { + t.Fatalf("没认领到却写了认领行:\n%s", buf.String()) + } +} + +func TestTakeoverClaimNotLoggedWhenAttachFails(t *testing.T) { + buf := newCapture(t) + job := takeoverTestJob() + store := &fakeResumeStore{interrupted: []governance.LibraryJob{job}, claimOK: true, + enqueueJob: &jobs.Job{ID: governance.QueueJobIDFor(job.ID)}, + attachErr: errors.New("attach failed")} + + resumed, _, err := resumeInterruptedLibraryJobs(context.Background(), store, configForResumeTest()) + if err != nil { + t.Fatalf("resume 出错:%v", err) + } + if resumed != 0 { + t.Fatalf("resumed=%d, want 0(attach 失败=没认领成功)", resumed) + } + if strings.Contains(buf.String(), "takeover claimed") { + t.Fatalf("attach 失败却写了认领行:\n%s", buf.String()) + } +} + +func TestTakeoverDecisionLineCarriesDecisionAndIdentity(t *testing.T) { + at := time.Date(2026, 10, 29, 19, 57, 22, 0, time.UTC) + line := takeoverDecisionLine("librestore-internal-1", takeoverFresh, governance.LibraryJobRestore, + "internal", "target-snap", "backup-snap", false, at) + kv := kvOf(t, line) + for key, want := range map[string]string{ + "decision": string(takeoverFresh), + "kind": string(governance.LibraryJobRestore), + "scope": "internal", + "target": "target-snap", + "backup": "backup-snap", + "finalize_only": "false", + "at": "2026-10-29T19:57:22Z", + } { + if kv[key] != want { + t.Fatalf("%s=%q, want %q(整行:%s)", key, kv[key], want, line) + } + } + if !strings.HasPrefix(line, "library job librestore-internal-1 takeover decision=") { + t.Fatalf("决策行前缀不对:%s", line) + } + if takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", true, at) == + takeoverDecisionLine("j", takeoverPendingFinal, governance.LibraryJobRestore, "internal", "t", "b", false, at) { + t.Fatal("finalize_only 没有进决策行:两种决策文本一样") + } +} + +func TestTakeoverClaimLineIsNotTheAggregateCounter(t *testing.T) { + job := takeoverTestJob() + line := takeoverClaimLine(job, "queue-1", takeoverPathEnqueue, time.Now()) + // 汇总行 "library job takeover sweep: 1 resumed" 这类没有作业 id 的行, + // 判定脚本必须认不出来;反过来本行必须带着本作业 id 与目标快照。 + if strings.Contains(line, "takeover sweep") { + t.Fatalf("认领行长得像汇总行:%s", line) + } + if !strings.Contains(line, job.ID) || !strings.Contains(line, "target="+job.TargetSnapshotID) { + t.Fatalf("认领行缺作业身份:%s", line) + } +} + +// resume 的 config 参数当前未使用(接管决策不依赖配置),这里给零值即可。 +func configForResumeTest() appconfig.Config { return appconfig.Config{} } diff --git a/cmd/worker/library_writeset.go b/cmd/worker/library_writeset.go index b5dc6ad..e9de3c3 100644 --- a/cmd/worker/library_writeset.go +++ b/cmd/worker/library_writeset.go @@ -128,6 +128,9 @@ func (s ledgerBackupStore) Prepare(context.Context, string, libraryrelease.Curre } ref := s.ref ref.WriteSet = s.write + // 明确的"复用":调用方据此把 backup_reused 落进版本证据,而不是让外部 + // 从 created_at 变了没变去猜(那正是被同快照收尾伪造过的地方)。 + ref.Reused = true return ref, nil } diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 1a56e73..9c69d78 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -87,6 +87,11 @@ type BackupRef struct { // WriteSet 是本次发布的写集合与整实例内容证据。它在写第一笔之前就落盘并 // 登记摘要;恢复必须按它清理,缺了它恢复要失败(不许降级成"跳过比对")。 WriteSet *WriteSet `json:"write_set,omitempty"` + // Reused 表示"这是接管时复用上一次的预备份,没有重新备份"。 + // 崩溃接管绝不允许重新备份:那一刻目标已被写过,新备份会把半写状态当成 + // 可回滚基线,覆盖真正干净的那一份。外部要能区分"复用"与"重做", + // 所以复用必须显式落到版本证据里。 + Reused bool `json:"reused,omitempty"` } type Target interface {