姚章浩
6bdc17474c
2026-10-29T19:56Z 的 after:database 演练里，worker 只留下汇总行 `library job takeover sweep: 1 resumed, 0 failed without backup`：既不说哪个 作业，也不说它是否真被本进程认领（被别的 worker 抢走、attach 失败都只影响 计数）。演练脚本要的老文案只在"新建队列行"那条路径打印，真实走的是"队列行 已存在"那条 ⇒ 判据与实现各说一套，演练 exit 5。 - 新增 `library job <id> takeover claimed at=… kind=… scope=… target=… backup=… queue=… path=…`：claim + attach 都成功之后才写，两条认领路径共用 （path=enqueue / existing-queue）； - 新增 `library job <id> takeover decision=…`：分类后、动手之前写，含 finalize_only。只留阶段表的话，同快照收尾能把它伪造成完整续作； - resume 的 store 参数抽成接口，单测才能证明"没认领/attach 失败时一行都不写" （变异验证：把认领行提到 attach 之前 ⇒ 红；删掉 existing-queue 那行 ⇒ 红）； - 接管复用的预备份在证据里写 `backup_reused=true`：分不清"复用"与"崩溃后重做 备份"，后者会把半写状态当成可回滚基线。 verify: go test ./cmd/worker -run 'TestTakeover|TestLedgerBackupStore' -count=1 全绿； make verify overall success（16 项必需全绿）。
64 lines
3.3 KiB
Go
64 lines
3.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
"knowledge-governance/internal/jobs"
|
|
)
|
|
|
|
// libraryResumeStore 是"启动接管"用到的存储动作。
|
|
//
|
|
// 为什么抽成接口：日志必须证明"这个作业真的被本 worker 认领了"，而不是
|
|
// "扫描循环数了一下"。只有单测能控制 claim/attach 的返回时，才验证得了
|
|
// "认领失败时一行都不许写"（2026-10-29 的演练现场就是只有汇总行
|
|
// `library job takeover sweep: 1 resumed`，外部无法判断认领的是哪个作业）。
|
|
type libraryResumeStore interface {
|
|
InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error)
|
|
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState,
|
|
stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
|
ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error)
|
|
Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error)
|
|
ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error
|
|
AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error
|
|
}
|
|
|
|
// 认领路径：两条都是"真认领"，差别只在队列行是新建的、还是上一次崩溃前就存在的。
|
|
// 必须写进日志：上一轮只有新建那一支打日志，"队列行已存在"那一支只把计数加一，
|
|
// 于是外部只看到汇总，无法证明是哪个作业、也无从证明它被认领过。
|
|
const (
|
|
takeoverPathEnqueue = "enqueue"
|
|
takeoverPathExistingQueue = "existing-queue"
|
|
)
|
|
|
|
// takeoverClaimLine 是"作业被本 worker 认领"的唯一证据行（一行、字段全 key=value）。
|
|
// 判据脚本按字段名取值，不按位置猜；改字段名会让演练直接红，而不是静默放过。
|
|
func takeoverClaimLine(job governance.LibraryJob, queueJobID, path string, at time.Time) string {
|
|
return fmt.Sprintf(
|
|
"library job %s takeover claimed at=%s kind=%s scope=%s target=%s backup=%s queue=%s path=%s",
|
|
job.ID, at.UTC().Format(time.RFC3339Nano), job.Kind, job.Scope,
|
|
job.TargetSnapshotID, job.SnapshotID, queueJobID, path)
|
|
}
|
|
|
|
// takeoverDecisionLine 说明"接管时判定要怎么做"：fresh=整条写入链路重跑；
|
|
// pending_final=只复核与收尾；superseded / foreign_effective / unknown=拒绝接管。
|
|
// 没有这一行，事后只能从阶段表倒推决策 —— 而阶段表正是同快照收尾能伪造的地方。
|
|
func takeoverDecisionLine(jobID string, decision takeoverKind, kind governance.LibraryJobKind,
|
|
scope, target, backup string, finalizeOnly bool, at time.Time) string {
|
|
return fmt.Sprintf(
|
|
"library job %s takeover decision=%s at=%s kind=%s scope=%s target=%s backup=%s finalize_only=%t",
|
|
jobID, decision, at.UTC().Format(time.RFC3339Nano), kind, scope, target, backup, finalizeOnly)
|
|
}
|
|
|
|
func logTakeoverClaim(job governance.LibraryJob, queueJobID, path string, at time.Time) {
|
|
log.Print(takeoverClaimLine(job, queueJobID, path, at))
|
|
}
|
|
|
|
func logTakeoverDecision(jobID string, decision takeoverKind, kind governance.LibraryJobKind,
|
|
scope, target, backup string, finalizeOnly bool, at time.Time) {
|
|
log.Print(takeoverDecisionLine(jobID, decision, kind, scope, target, backup, finalizeOnly, at))
|
|
}
|