Files
knowledge-governance/cmd/worker/library_handlers.go
T
姚章浩 a7d643635a
build / build (push) Successful in 1m27s
fix(library): 守卫按行取作业身份、同作业多代次、原作业续作与执行域准入
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2,
导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的
owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。

R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引
(job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与
operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、
新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律
ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。

R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式
结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。

R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release
路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。

同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点
改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、
恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。

测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例;
go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
2026-10-30 17:20:40 +08:00

734 lines
39 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
"knowledge-governance/internal/pgdsn"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/testbarrier"
)
// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放:
// - contentUntouched:这次窗口还没写过目标(可以安全开放)
// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放)
// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护)
const (
contentUntouched = ""
contentSucceeded = "succeeded"
contentRolledBack = "rolled_back"
contentHalfWritten = "half_written"
contentRollbackFails = "rollback_failed"
)
// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
//
// 另外它是本机中断演练的固定点位:每个阶段写完、下一个阶段还没开始时调用
// testbarrier.Wait(默认关闭,两个环境变量同时在场才生效)。放在这一层而不是
// 发布代码里,是因为"阶段之间"正是真实半写状态的边界,而发布链路本身不需要知道演练。
type contentTrackedTarget struct {
inner libraryrelease.Target
jobID string
onApply func()
}
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
return t.inner.Current(ctx)
}
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
t.onApply()
if err := t.inner.Apply(ctx, stage, snapshotID); err != nil {
return err
}
// 屏障错误只记日志:演练设施坏掉不许让一次真实发布失败。
if barrierErr := testbarrier.Wait(ctx, "after:"+string(stage), t.jobID); barrierErr != nil {
log.Printf("library test barrier point=after:%s job=%s unavailable: %v", stage, t.jobID, barrierErr)
}
return nil
}
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
return t.inner.Verify(ctx, expect)
}
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
return t.inner.Restore(ctx, backup)
}
// PlanWriteSet 必须显式转发:orchestrator 是按 `target.(WriteSetPlanner)` 这个
// 类型断言决定要不要在写第一笔之前算写集合的,包一层之后断言就失败了——于是
// 台账既不落盘也不登记,恢复/接管时根本没有"这次明确写了什么"可读。
// 真实后果:2026-10-29 的两次真作业的预备份证据里都没有 writeset_digest,
// 归档根下也没有 writesets/;一旦 worker 崩在写入中间,接管必然失败。
//
// 内层不支持预检时返回硬错误,不退回"没有台账也照写":那会让恢复只能按目标
// 现状推断残渣,从而误删历史孤儿资源。
func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
planner, ok := t.inner.(libraryrelease.WriteSetPlanner)
if !ok {
return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产")
}
set, err := planner.PlanWriteSet(ctx, plan)
if err == nil {
// 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。
log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s",
set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID)
}
return set, err
}
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
type libraryReleasePayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
FinalizeOnly bool `json:"finalize_only,omitempty"`
}
// openLibraryMaintenance 定义在 library_wiring.go:维护目标的装配与目标整库栈共用
// 同一个包(internal/librarystack),不在这里再拼一份。
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config appconfig.Config,
job jobs.Job, kind governance.LibraryJobKind) (err error) {
var payload libraryReleasePayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
return fmt.Errorf("invalid library %s payload", kind)
}
// 准入必须在任何维护/目标动作之前:业务行存在、kind/scope 一致、队列关联不可伪造。
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil {
return admitErr
}
// 执行代次:这一次投递是一个确定的执行者身份。所有治理写入都走 writer
// (同名方法带 epoch 守卫),失租的旧执行从此写不进业务状态/维护锁(F13)。
writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID))
if attemptErr != nil {
return attemptErr
}
defer func() {
if err != nil {
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptFailed, err.Error())
return
}
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "")
}()
store := writer
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: store.attempt.OperationID}
// 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。
if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil {
if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil {
recorder.totals = stageSizesFrom(manifest)
}
archive.Close()
}
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
resumed := err == nil && previous.SnapshotID != ""
// 续作失败时不能写"无需恢复":上一次尝试可能已经把目标写了一半。
// 这一处按"有没有持久化预备份"统一选择失败出口,避免每个早退点各写一套文案。
fail := func(message, stage string) error {
return failLibraryJobOrTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID,
previous.SnapshotID, resumed, message, stage)
}
if resumed {
facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID)
resume := loadTakeoverResumeFacts(ctx, store, config.LibraryArchiveRoot, previous, now)
decision := classifyTakeover(previous, facts, resume)
// 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份:
// 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。
logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope,
payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC())
switch decision {
case takeoverUnknown:
// 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。
return fail("接管时读不到该目标的版本状态,无法判断是只差收尾还是要重做;本作业未继续写目标,稍后重试", "prebackup")
case takeoverSuperseded:
return fail("本快照的生效版本已被别的快照顶替,且本作业没有新的恢复授权/独占维护窗口/未变基线,拒绝用旧快照覆盖当前生产版本", "prebackup")
case takeoverForeignEffective:
// 同一个快照由别的作业发布过:那只能证明生产上有这份内容,不能证明本作业
// 写完过、更不能证明它只差收尾。按同快照收尾会把只写了第一个阶段的作业
// 标成成功,并改写版本归属(实测过的假完成)。
owner := facts.Current.SourceJobID
if facts.HasSnapshot {
owner = facts.Snapshot.SourceJobID
}
return failLibraryTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID,
"接管时目标快照 "+shortSnapshotID(payload.TargetSnapshotID)+" 的生效版本由作业 "+owner+
" 登记:本作业没有自己的完成证据,不按同快照收尾")
case takeoverPendingFinal:
// 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。
payload.FinalizeOnly = true
}
}
if resumed {
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
}
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
recovery := "未对生产执行任何写入,无需恢复"
if resumed {
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
recovery, previous.SnapshotID, "", time.Now().UTC())
return errors.New(message)
}
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
if err != nil {
return fail("无法计算请求摘要", "prebackup")
}
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if err != nil || !found {
return fail("没有与目标快照匹配的有效发布授权", "prebackup")
}
// 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己
// 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。
priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8)
if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil {
log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr)
} else {
for _, source := range sources {
priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID,
SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt})
}
}
// 生产索引映射基线:权威来源是当前生效版本(精确查),其次任何一次成功记录的映射。
// 只认"生效版本"会在恢复把版本顶替掉之后误判"索引被改动"。
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
if wireErr == nil {
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = mapping
}
}
if wireErr != nil {
return fail(wireErr.Error(), "prebackup")
}
defer target.Close()
// 容量门禁必须在**任何副作用之前**(F11):预算不足就不该进维护、不该拍备份。
// 旧实现只在存储页 GET 里算这个数,worker 从不读它,于是"预算为正但不够"照样写。
if capErr := checkLibraryCapacity(ctx, config, payload.TargetSnapshotID); capErr != nil {
return fail(capErr.Error(), "prebackup")
}
// 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。
manager, managerErr := openLibraryMaintenance(config)
if managerErr != nil {
return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
}
defer manager.Close()
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
}
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
originalState, stateErr := manager.OriginalState(ctx)
if stateErr != nil {
return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
}
identity := manager.Target()
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: identity.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if instanceErr != nil {
return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
}
_ = instanceLock
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: targetIdentity + " | " + manager.Description(),
Level: librarymaintenance.LevelInstance,
TargetFingerprint: identity.Fingerprint,
Services: identity.Writers,
IngressState: "unknown",
State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if acquireErr != nil {
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
return fail("目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
// 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把
// 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。
if resumed {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
} else {
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
}
return fail("维护协调器不可用:"+openErr.Error(), "prebackup")
}
defer control.Close()
defer coordinator.Close()
// 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前,
// 一律按半写对待,任何早退都不许重新开放目标。
contentState := contentUntouched
if resumed {
contentState = contentHalfWritten
}
pending := successBoundary{}
tracked := &contentTrackedTarget{inner: target, jobID: payload.LibraryJobID,
onApply: func() { contentState = contentHalfWritten }}
allowRestore := func(error) bool {
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
}
execute := func(ctx context.Context) error {
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
contentState = contentHalfWritten
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
contentState = contentSucceeded
mappingDigest, _ := target.Search().MappingSHA256(ctx)
// 收尾续作只补登记,不允许把首次锚点洗掉:字节数与写集合摘要从已持久化的
// 预备份证据合并进来(F09)。缺失即拒绝,不写一个"看起来完整"的空证据。
evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID,
previous.SnapshotID, payload.TargetSnapshotID, "finalize_only", mappingDigest)
if evidenceErr != nil {
message := "收尾续作缺少首次锚点,拒绝登记生效版本:" + evidenceErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留;补齐首次锚点证据后再续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: evidence,
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
// 成功要等维护锁释放完才暴露:这里只记"待收尾"。
pending = successBoundary{Ready: true, Backup: previous.SnapshotID,
Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()}
return nil
}
var backupSource libraryrelease.BackupStore = backups
if resumed {
// 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣,
// 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。
set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.LibraryJobID, previous.SnapshotID)
if ledgerErr != nil {
// 读不到就一笔都不许写:目标可能半写,恢复口径未知。
message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
recorder.Stage(ctx, "prebackup", "failed", message)
return errors.New(message)
}
// 复用不重做备份,真实字节只能从首次登记的证据里取:缺了就必须停,
// 否则生效版本证据会写成 backup_bytes:0(现场就是这样丢的)。
backupBytes, bytesErr := prebackupBytesOf(ctx, store, payload.Scope, payload.LibraryJobID, previous.SnapshotID)
if bytesErr != nil {
message := "接管时读不回预备份的真实字节数,拒绝继续写目标:" + bytesErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
recorder.Stage(ctx, "prebackup", "failed", message)
return errors.New(message)
}
backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID, Bytes: backupBytes}, write: set}
}
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
PriorSnapshots: priorSnapshots,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
contentState = contentSucceeded
mappingDigest := ""
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = mapping
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: effectiveEvidence(outcome.Backup, string(kind), mappingDigest),
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
}
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
// 内容与版本都落定了,但还不能报成功:服务健康、入口开放、两把维护锁
// 释放完成才允许暴露 succeeded(见 finishLibrarySuccess)。
pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID,
Target: payload.TargetSnapshotID, FinishedAt: finishedAt}
return nil
case "rolled_back":
contentState = contentRolledBack
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
contentState = contentRollbackFails
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
}
}
prepare := func(ctx context.Context) error {
// 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口,
// 只会把"本来坏的"记成"维护造成的"。
return manager.VerifyRunning(ctx)
}
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
return err
}
return w.MarkContentRisk(backupCtx)
}
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
// 记录失败不影响隔离本身,但必须留痕,不能假装记上了。
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
}
return "", execute(ctx)
}
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
// 终态重放(F05):journal 里已经 succeeded,本次没有再执行 work。内容/服务子协议
// 已经完成,缺的只可能是"业务收尾"(版本登记、锁释放、终态)。这时必须从**持久事实**
// 重建收尾凭据,而不是靠进程内 pending——那个指针早就随崩溃没了。
if windowErr == nil && window.State == "succeeded" && contentState != contentSucceeded {
rebuilt, rebuildErr := rebuildSuccessBoundary(ctx, store, recorder, payload, previous, kind, grant)
if rebuildErr != nil {
note := "窗口已成功但收尾凭据无法从持久事实重建,目标保持维护:" + rebuildErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
pending = rebuilt
contentState = contentSucceeded
}
// 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。
now2 := time.Now().UTC()
switch {
case window.State == "held":
note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复"
if windowErr != nil {
note += ":" + windowErr.Error()
}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
if windowErr == nil {
return errors.New(note)
}
return windowErr
case contentState == contentSucceeded:
if !pending.Ready {
// 内容状态说成功但没有收尾凭据(不可能两者都缺):按未收尾处理,保持维护。
note := "内容状态为已验证,但缺少收尾凭据,目标保持维护"
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
return errors.New(note)
}
if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope,
payload.LibraryJobID, pending); finishErr != nil {
return finishErr
}
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup)
return windowErr
case contentState == contentRolledBack:
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
}
return windowErr
default:
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未写目标,维护窗口未开启", now2); err != nil {
return fmt.Errorf("维护占用释放失败: %w", err)
}
return windowErr
}
}
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) {
interrupted, err := store.InterruptedLibraryJobs(ctx)
if err != nil {
return 0, 0, err
}
resumed, failed := 0, 0
for _, job := range interrupted {
now := time.Now().UTC()
if job.SnapshotID == "" {
message := "worker 中断时还没有完成预备份,不能盲写目标"
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行写入", "", "", now)
failed++
continue
}
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
if claimErr != nil {
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
continue
}
if !claimed {
continue
}
queueType := librarySyncJobType
switch job.Kind {
case governance.LibraryJobRestore:
queueType = libraryRestoreJobType
case governance.LibraryJobBaseline:
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
queueType = libraryBaselineJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
AvailableAt: now, MaxAttempts: 1,
IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
if err != nil && strings.Contains(err.Error(), "duplicate key") {
queueID := governance.QueueJobIDFor(job.ID)
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
} else {
resumed++
logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now)
}
continue
}
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
continue
}
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
continue
}
resumed++
// 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑",
// 外部用它判接管来源(汇总计数证明不了是哪个作业)。
logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now)
}
return resumed, failed, nil
}
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
//
// 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把
// "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行,
// 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。
//
// ⚠️ 证据必须**合并**进已有那一行,不能每次造一份新的:RecordLibraryVersion 是
// `evidence = EXCLUDED.evidence`(整份覆盖)。2026-10-30 的演练里,接管复用时
// 新造的证据只有 backup_* / job / index_mapping_sha256,把首次规划写入的
// writeset_digest/writeset_rows… 整段抹掉,第二次中断再接管就再也读不回写集合
// (verify_effective 因此从有摘要变成空)。合并只对**同一作业 + 同一份预备份**
// 成立:身份不符一律失败,绝不把别的作业的证据带进来。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if r.onBackup != nil {
if err := r.onBackup(ctx, backup); err != nil {
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
}
}
// 先读既有证据:复用声称的那一份必须真的登记过,新造的也不许盖掉别人的行。
existing, found, err := r.store.LibraryVersionByRef(ctx, r.scope,
governance.LibraryVersionProductionPrebackup, r.jobID, backup.SnapshotID)
if err != nil {
return fmt.Errorf("读不到现有预备份证据: %w", err)
}
if !found && backup.Reused {
return fmt.Errorf("预备份 %s 声称复用,但没有作业 %s 的版本证据:拒绝把复用登记成新备份",
backup.SnapshotID, r.jobID)
}
if found && !backup.Reused {
// 同一份预备份已经登记过,本轮却声称是新备份:那不是"重做备份",
// 而是用一份新证据替换已持久化的写集合锚点。
return fmt.Errorf("预备份 %s 已登记为作业 %s 的版本证据,本轮却不是复用:拒绝覆盖已持久化证据",
backup.SnapshotID, r.jobID)
}
evidence, err := prebackupEvidence(existing.Evidence, backup, r.jobID, r.mappingDigest)
if err != nil {
return fmt.Errorf("合并预备份证据失败: %w", err)
}
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
}
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID, r.operationID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker",
Evidence: evidence,
CreatedAt: now,
}); err != nil {
return fmt.Errorf("登记预备份版本失败: %w", err)
}
return nil
}
func failLibraryJob(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, jobID, message, stage string) error {
recorder.Stage(ctx, stage, "failed", message)
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
// failLibraryJobOrTakeover 按"这个作业有没有持久化预备份"选择失败出口。
//
// 有预备份就说明上一次尝试可能已经写过目标:这时任何早退(授权过期、装配失败、
// 维护被别人占用……)都必须保留 needs_restore 并点名基线,不能写"无需恢复"——
// 那会让页面把半写目标当成完好,也让操作者手里没有可回滚的那一份。
func failLibraryJobOrTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
scope, jobID, backupSnapshotID string, resumed bool, message, stage string) error {
if resumed {
return failLibraryTakeover(ctx, store, recorder, scope, jobID, backupSnapshotID, message)
}
return failLibraryJob(ctx, store, recorder, jobID, message, stage)
}
// failLibraryTakeover 是"拒绝接管"的失败出口:目标可能已被上一次尝试写过,
// 不能像 failLibraryJob 那样写"无需恢复"。维护占用保持现状(held/acquired 就是
// 安全状态),scope 行显式标 needs_restore,作业终态失败并指明可回滚的预备份。
//
// 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知;
// 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放
// 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。
func failLibraryTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
scope, jobID, backupSnapshotID, message string) error {
recorder.Stage(ctx, "prebackup", "failed", message)
recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理"
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "prebackup", message, recovery,
backupSnapshotID, "", time.Now().UTC())
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", message, time.Now().UTC())
return errors.New(message)
}
func kindLabel(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "恢复"
}
return "同步"
}
func digestKey(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "restore_snapshot_id"
}
return "target_snapshot_id"
}