Files
knowledge-governance/cmd/worker/library_handlers.go
T
姚章浩 5e1482e349
build / build (push) Successful in 1m40s
feat(library): 恢复验证成为可页面发起的真实作业
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
2026-11-03 10:14:08 +08:00

786 lines
42 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
"knowledge-governance/internal/pgdsn"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/testbarrier"
)
// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放:
// - contentUntouched:这次窗口还没写过目标(可以安全开放)
// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放)
// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护)
const (
contentUntouched = ""
contentSucceeded = "succeeded"
contentRolledBack = "rolled_back"
contentHalfWritten = "half_written"
contentRollbackFails = "rollback_failed"
)
// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
//
// 另外它是本机中断演练的固定点位:每个阶段写完、下一个阶段还没开始时调用
// testbarrier.Wait(默认关闭,两个环境变量同时在场才生效)。放在这一层而不是
// 发布代码里,是因为"阶段之间"正是真实半写状态的边界,而发布链路本身不需要知道演练。
type contentTrackedTarget struct {
inner libraryrelease.Target
jobID string
onApply func()
// revalidateGrant 由 runLibraryRelease 无条件接线:每笔副作用前按治理库里的
// 授权行重新判定(撤销/过期即停)。测试夹具可以显式传 nil 表示"这份夹具没有
// 持久授权行",但真实发布路径不允许为空 —— 那里的空值等于回到只信内存快照。
revalidateGrant func(context.Context, libraryrelease.Grant) error
}
// revalidateLibraryGrant 是"授权没有被中途撤销/过期"的唯一判定实现:
// 真实发布链路与测试都走它,避免测试自己写一份看起来相同的查询。
func revalidateLibraryGrant(store *postgresstore.Store) func(context.Context, libraryrelease.Grant) error {
return func(ctx context.Context, current libraryrelease.Grant) error {
// 权威来源是治理库里的授权行:参数摘要一变就查不到,撤销/过期立即消失。
found, ok, err := store.ActiveLibraryAuthorization(ctx, current.Target, current.SnapshotID,
current.ParamsDigest, time.Now().UTC())
if err != nil {
return fmt.Errorf("读取持久授权失败:%w", err)
}
if !ok {
return errors.New("当前授权已撤销或已过期")
}
if found.AuthorizedBy != current.AuthorizedBy {
return errors.New("持久授权与本次执行身份不一致")
}
return nil
}
}
func (t *contentTrackedTarget) RevalidateGrant(ctx context.Context, grant libraryrelease.Grant) error {
if t.revalidateGrant != nil {
return t.revalidateGrant(ctx, grant)
}
if inner, ok := t.inner.(libraryrelease.GrantRevalidator); ok {
return inner.RevalidateGrant(ctx, grant)
}
return nil
}
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
return t.inner.Current(ctx)
}
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
t.onApply()
if err := t.inner.Apply(ctx, stage, snapshotID); err != nil {
return err
}
// 屏障错误只记日志:演练设施坏掉不许让一次真实发布失败。
if barrierErr := testbarrier.Wait(ctx, "after:"+string(stage), t.jobID); barrierErr != nil {
log.Printf("library test barrier point=after:%s job=%s unavailable: %v", stage, t.jobID, barrierErr)
}
return nil
}
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
return t.inner.Verify(ctx, expect)
}
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
return t.inner.Restore(ctx, backup)
}
// PlanWriteSet 必须显式转发:orchestrator 是按 `target.(WriteSetPlanner)` 这个
// 类型断言决定要不要在写第一笔之前算写集合的,包一层之后断言就失败了——于是
// 台账既不落盘也不登记,恢复/接管时根本没有"这次明确写了什么"可读。
// 真实后果:2026-10-29 的两次真作业的预备份证据里都没有 writeset_digest,
// 归档根下也没有 writesets/;一旦 worker 崩在写入中间,接管必然失败。
//
// 内层不支持预检时返回硬错误,不退回"没有台账也照写":那会让恢复只能按目标
// 现状推断残渣,从而误删历史孤儿资源。
func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
planner, ok := t.inner.(libraryrelease.WriteSetPlanner)
if !ok {
return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产")
}
set, err := planner.PlanWriteSet(ctx, plan)
if err == nil {
// 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。
log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s",
set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID)
}
return set, err
}
func (t *contentTrackedTarget) AdoptWriteSet(set libraryrelease.WriteSet) error {
adopter, ok := t.inner.(libraryrelease.WriteSetAdopter)
if !ok {
return errors.New("目标实现不支持接管已持久化写集合,拒绝重建计划")
}
if set.JobID != t.jobID {
return errors.New("写集合与 worker 作业身份不符")
}
return adopter.AdoptWriteSet(set)
}
var _ libraryrelease.WriteSetPlanner = (*contentTrackedTarget)(nil)
var _ libraryrelease.WriteSetAdopter = (*contentTrackedTarget)(nil)
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
type libraryReleasePayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
FinalizeOnly bool `json:"finalize_only,omitempty"`
}
// openLibraryMaintenance 定义在 library_wiring.go:维护目标的装配与目标整库栈共用
// 同一个包(internal/librarystack),不在这里再拼一份。
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config appconfig.Config,
job jobs.Job, kind governance.LibraryJobKind) (err error) {
var payload libraryReleasePayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
return fmt.Errorf("invalid library %s payload", kind)
}
// 准入必须在任何维护/目标动作之前:业务行存在、kind/scope 一致、队列关联不可伪造。
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, func(runCtx context.Context, writer *libraryAttemptWriter) error {
return runLibraryReleaseAttempt(runCtx, writer, config, kind, payload)
})
}
func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
kind governance.LibraryJobKind, payload libraryReleasePayload) (err error) {
store := writer
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: store.attempt.OperationID}
// 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。
if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil {
if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil {
recorder.totals = stageSizesFrom(manifest)
}
archive.Close()
}
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
resumed := err == nil && previous.SnapshotID != ""
// 续作失败时不能写"无需恢复":上一次尝试可能已经把目标写了一半。
// 这一处按"有没有持久化预备份"统一选择失败出口,避免每个早退点各写一套文案。
fail := func(message, stage string) error {
return failLibraryJobOrTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID,
previous.SnapshotID, resumed, message, stage)
}
if resumed {
facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID)
resume := loadTakeoverResumeFacts(ctx, store, config.LibraryArchiveRoot, previous, now)
decision := classifyTakeover(previous, facts, resume)
// 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份:
// 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。
logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope,
payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC())
switch decision {
case takeoverUnknown:
// 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。
return fail("接管时读不到该目标的版本状态,无法判断是只差收尾还是要重做;本作业未继续写目标,稍后重试", "prebackup")
case takeoverSuperseded:
return fail("本快照的生效版本已被别的快照顶替,且本作业没有新的恢复授权/独占维护窗口/未变基线,拒绝用旧快照覆盖当前生产版本", "prebackup")
case takeoverForeignEffective:
// 同一个快照由别的作业发布过:那只能证明生产上有这份内容,不能证明本作业
// 写完过、更不能证明它只差收尾。按同快照收尾会把只写了第一个阶段的作业
// 标成成功,并改写版本归属(实测过的假完成)。
owner := facts.Current.SourceJobID
if facts.HasSnapshot {
owner = facts.Snapshot.SourceJobID
}
return failLibraryTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID,
"接管时目标快照 "+shortSnapshotID(payload.TargetSnapshotID)+" 的生效版本由作业 "+owner+
" 登记:本作业没有自己的完成证据,不按同快照收尾")
case takeoverPendingFinal:
// 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。
payload.FinalizeOnly = true
}
}
if resumed {
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
}
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
gateReasons = append(gateReasons, reason)
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
recovery := "未对生产执行任何写入,无需恢复"
if resumed {
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
recovery, previous.SnapshotID, "", time.Now().UTC())
return errors.New(message)
}
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
if err != nil {
return fail("无法计算请求摘要", "prebackup")
}
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if err != nil || !found {
return fail("没有与目标快照匹配的有效发布授权", "prebackup")
}
// 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己
// 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。
priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8)
if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil {
log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr)
} else {
for _, source := range sources {
priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID,
SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt})
}
}
// 生产索引映射基线:权威来源是当前生效版本(精确查),其次任何一次成功记录的映射。
// 只认"生效版本"会在恢复把版本顶替掉之后误判"索引被改动"。
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, backups, wireErr := buildLibraryProductionStack(ctx, config, restoreDispatchGate(config, store.Store), payload.Scope, payload.TargetSnapshotID, knownMapping)
if wireErr == nil {
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = mapping
}
}
if wireErr != nil {
return fail(wireErr.Error(), "prebackup")
}
defer target.Close()
// 容量门禁必须在**任何副作用之前**(F11):预算不足就不该进维护、不该拍备份。
// 旧实现只在存储页 GET 里算这个数,worker 从不读它,于是"预算为正但不够"照样写。
if capErr := checkLibraryCapacity(ctx, config, payload.TargetSnapshotID); capErr != nil {
return fail(capErr.Error(), "prebackup")
}
// 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。
manager, managerErr := openLibraryMaintenance(config)
if managerErr != nil {
return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
}
defer manager.Close()
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
}
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
originalState, stateErr := manager.OriginalState(ctx)
if stateErr != nil {
return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
}
identity := manager.Target()
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: identity.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if instanceErr != nil {
return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
}
_ = instanceLock
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: targetIdentity + " | " + manager.Description(),
Level: librarymaintenance.LevelInstance,
TargetFingerprint: identity.Fingerprint,
Services: identity.Writers,
IngressState: "unknown",
State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if acquireErr != nil {
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
return fail("目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
// 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把
// 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。
if resumed {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
} else {
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
}
return fail("维护协调器不可用:"+openErr.Error(), "prebackup")
}
defer control.Close()
defer coordinator.Close()
// 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前,
// 一律按半写对待,任何早退都不许重新开放目标。
contentState := contentUntouched
if resumed {
contentState = contentHalfWritten
}
pending := successBoundary{}
tracked := &contentTrackedTarget{inner: target, jobID: payload.LibraryJobID,
onApply: func() { contentState = contentHalfWritten },
revalidateGrant: revalidateLibraryGrant(store.Store)}
allowRestore := func(error) bool {
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
}
execute := func(ctx context.Context) error {
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
contentState = contentHalfWritten
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
contentState = contentSucceeded
mappingDigest, _ := target.Search().MappingSHA256(ctx)
// 收尾续作只补登记,不允许把首次锚点洗掉:字节数与写集合摘要从已持久化的
// 预备份证据合并进来(F09)。缺失即拒绝,不写一个"看起来完整"的空证据。
evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID,
previous.SnapshotID, payload.TargetSnapshotID, "finalize_only", mappingDigest)
if evidenceErr != nil {
message := "收尾续作缺少首次锚点,拒绝登记生效版本:" + evidenceErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留;补齐首次锚点证据后再续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: evidence,
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
// 成功要等维护锁释放完才暴露:这里只记"待收尾"。
pending = successBoundary{Ready: true, Backup: previous.SnapshotID,
Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()}
return nil
}
var backupSource libraryrelease.BackupStore = backups
if resumed {
// 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣,
// 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。
set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.LibraryJobID, previous.SnapshotID)
if ledgerErr != nil {
// 读不到就一笔都不许写:目标可能半写,恢复口径未知。
message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
recorder.Stage(ctx, "prebackup", "failed", message)
return errors.New(message)
}
// 复用不重做备份,真实字节只能从首次登记的证据里取:缺了就必须停,
// 否则生效版本证据会写成 backup_bytes:0(现场就是这样丢的)。
backupBytes, bytesErr := prebackupBytesOf(ctx, store, payload.Scope, payload.LibraryJobID, previous.SnapshotID)
if bytesErr != nil {
message := "接管时读不回预备份的真实字节数,拒绝继续写目标:" + bytesErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
recorder.Stage(ctx, "prebackup", "failed", message)
return errors.New(message)
}
backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID, Bytes: backupBytes}, write: set}
}
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
PriorSnapshots: priorSnapshots,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
contentState = contentSucceeded
mappingDigest := ""
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = mapping
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: effectiveEvidence(outcome.Backup, string(kind), mappingDigest),
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
}
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
// 内容与版本都落定了,但还不能报成功:服务健康、入口开放、两把维护锁
// 释放完成才允许暴露 succeeded(见 finishLibrarySuccess)。
pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID,
Target: payload.TargetSnapshotID, FinishedAt: finishedAt}
return nil
case "rolled_back":
contentState = contentRolledBack
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
contentState = contentRollbackFails
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
}
}
prepare := func(ctx context.Context) error {
// 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口,
// 只会把"本来坏的"记成"维护造成的"。
return manager.VerifyRunning(ctx)
}
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
return err
}
return w.MarkContentRisk(backupCtx)
}
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
// 记录失败不影响隔离本身,但必须留痕,不能假装记上了。
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
}
return "", execute(ctx)
}
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
// 终态重放(F05):journal 里已经 succeeded,本次没有再执行 work。内容/服务子协议
// 已经完成,缺的只可能是"业务收尾"(版本登记、锁释放、终态)。这时必须从**持久事实**
// 重建收尾凭据,而不是靠进程内 pending——那个指针早就随崩溃没了。
if windowErr == nil && window.State == "succeeded" && contentState != contentSucceeded {
rebuilt, rebuildErr := rebuildSuccessBoundary(ctx, store, recorder, payload, previous, kind, grant)
if rebuildErr != nil {
note := "窗口已成功但收尾凭据无法从持久事实重建,目标保持维护:" + rebuildErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
pending = rebuilt
contentState = contentSucceeded
}
// 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。
now2 := time.Now().UTC()
switch {
case window.State == "held":
note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复"
if windowErr != nil {
note += ":" + windowErr.Error()
}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
if windowErr == nil {
return errors.New(note)
}
return windowErr
case contentState == contentSucceeded:
if !pending.Ready {
// 内容状态说成功但没有收尾凭据(不可能两者都缺):按未收尾处理,保持维护。
note := "内容状态为已验证,但缺少收尾凭据,目标保持维护"
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
return errors.New(note)
}
if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope,
payload.LibraryJobID, pending); finishErr != nil {
return finishErr
}
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup)
return windowErr
case contentState == contentRolledBack:
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
}
return windowErr
default:
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未写目标,维护窗口未开启", now2); err != nil {
return fmt.Errorf("维护占用释放失败: %w", err)
}
return windowErr
}
}
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) {
if recoveries, ok := store.(interface{ ResumeExpiredLibraryRecoveryQueues(context.Context) error }); ok {
if err := recoveries.ResumeExpiredLibraryRecoveryQueues(ctx); err != nil {
return 0, 0, err
}
}
interrupted, err := store.InterruptedLibraryJobs(ctx)
if err != nil {
return 0, 0, err
}
resumed, failed := 0, 0
for _, job := range interrupted {
now := time.Now().UTC()
if job.SnapshotID == "" {
message := "worker 中断时还没有完成预备份,不能盲写目标"
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行写入", "", "", now)
failed++
continue
}
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
if claimErr != nil {
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
continue
}
if !claimed {
continue
}
queueType := librarySyncJobType
switch job.Kind {
case governance.LibraryJobRestore:
queueType = libraryRestoreJobType
case governance.LibraryJobBaseline:
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
queueType = libraryBaselineJobType
case governance.LibraryJobVerify:
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
// 否则目标会一直停在维护里而作业显示失败。
queueType = libraryVerifyJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
AvailableAt: now, MaxAttempts: 1,
IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
if err != nil && strings.Contains(err.Error(), "duplicate key") {
queueID := governance.QueueJobIDFor(job.ID)
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
} else {
resumed++
logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now)
}
continue
}
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
continue
}
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
continue
}
resumed++
// 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑",
// 外部用它判接管来源(汇总计数证明不了是哪个作业)。
logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now)
}
return resumed, failed, nil
}
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
//
// 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把
// "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行,
// 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。
//
// ⚠️ 证据必须**合并**进已有那一行,不能每次造一份新的:RecordLibraryVersion 是
// `evidence = EXCLUDED.evidence`(整份覆盖)。2026-10-30 的演练里,接管复用时
// 新造的证据只有 backup_* / job / index_mapping_sha256,把首次规划写入的
// writeset_digest/writeset_rows… 整段抹掉,第二次中断再接管就再也读不回写集合
// (verify_effective 因此从有摘要变成空)。合并只对**同一作业 + 同一份预备份**
// 成立:身份不符一律失败,绝不把别的作业的证据带进来。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if r.onBackup != nil {
if err := r.onBackup(ctx, backup); err != nil {
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
}
}
// 先读既有证据:复用声称的那一份必须真的登记过,新造的也不许盖掉别人的行。
existing, found, err := r.store.LibraryVersionByRef(ctx, r.scope,
governance.LibraryVersionProductionPrebackup, r.jobID, backup.SnapshotID)
if err != nil {
return fmt.Errorf("读不到现有预备份证据: %w", err)
}
if !found && backup.Reused {
return fmt.Errorf("预备份 %s 声称复用,但没有作业 %s 的版本证据:拒绝把复用登记成新备份",
backup.SnapshotID, r.jobID)
}
if found && !backup.Reused {
// 同一份预备份已经登记过,本轮却声称是新备份:那不是"重做备份",
// 而是用一份新证据替换已持久化的写集合锚点。
return fmt.Errorf("预备份 %s 已登记为作业 %s 的版本证据,本轮却不是复用:拒绝覆盖已持久化证据",
backup.SnapshotID, r.jobID)
}
evidence, err := prebackupEvidence(existing.Evidence, backup, r.jobID, r.mappingDigest)
if err != nil {
return fmt.Errorf("合并预备份证据失败: %w", err)
}
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
}
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID, r.operationID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker",
Evidence: evidence,
CreatedAt: now,
}); err != nil {
return fmt.Errorf("登记预备份版本失败: %w", err)
}
return nil
}
func failLibraryJob(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, jobID, message, stage string) error {
recorder.Stage(ctx, stage, "failed", message)
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
// failLibraryJobOrTakeover 按"这个作业有没有持久化预备份"选择失败出口。
//
// 有预备份就说明上一次尝试可能已经写过目标:这时任何早退(授权过期、装配失败、
// 维护被别人占用……)都必须保留 needs_restore 并点名基线,不能写"无需恢复"——
// 那会让页面把半写目标当成完好,也让操作者手里没有可回滚的那一份。
func failLibraryJobOrTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
scope, jobID, backupSnapshotID string, resumed bool, message, stage string) error {
if resumed {
return failLibraryTakeover(ctx, store, recorder, scope, jobID, backupSnapshotID, message)
}
return failLibraryJob(ctx, store, recorder, jobID, message, stage)
}
// failLibraryTakeover 是"拒绝接管"的失败出口:目标可能已被上一次尝试写过,
// 不能像 failLibraryJob 那样写"无需恢复"。维护占用保持现状(held/acquired 就是
// 安全状态),scope 行显式标 needs_restore,作业终态失败并指明可回滚的预备份。
//
// 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知;
// 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放
// 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。
func failLibraryTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
scope, jobID, backupSnapshotID, message string) error {
recorder.Stage(ctx, "prebackup", "failed", message)
recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理"
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "prebackup", message, recovery,
backupSnapshotID, "", time.Now().UTC())
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", message, time.Now().UTC())
return errors.New(message)
}
func kindLabel(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "恢复"
}
return "同步"
}
func digestKey(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "restore_snapshot_id"
}
return "target_snapshot_id"
}