a7d643635a
build / build (push) Successful in 1m27s
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2, 导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的 owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。 R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引 (job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与 operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、 新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律 ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。 R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式 结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。 R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release 路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。 同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点 改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、 恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。 测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例; go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
734 lines
39 KiB
Go
734 lines
39 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
"knowledge-governance/internal/pgdsn"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
"knowledge-governance/internal/testbarrier"
|
||
)
|
||
|
||
// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放:
|
||
// - contentUntouched:这次窗口还没写过目标(可以安全开放)
|
||
// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放)
|
||
// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护)
|
||
const (
|
||
contentUntouched = ""
|
||
contentSucceeded = "succeeded"
|
||
contentRolledBack = "rolled_back"
|
||
contentHalfWritten = "half_written"
|
||
contentRollbackFails = "rollback_failed"
|
||
)
|
||
|
||
// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。
|
||
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
|
||
//
|
||
// 另外它是本机中断演练的固定点位:每个阶段写完、下一个阶段还没开始时调用
|
||
// testbarrier.Wait(默认关闭,两个环境变量同时在场才生效)。放在这一层而不是
|
||
// 发布代码里,是因为"阶段之间"正是真实半写状态的边界,而发布链路本身不需要知道演练。
|
||
type contentTrackedTarget struct {
|
||
inner libraryrelease.Target
|
||
jobID string
|
||
onApply func()
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
|
||
return t.inner.Current(ctx)
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
|
||
t.onApply()
|
||
if err := t.inner.Apply(ctx, stage, snapshotID); err != nil {
|
||
return err
|
||
}
|
||
// 屏障错误只记日志:演练设施坏掉不许让一次真实发布失败。
|
||
if barrierErr := testbarrier.Wait(ctx, "after:"+string(stage), t.jobID); barrierErr != nil {
|
||
log.Printf("library test barrier point=after:%s job=%s unavailable: %v", stage, t.jobID, barrierErr)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
|
||
return t.inner.Verify(ctx, expect)
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||
return t.inner.Restore(ctx, backup)
|
||
}
|
||
|
||
// PlanWriteSet 必须显式转发:orchestrator 是按 `target.(WriteSetPlanner)` 这个
|
||
// 类型断言决定要不要在写第一笔之前算写集合的,包一层之后断言就失败了——于是
|
||
// 台账既不落盘也不登记,恢复/接管时根本没有"这次明确写了什么"可读。
|
||
// 真实后果:2026-10-29 的两次真作业的预备份证据里都没有 writeset_digest,
|
||
// 归档根下也没有 writesets/;一旦 worker 崩在写入中间,接管必然失败。
|
||
//
|
||
// 内层不支持预检时返回硬错误,不退回"没有台账也照写":那会让恢复只能按目标
|
||
// 现状推断残渣,从而误删历史孤儿资源。
|
||
func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||
planner, ok := t.inner.(libraryrelease.WriteSetPlanner)
|
||
if !ok {
|
||
return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产")
|
||
}
|
||
set, err := planner.PlanWriteSet(ctx, plan)
|
||
if err == nil {
|
||
// 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。
|
||
log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s",
|
||
set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID)
|
||
}
|
||
return set, err
|
||
}
|
||
|
||
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
|
||
type libraryReleasePayload struct {
|
||
LibraryJobID string `json:"library_job_id"`
|
||
Scope string `json:"scope"`
|
||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
|
||
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
|
||
FinalizeOnly bool `json:"finalize_only,omitempty"`
|
||
}
|
||
|
||
// openLibraryMaintenance 定义在 library_wiring.go:维护目标的装配与目标整库栈共用
|
||
// 同一个包(internal/librarystack),不在这里再拼一份。
|
||
|
||
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
|
||
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
|
||
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
|
||
func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config appconfig.Config,
|
||
job jobs.Job, kind governance.LibraryJobKind) (err error) {
|
||
var payload libraryReleasePayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
|
||
return fmt.Errorf("invalid library %s payload", kind)
|
||
}
|
||
// 准入必须在任何维护/目标动作之前:业务行存在、kind/scope 一致、队列关联不可伪造。
|
||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil {
|
||
return admitErr
|
||
}
|
||
// 执行代次:这一次投递是一个确定的执行者身份。所有治理写入都走 writer
|
||
// (同名方法带 epoch 守卫),失租的旧执行从此写不进业务状态/维护锁(F13)。
|
||
writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID))
|
||
if attemptErr != nil {
|
||
return attemptErr
|
||
}
|
||
defer func() {
|
||
if err != nil {
|
||
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptFailed, err.Error())
|
||
return
|
||
}
|
||
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "")
|
||
}()
|
||
store := writer
|
||
now := time.Now().UTC()
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
|
||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||
operationID: store.attempt.OperationID}
|
||
// 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。
|
||
if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil {
|
||
if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil {
|
||
recorder.totals = stageSizesFrom(manifest)
|
||
}
|
||
archive.Close()
|
||
}
|
||
|
||
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
|
||
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
|
||
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
|
||
resumed := err == nil && previous.SnapshotID != ""
|
||
// 续作失败时不能写"无需恢复":上一次尝试可能已经把目标写了一半。
|
||
// 这一处按"有没有持久化预备份"统一选择失败出口,避免每个早退点各写一套文案。
|
||
fail := func(message, stage string) error {
|
||
return failLibraryJobOrTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID,
|
||
previous.SnapshotID, resumed, message, stage)
|
||
}
|
||
if resumed {
|
||
facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID)
|
||
resume := loadTakeoverResumeFacts(ctx, store, config.LibraryArchiveRoot, previous, now)
|
||
decision := classifyTakeover(previous, facts, resume)
|
||
// 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份:
|
||
// 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。
|
||
logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope,
|
||
payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC())
|
||
switch decision {
|
||
case takeoverUnknown:
|
||
// 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。
|
||
return fail("接管时读不到该目标的版本状态,无法判断是只差收尾还是要重做;本作业未继续写目标,稍后重试", "prebackup")
|
||
case takeoverSuperseded:
|
||
return fail("本快照的生效版本已被别的快照顶替,且本作业没有新的恢复授权/独占维护窗口/未变基线,拒绝用旧快照覆盖当前生产版本", "prebackup")
|
||
case takeoverForeignEffective:
|
||
// 同一个快照由别的作业发布过:那只能证明生产上有这份内容,不能证明本作业
|
||
// 写完过、更不能证明它只差收尾。按同快照收尾会把只写了第一个阶段的作业
|
||
// 标成成功,并改写版本归属(实测过的假完成)。
|
||
owner := facts.Current.SourceJobID
|
||
if facts.HasSnapshot {
|
||
owner = facts.Snapshot.SourceJobID
|
||
}
|
||
return failLibraryTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID,
|
||
"接管时目标快照 "+shortSnapshotID(payload.TargetSnapshotID)+" 的生效版本由作业 "+owner+
|
||
" 登记:本作业没有自己的完成证据,不按同快照收尾")
|
||
case takeoverPendingFinal:
|
||
// 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。
|
||
payload.FinalizeOnly = true
|
||
}
|
||
}
|
||
if resumed {
|
||
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
|
||
}
|
||
|
||
gateReasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
if len(gateReasons) > 0 {
|
||
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
|
||
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
|
||
recovery := "未对生产执行任何写入,无需恢复"
|
||
if resumed {
|
||
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
|
||
recovery, previous.SnapshotID, "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
|
||
if err != nil {
|
||
return fail("无法计算请求摘要", "prebackup")
|
||
}
|
||
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
|
||
if err != nil || !found {
|
||
return fail("没有与目标快照匹配的有效发布授权", "prebackup")
|
||
}
|
||
// 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己
|
||
// 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。
|
||
priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8)
|
||
if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil {
|
||
log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr)
|
||
} else {
|
||
for _, source := range sources {
|
||
priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID,
|
||
SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt})
|
||
}
|
||
}
|
||
// 生产索引映射基线:权威来源是当前生效版本(精确查),其次任何一次成功记录的映射。
|
||
// 只认"生效版本"会在恢复把版本顶替掉之后误判"索引被改动"。
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
|
||
if wireErr == nil {
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
recorder.mappingDigest = mapping
|
||
}
|
||
}
|
||
if wireErr != nil {
|
||
return fail(wireErr.Error(), "prebackup")
|
||
}
|
||
defer target.Close()
|
||
|
||
// 容量门禁必须在**任何副作用之前**(F11):预算不足就不该进维护、不该拍备份。
|
||
// 旧实现只在存储页 GET 里算这个数,worker 从不读它,于是"预算为正但不够"照样写。
|
||
if capErr := checkLibraryCapacity(ctx, config, payload.TargetSnapshotID); capErr != nil {
|
||
return fail(capErr.Error(), "prebackup")
|
||
}
|
||
|
||
// 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。
|
||
manager, managerErr := openLibraryMaintenance(config)
|
||
if managerErr != nil {
|
||
return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
|
||
}
|
||
defer manager.Close()
|
||
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
|
||
}
|
||
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
|
||
pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
|
||
originalState, stateErr := manager.OriginalState(ctx)
|
||
if stateErr != nil {
|
||
return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
|
||
}
|
||
identity := manager.Target()
|
||
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
|
||
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
|
||
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
|
||
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
|
||
Services: identity.Writers, IngressState: "unknown", State: "acquired",
|
||
OriginalServiceState: originalState,
|
||
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
|
||
}, now)
|
||
if instanceErr != nil {
|
||
return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
|
||
}
|
||
_ = instanceLock
|
||
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
|
||
TargetIdentity: targetIdentity + " | " + manager.Description(),
|
||
Level: librarymaintenance.LevelInstance,
|
||
TargetFingerprint: identity.Fingerprint,
|
||
Services: identity.Writers,
|
||
IngressState: "unknown",
|
||
State: "acquired",
|
||
OriginalServiceState: originalState,
|
||
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
|
||
}, now)
|
||
if acquireErr != nil {
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
|
||
return fail("目标正在维护中:"+acquireErr.Error(), "prebackup")
|
||
}
|
||
_ = maintenance
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
// 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把
|
||
// 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。
|
||
if resumed {
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
|
||
} else {
|
||
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
|
||
}
|
||
return fail("维护协调器不可用:"+openErr.Error(), "prebackup")
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
// 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前,
|
||
// 一律按半写对待,任何早退都不许重新开放目标。
|
||
contentState := contentUntouched
|
||
if resumed {
|
||
contentState = contentHalfWritten
|
||
}
|
||
pending := successBoundary{}
|
||
tracked := &contentTrackedTarget{inner: target, jobID: payload.LibraryJobID,
|
||
onApply: func() { contentState = contentHalfWritten }}
|
||
allowRestore := func(error) bool {
|
||
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
|
||
}
|
||
|
||
execute := func(ctx context.Context) error {
|
||
if payload.FinalizeOnly {
|
||
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
|
||
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
|
||
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
|
||
contentState = contentHalfWritten
|
||
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
contentState = contentSucceeded
|
||
mappingDigest, _ := target.Search().MappingSHA256(ctx)
|
||
// 收尾续作只补登记,不允许把首次锚点洗掉:字节数与写集合摘要从已持久化的
|
||
// 预备份证据合并进来(F09)。缺失即拒绝,不写一个"看起来完整"的空证据。
|
||
evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID,
|
||
previous.SnapshotID, payload.TargetSnapshotID, "finalize_only", mappingDigest)
|
||
if evidenceErr != nil {
|
||
message := "收尾续作缺少首次锚点,拒绝登记生效版本:" + evidenceErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"预备份 "+previous.SnapshotID+" 保留;补齐首次锚点证据后再续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
|
||
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
|
||
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
|
||
Evidence: evidence,
|
||
CreatedAt: time.Now().UTC(),
|
||
}); err != nil {
|
||
message := "收尾续作仍无法登记生效版本:" + err.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
|
||
// 成功要等维护锁释放完才暴露:这里只记"待收尾"。
|
||
pending = successBoundary{Ready: true, Backup: previous.SnapshotID,
|
||
Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()}
|
||
return nil
|
||
}
|
||
|
||
var backupSource libraryrelease.BackupStore = backups
|
||
if resumed {
|
||
// 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣,
|
||
// 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。
|
||
set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||
payload.LibraryJobID, previous.SnapshotID)
|
||
if ledgerErr != nil {
|
||
// 读不到就一笔都不许写:目标可能半写,恢复口径未知。
|
||
message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error()
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
return errors.New(message)
|
||
}
|
||
// 复用不重做备份,真实字节只能从首次登记的证据里取:缺了就必须停,
|
||
// 否则生效版本证据会写成 backup_bytes:0(现场就是这样丢的)。
|
||
backupBytes, bytesErr := prebackupBytesOf(ctx, store, payload.Scope, payload.LibraryJobID, previous.SnapshotID)
|
||
if bytesErr != nil {
|
||
message := "接管时读不回预备份的真实字节数,拒绝继续写目标:" + bytesErr.Error()
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
return errors.New(message)
|
||
}
|
||
backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
|
||
SnapshotID: previous.SnapshotID, Bytes: backupBytes}, write: set}
|
||
}
|
||
|
||
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
|
||
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
|
||
PriorSnapshots: priorSnapshots,
|
||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
|
||
finishedAt := time.Now().UTC()
|
||
switch outcome.State {
|
||
case "succeeded":
|
||
contentState = contentSucceeded
|
||
mappingDigest := ""
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
mappingDigest = mapping
|
||
}
|
||
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
|
||
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
|
||
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
|
||
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID),
|
||
Scope: payload.Scope,
|
||
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
|
||
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
|
||
Evidence: effectiveEvidence(outcome.Backup, string(kind), mappingDigest),
|
||
CreatedAt: finishedAt,
|
||
})
|
||
if versionErr != nil {
|
||
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
|
||
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
|
||
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
|
||
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
|
||
}
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
|
||
// 内容与版本都落定了,但还不能报成功:服务健康、入口开放、两把维护锁
|
||
// 释放完成才允许暴露 succeeded(见 finishLibrarySuccess)。
|
||
pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID,
|
||
Target: payload.TargetSnapshotID, FinishedAt: finishedAt}
|
||
return nil
|
||
case "rolled_back":
|
||
contentState = contentRolledBack
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
case "rollback_failed":
|
||
contentState = contentRollbackFails
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
|
||
default:
|
||
recovery := outcome.Recovery
|
||
if recovery == "" {
|
||
recovery = "未对生产执行任何写入,无需恢复"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
|
||
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
}
|
||
}
|
||
|
||
prepare := func(ctx context.Context) error {
|
||
// 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口,
|
||
// 只会把"本来坏的"记成"维护造成的"。
|
||
return manager.VerifyRunning(ctx)
|
||
}
|
||
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
|
||
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
|
||
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
|
||
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
|
||
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
|
||
return err
|
||
}
|
||
return w.MarkContentRisk(backupCtx)
|
||
}
|
||
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
|
||
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
|
||
// 记录失败不影响隔离本身,但必须留痕,不能假装记上了。
|
||
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
|
||
}
|
||
return "", execute(ctx)
|
||
}
|
||
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
|
||
|
||
// 终态重放(F05):journal 里已经 succeeded,本次没有再执行 work。内容/服务子协议
|
||
// 已经完成,缺的只可能是"业务收尾"(版本登记、锁释放、终态)。这时必须从**持久事实**
|
||
// 重建收尾凭据,而不是靠进程内 pending——那个指针早就随崩溃没了。
|
||
if windowErr == nil && window.State == "succeeded" && contentState != contentSucceeded {
|
||
rebuilt, rebuildErr := rebuildSuccessBoundary(ctx, store, recorder, payload, previous, kind, grant)
|
||
if rebuildErr != nil {
|
||
note := "窗口已成功但收尾凭据无法从持久事实重建,目标保持维护:" + rebuildErr.Error()
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
pending = rebuilt
|
||
contentState = contentSucceeded
|
||
}
|
||
|
||
// 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。
|
||
now2 := time.Now().UTC()
|
||
switch {
|
||
case window.State == "held":
|
||
note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复"
|
||
if windowErr != nil {
|
||
note += ":" + windowErr.Error()
|
||
}
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
|
||
if windowErr == nil {
|
||
return errors.New(note)
|
||
}
|
||
return windowErr
|
||
case contentState == contentSucceeded:
|
||
if !pending.Ready {
|
||
// 内容状态说成功但没有收尾凭据(不可能两者都缺):按未收尾处理,保持维护。
|
||
note := "内容状态为已验证,但缺少收尾凭据,目标保持维护"
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
|
||
return errors.New(note)
|
||
}
|
||
if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope,
|
||
payload.LibraryJobID, pending); finishErr != nil {
|
||
return finishErr
|
||
}
|
||
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
|
||
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup)
|
||
return windowErr
|
||
case contentState == contentRolledBack:
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
|
||
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
|
||
}
|
||
return windowErr
|
||
default:
|
||
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"未写目标,维护窗口未开启", now2); err != nil {
|
||
return fmt.Errorf("维护占用释放失败: %w", err)
|
||
}
|
||
return windowErr
|
||
}
|
||
}
|
||
|
||
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
|
||
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
|
||
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
|
||
func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) {
|
||
interrupted, err := store.InterruptedLibraryJobs(ctx)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
resumed, failed := 0, 0
|
||
for _, job := range interrupted {
|
||
now := time.Now().UTC()
|
||
if job.SnapshotID == "" {
|
||
message := "worker 中断时还没有完成预备份,不能盲写目标"
|
||
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
|
||
"未对生产执行写入", "", "", now)
|
||
failed++
|
||
continue
|
||
}
|
||
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
|
||
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
|
||
if claimErr != nil {
|
||
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
|
||
continue
|
||
}
|
||
if !claimed {
|
||
continue
|
||
}
|
||
queueType := librarySyncJobType
|
||
switch job.Kind {
|
||
case governance.LibraryJobRestore:
|
||
queueType = libraryRestoreJobType
|
||
case governance.LibraryJobBaseline:
|
||
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
|
||
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
|
||
queueType = libraryBaselineJobType
|
||
}
|
||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||
"target_snapshot_id": job.TargetSnapshotID})
|
||
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
|
||
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
|
||
AvailableAt: now, MaxAttempts: 1,
|
||
IdempotencyKey: "library.resume.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
|
||
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
|
||
if err != nil && strings.Contains(err.Error(), "duplicate key") {
|
||
queueID := governance.QueueJobIDFor(job.ID)
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
|
||
} else {
|
||
resumed++
|
||
logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now)
|
||
}
|
||
continue
|
||
}
|
||
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
|
||
continue
|
||
}
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
|
||
continue
|
||
}
|
||
resumed++
|
||
// 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑",
|
||
// 外部用它判接管来源(汇总计数证明不了是哪个作业)。
|
||
logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now)
|
||
}
|
||
return resumed, failed, nil
|
||
}
|
||
|
||
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
|
||
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
|
||
//
|
||
// 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把
|
||
// "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行,
|
||
// 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。
|
||
//
|
||
// ⚠️ 证据必须**合并**进已有那一行,不能每次造一份新的:RecordLibraryVersion 是
|
||
// `evidence = EXCLUDED.evidence`(整份覆盖)。2026-10-30 的演练里,接管复用时
|
||
// 新造的证据只有 backup_* / job / index_mapping_sha256,把首次规划写入的
|
||
// writeset_digest/writeset_rows… 整段抹掉,第二次中断再接管就再也读不回写集合
|
||
// (verify_effective 因此从有摘要变成空)。合并只对**同一作业 + 同一份预备份**
|
||
// 成立:身份不符一律失败,绝不把别的作业的证据带进来。
|
||
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||
now := time.Now().UTC()
|
||
if r.onBackup != nil {
|
||
if err := r.onBackup(ctx, backup); err != nil {
|
||
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
|
||
}
|
||
}
|
||
// 先读既有证据:复用声称的那一份必须真的登记过,新造的也不许盖掉别人的行。
|
||
existing, found, err := r.store.LibraryVersionByRef(ctx, r.scope,
|
||
governance.LibraryVersionProductionPrebackup, r.jobID, backup.SnapshotID)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到现有预备份证据: %w", err)
|
||
}
|
||
if !found && backup.Reused {
|
||
return fmt.Errorf("预备份 %s 声称复用,但没有作业 %s 的版本证据:拒绝把复用登记成新备份",
|
||
backup.SnapshotID, r.jobID)
|
||
}
|
||
if found && !backup.Reused {
|
||
// 同一份预备份已经登记过,本轮却声称是新备份:那不是"重做备份",
|
||
// 而是用一份新证据替换已持久化的写集合锚点。
|
||
return fmt.Errorf("预备份 %s 已登记为作业 %s 的版本证据,本轮却不是复用:拒绝覆盖已持久化证据",
|
||
backup.SnapshotID, r.jobID)
|
||
}
|
||
evidence, err := prebackupEvidence(existing.Evidence, backup, r.jobID, r.mappingDigest)
|
||
if err != nil {
|
||
return fmt.Errorf("合并预备份证据失败: %w", err)
|
||
}
|
||
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
|
||
backup.SnapshotID, "", now); err != nil {
|
||
return fmt.Errorf("持久化预备份引用失败: %w", err)
|
||
}
|
||
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID, r.operationID),
|
||
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
|
||
SourceJobID: r.jobID, CreatedBy: "worker",
|
||
Evidence: evidence,
|
||
CreatedAt: now,
|
||
}); err != nil {
|
||
return fmt.Errorf("登记预备份版本失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func failLibraryJob(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, jobID, message, stage string) error {
|
||
recorder.Stage(ctx, stage, "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
|
||
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
// failLibraryJobOrTakeover 按"这个作业有没有持久化预备份"选择失败出口。
|
||
//
|
||
// 有预备份就说明上一次尝试可能已经写过目标:这时任何早退(授权过期、装配失败、
|
||
// 维护被别人占用……)都必须保留 needs_restore 并点名基线,不能写"无需恢复"——
|
||
// 那会让页面把半写目标当成完好,也让操作者手里没有可回滚的那一份。
|
||
func failLibraryJobOrTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
|
||
scope, jobID, backupSnapshotID string, resumed bool, message, stage string) error {
|
||
if resumed {
|
||
return failLibraryTakeover(ctx, store, recorder, scope, jobID, backupSnapshotID, message)
|
||
}
|
||
return failLibraryJob(ctx, store, recorder, jobID, message, stage)
|
||
}
|
||
|
||
// failLibraryTakeover 是"拒绝接管"的失败出口:目标可能已被上一次尝试写过,
|
||
// 不能像 failLibraryJob 那样写"无需恢复"。维护占用保持现状(held/acquired 就是
|
||
// 安全状态),scope 行显式标 needs_restore,作业终态失败并指明可回滚的预备份。
|
||
//
|
||
// 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知;
|
||
// 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放
|
||
// 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。
|
||
func failLibraryTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder,
|
||
scope, jobID, backupSnapshotID, message string) error {
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理"
|
||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "prebackup", message, recovery,
|
||
backupSnapshotID, "", time.Now().UTC())
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", message, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
func kindLabel(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "恢复"
|
||
}
|
||
return "同步"
|
||
}
|
||
|
||
func digestKey(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "restore_snapshot_id"
|
||
}
|
||
return "target_snapshot_id"
|
||
}
|