package main import ( "context" "encoding/json" "errors" "fmt" "log" "strings" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryfreeze" "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" "knowledge-governance/internal/pgdsn" postgresstore "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/testbarrier" ) // 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放: // - contentUntouched:这次窗口还没写过目标(可以安全开放) // - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放) // - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护) const ( contentUntouched = "" contentSucceeded = "succeeded" contentRolledBack = "rolled_back" contentHalfWritten = "half_written" contentRollbackFails = "rollback_failed" ) // contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。 // 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。 // // 另外它是本机中断演练的固定点位:每个阶段写完、下一个阶段还没开始时调用 // testbarrier.Wait(默认关闭,两个环境变量同时在场才生效)。放在这一层而不是 // 发布代码里,是因为"阶段之间"正是真实半写状态的边界,而发布链路本身不需要知道演练。 type contentTrackedTarget struct { inner libraryrelease.Target jobID string onApply func() } func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) { return t.inner.Current(ctx) } func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error { t.onApply() if err := t.inner.Apply(ctx, stage, snapshotID); err != nil { return err } // 屏障错误只记日志:演练设施坏掉不许让一次真实发布失败。 if barrierErr := testbarrier.Wait(ctx, "after:"+string(stage), t.jobID); barrierErr != nil { log.Printf("library test barrier point=after:%s job=%s unavailable: %v", stage, t.jobID, barrierErr) } return nil } func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error { return t.inner.Verify(ctx, expect) } func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error { return t.inner.Restore(ctx, backup) } // PlanWriteSet 必须显式转发:orchestrator 是按 `target.(WriteSetPlanner)` 这个 // 类型断言决定要不要在写第一笔之前算写集合的,包一层之后断言就失败了——于是 // 台账既不落盘也不登记,恢复/接管时根本没有"这次明确写了什么"可读。 // 真实后果:2026-10-29 的两次真作业的预备份证据里都没有 writeset_digest, // 归档根下也没有 writesets/;一旦 worker 崩在写入中间,接管必然失败。 // // 内层不支持预检时返回硬错误,不退回"没有台账也照写":那会让恢复只能按目标 // 现状推断残渣,从而误删历史孤儿资源。 func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) { planner, ok := t.inner.(libraryrelease.WriteSetPlanner) if !ok { return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产") } set, err := planner.PlanWriteSet(ctx, plan) if err == nil { // 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。 log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s", set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID) } return set, err } // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 type libraryReleasePayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` TargetSnapshotID string `json:"target_snapshot_id"` // FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。 // 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。 FinalizeOnly bool `json:"finalize_only,omitempty"` } // openLibraryMaintenance 定义在 library_wiring.go:维护目标的装配与目标整库栈共用 // 同一个包(internal/librarystack),不在这里再拼一份。 // runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 → // 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。 // 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。 func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job, kind governance.LibraryJobKind) (err error) { var payload libraryReleasePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" { return fmt.Errorf("invalid library %s payload", kind) } // 准入必须在任何维护/目标动作之前:业务行存在、kind/scope 一致、队列关联不可伪造。 if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil { return admitErr } // 执行代次:这一次投递是一个确定的执行者身份。所有治理写入都走 writer // (同名方法带 epoch 守卫),失租的旧执行从此写不进业务状态/维护锁(F13)。 writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID)) if attemptErr != nil { return attemptErr } defer func() { if err != nil { writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptFailed, err.Error()) return } writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "") }() store := writer now := time.Now().UTC() _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now) recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, operationID: store.attempt.OperationID} // 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。 if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil { if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil { recorder.totals = stageSizesFrom(manifest) } archive.Close() } // 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续, // 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。 previous, err := store.LibraryJob(ctx, payload.LibraryJobID) resumed := err == nil && previous.SnapshotID != "" // 续作失败时不能写"无需恢复":上一次尝试可能已经把目标写了一半。 // 这一处按"有没有持久化预备份"统一选择失败出口,避免每个早退点各写一套文案。 fail := func(message, stage string) error { return failLibraryJobOrTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID, resumed, message, stage) } if resumed { facts := loadLibraryVersionFacts(ctx, store, payload.Scope, previous.TargetSnapshotID) resume := loadTakeoverResumeFacts(ctx, store, config.LibraryArchiveRoot, previous, now) decision := classifyTakeover(previous, facts, resume) // 接管决策必须"动手之前"落日志,并带作业身份与复用的预备份: // 只留阶段表的话,同快照收尾能把它伪造成"完整续作"(上一轮的假完成)。 logTakeoverDecision(payload.LibraryJobID, decision, kind, payload.Scope, payload.TargetSnapshotID, previous.SnapshotID, decision == takeoverPendingFinal, time.Now().UTC()) switch decision { case takeoverUnknown: // 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。 return fail("接管时读不到该目标的版本状态,无法判断是只差收尾还是要重做;本作业未继续写目标,稍后重试", "prebackup") case takeoverSuperseded: return fail("本快照的生效版本已被别的快照顶替,且本作业没有新的恢复授权/独占维护窗口/未变基线,拒绝用旧快照覆盖当前生产版本", "prebackup") case takeoverForeignEffective: // 同一个快照由别的作业发布过:那只能证明生产上有这份内容,不能证明本作业 // 写完过、更不能证明它只差收尾。按同快照收尾会把只写了第一个阶段的作业 // 标成成功,并改写版本归属(实测过的假完成)。 owner := facts.Current.SourceJobID if facts.HasSnapshot { owner = facts.Snapshot.SourceJobID } return failLibraryTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID, "接管时目标快照 "+shortSnapshotID(payload.TargetSnapshotID)+" 的生效版本由作业 "+owner+ " 登记:本作业没有自己的完成证据,不按同快照收尾") case takeoverPendingFinal: // 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。 payload.FinalizeOnly = true } } if resumed { recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID) } gateReasons := make([]string, 0, 3) if !config.ProductionReleaseRequestsEnabled { gateReasons = append(gateReasons, "生产写入闸口关闭") } if !config.ProductionBackupRestoreVerified { gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据") } if len(gateReasons) > 0 { message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";") recorder.Stage(ctx, "prebackup", "failed", message) // 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的, // 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。 recovery := "未对生产执行任何写入,无需恢复" if resumed { recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message, recovery, previous.SnapshotID, "", time.Now().UTC()) return errors.New(message) } digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID}) if err != nil { return fail("无法计算请求摘要", "prebackup") } grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now) if err != nil || !found { return fail("没有与目标快照匹配的有效发布授权", "prebackup") } // 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己 // 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。 priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8) if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil { log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr) } else { for _, source := range sources { priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID, SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt}) } } // 生产索引映射基线:权威来源是当前生效版本(精确查),其次任何一次成功记录的映射。 // 只认"生效版本"会在恢复把版本顶替掉之后误判"索引被改动"。 knownMapping := knownIndexMapping(ctx, store, payload.Scope) target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping) if wireErr == nil { if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { recorder.mappingDigest = mapping } } if wireErr != nil { return fail(wireErr.Error(), "prebackup") } defer target.Close() // 容量门禁必须在**任何副作用之前**(F11):预算不足就不该进维护、不该拍备份。 // 旧实现只在存储页 GET 里算这个数,worker 从不读它,于是"预算为正但不够"照样写。 if capErr := checkLibraryCapacity(ctx, config, payload.TargetSnapshotID); capErr != nil { return fail(capErr.Error(), "prebackup") } // 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。 manager, managerErr := openLibraryMaintenance(config) if managerErr != nil { return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup") } defer manager.Close() // 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。 if topoErr := manager.VerifyTopology(ctx); topoErr != nil { return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup") } targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) originalState, stateErr := manager.OriginalState(ctx) if stateErr != nil { return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup") } identity := manager.Target() // 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。 instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope, TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if instanceErr != nil { return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup") } _ = instanceLock maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity + " | " + manager.Description(), Level: librarymaintenance.LevelInstance, TargetFingerprint: identity.Fingerprint, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if acquireErr != nil { _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", "scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now) return fail("目标正在维护中:"+acquireErr.Error(), "prebackup") } _ = maintenance coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) if openErr != nil { // 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把 // 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。 if resumed { _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) } else { _ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) } return fail("维护协调器不可用:"+openErr.Error(), "prebackup") } defer control.Close() defer coordinator.Close() // 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前, // 一律按半写对待,任何早退都不许重新开放目标。 contentState := contentUntouched if resumed { contentState = contentHalfWritten } pending := successBoundary{} tracked := &contentTrackedTarget{inner: target, jobID: payload.LibraryJobID, onApply: func() { contentState = contentHalfWritten }} allowRestore := func(error) bool { return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack } execute := func(ctx context.Context) error { if payload.FinalizeOnly { // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil { contentState = contentHalfWritten message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } contentState = contentSucceeded mappingDigest, _ := target.Search().MappingSHA256(ctx) // 收尾续作只补登记,不允许把首次锚点洗掉:字节数与写集合摘要从已持久化的 // 预备份证据合并进来(F09)。缺失即拒绝,不写一个"看起来完整"的空证据。 evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID, previous.SnapshotID, payload.TargetSnapshotID, "finalize_only", mappingDigest) if evidenceErr != nil { message := "收尾续作缺少首次锚点,拒绝登记生效版本:" + evidenceErr.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+previous.SnapshotID+" 保留;补齐首次锚点证据后再续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(), }); err != nil { message := "收尾续作仍无法登记生效版本:" + err.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") // 成功要等维护锁释放完才暴露:这里只记"待收尾"。 pending = successBoundary{Ready: true, Backup: previous.SnapshotID, Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()} return nil } var backupSource libraryrelease.BackupStore = backups if resumed { // 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣, // 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。 set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, payload.LibraryJobID, previous.SnapshotID) if ledgerErr != nil { // 读不到就一笔都不许写:目标可能半写,恢复口径未知。 message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error() _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) recorder.Stage(ctx, "prebackup", "failed", message) return errors.New(message) } // 复用不重做备份,真实字节只能从首次登记的证据里取:缺了就必须停, // 否则生效版本证据会写成 backup_bytes:0(现场就是这样丢的)。 backupBytes, bytesErr := prebackupBytesOf(ctx, store, payload.Scope, payload.LibraryJobID, previous.SnapshotID) if bytesErr != nil { message := "接管时读不回预备份的真实字节数,拒绝继续写目标:" + bytesErr.Error() _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) recorder.Stage(ctx, "prebackup", "failed", message) return errors.New(message) } backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, SnapshotID: previous.SnapshotID, Bytes: backupBytes}, write: set} } request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now, PriorSnapshots: priorSnapshots, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder) finishedAt := time.Now().UTC() switch outcome.State { case "succeeded": contentState = contentSucceeded mappingDigest := "" if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { mappingDigest = mapping } // 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。 // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, Evidence: effectiveEvidence(outcome.Backup, string(kind), mappingDigest), CreatedAt: finishedAt, }) if versionErr != nil { message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error() recorder.Stage(ctx, "finalize", "failed", message) // 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。 if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记", outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr) } return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") // 内容与版本都落定了,但还不能报成功:服务健康、入口开放、两把维护锁 // 释放完成才允许暴露 succeeded(见 finishLibrarySuccess)。 pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID, Target: payload.TargetSnapshotID, FinishedAt: finishedAt} return nil case "rolled_back": contentState = contentRolledBack _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return errors.New(outcome.Error) case "rollback_failed": contentState = contentRollbackFails _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) default: recovery := outcome.Recovery if recovery == "" { recovery = "未对生产执行任何写入,无需恢复" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt) return errors.New(outcome.Error) } } prepare := func(ctx context.Context) error { // 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口, // 只会把"本来坏的"记成"维护造成的"。 return manager.VerifyRunning(ctx) } windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) { // 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的 // 窗口之后既不能自动恢复,也不许被任何收尾路径开放。 recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error { if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil { return err } return w.MarkContentRisk(backupCtx) } if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed", "目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil { // 记录失败不影响隔离本身,但必须留痕,不能假装记上了。 log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr) } return "", execute(ctx) } window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore) // 终态重放(F05):journal 里已经 succeeded,本次没有再执行 work。内容/服务子协议 // 已经完成,缺的只可能是"业务收尾"(版本登记、锁释放、终态)。这时必须从**持久事实** // 重建收尾凭据,而不是靠进程内 pending——那个指针早就随崩溃没了。 if windowErr == nil && window.State == "succeeded" && contentState != contentSucceeded { rebuilt, rebuildErr := rebuildSuccessBoundary(ctx, store, recorder, payload, previous, kind, grant) if rebuildErr != nil { note := "窗口已成功但收尾凭据无法从持久事实重建,目标保持维护:" + rebuildErr.Error() _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, time.Now().UTC()) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, time.Now().UTC()) return errors.New(note) } pending = rebuilt contentState = contentSucceeded } // 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。 now2 := time.Now().UTC() switch { case window.State == "held": note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复" if windowErr != nil { note += ":" + windowErr.Error() } _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2) if windowErr == nil { return errors.New(note) } return windowErr case contentState == contentSucceeded: if !pending.Ready { // 内容状态说成功但没有收尾凭据(不可能两者都缺):按未收尾处理,保持维护。 note := "内容状态为已验证,但缺少收尾凭据,目标保持维护" _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2) return errors.New(note) } if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope, payload.LibraryJobID, pending); finishErr != nil { return finishErr } log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup) return windowErr case contentState == contentRolledBack: if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil { return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err) } return windowErr default: // 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。 if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未写目标,维护窗口未开启", now2); err != nil { return fmt.Errorf("维护占用释放失败: %w", err) } return windowErr } } // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) { interrupted, err := store.InterruptedLibraryJobs(ctx) if err != nil { return 0, 0, err } resumed, failed := 0, 0 for _, job := range interrupted { now := time.Now().UTC() if job.SnapshotID == "" { message := "worker 中断时还没有完成预备份,不能盲写目标" _ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message, "未对生产执行写入", "", "", now) failed++ continue } // 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。 claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now) if claimErr != nil { log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr) continue } if !claimed { continue } queueType := librarySyncJobType switch job.Kind { case governance.LibraryJobRestore: queueType = libraryRestoreJobType case governance.LibraryJobBaseline: // 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口, // 继续捕获/校验,而不是留下一个 held 目标等人工处理。 queueType = libraryBaselineJobType } payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope, "target_snapshot_id": job.TargetSnapshotID}) // 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。 queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{ ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID, }, now) if err != nil || queueJob == nil { // 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了, // 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。 if err != nil && strings.Contains(err.Error(), "duplicate key") { queueID := governance.QueueJobIDFor(job.ID) if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil { log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil { log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr) } else { resumed++ logTakeoverClaim(job, queueID, takeoverPathExistingQueue, now) } continue } log.Printf("library job %s resume enqueue failed: %v", job.ID, err) continue } if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil { log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil { log.Printf("library job %s resume attach failed: %v", job.ID, attachErr) continue } resumed++ // 只有 claim + attach 都成功才写这一行:它等于"这个作业归本 worker 继续跑", // 外部用它判接管来源(汇总计数证明不了是哪个作业)。 logTakeoverClaim(job, queueJob.ID, takeoverPathEnqueue, now) } return resumed, failed, nil } // BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误: // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 // // 接管时 BackupRef.Reused=true:复用上一次崩溃前登记的那一份。这时必须把 // "复用"写进证据 —— 否则外部只能看到一条 created_at 很新的版本行, // 分不清是复用还是重做了备份(重做备份正好把半写状态当成了基线)。 // // ⚠️ 证据必须**合并**进已有那一行,不能每次造一份新的:RecordLibraryVersion 是 // `evidence = EXCLUDED.evidence`(整份覆盖)。2026-10-30 的演练里,接管复用时 // 新造的证据只有 backup_* / job / index_mapping_sha256,把首次规划写入的 // writeset_digest/writeset_rows… 整段抹掉,第二次中断再接管就再也读不回写集合 // (verify_effective 因此从有摘要变成空)。合并只对**同一作业 + 同一份预备份** // 成立:身份不符一律失败,绝不把别的作业的证据带进来。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { now := time.Now().UTC() if r.onBackup != nil { if err := r.onBackup(ctx, backup); err != nil { return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err) } } // 先读既有证据:复用声称的那一份必须真的登记过,新造的也不许盖掉别人的行。 existing, found, err := r.store.LibraryVersionByRef(ctx, r.scope, governance.LibraryVersionProductionPrebackup, r.jobID, backup.SnapshotID) if err != nil { return fmt.Errorf("读不到现有预备份证据: %w", err) } if !found && backup.Reused { return fmt.Errorf("预备份 %s 声称复用,但没有作业 %s 的版本证据:拒绝把复用登记成新备份", backup.SnapshotID, r.jobID) } if found && !backup.Reused { // 同一份预备份已经登记过,本轮却声称是新备份:那不是"重做备份", // 而是用一份新证据替换已持久化的写集合锚点。 return fmt.Errorf("预备份 %s 已登记为作业 %s 的版本证据,本轮却不是复用:拒绝覆盖已持久化证据", backup.SnapshotID, r.jobID) } evidence, err := prebackupEvidence(existing.Evidence, backup, r.jobID, r.mappingDigest) if err != nil { return fmt.Errorf("合并预备份证据失败: %w", err) } if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "", backup.SnapshotID, "", now); err != nil { return fmt.Errorf("持久化预备份引用失败: %w", err) } if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID, r.operationID), Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: r.jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: now, }); err != nil { return fmt.Errorf("登记预备份版本失败: %w", err) } return nil } func failLibraryJob(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, jobID, message, stage string) error { recorder.Stage(ctx, stage, "failed", message) _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message, "未对生产执行任何写入,无需恢复", "", "", time.Now().UTC()) return errors.New(message) } // failLibraryJobOrTakeover 按"这个作业有没有持久化预备份"选择失败出口。 // // 有预备份就说明上一次尝试可能已经写过目标:这时任何早退(授权过期、装配失败、 // 维护被别人占用……)都必须保留 needs_restore 并点名基线,不能写"无需恢复"—— // 那会让页面把半写目标当成完好,也让操作者手里没有可回滚的那一份。 func failLibraryJobOrTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, scope, jobID, backupSnapshotID string, resumed bool, message, stage string) error { if resumed { return failLibraryTakeover(ctx, store, recorder, scope, jobID, backupSnapshotID, message) } return failLibraryJob(ctx, store, recorder, jobID, message, stage) } // failLibraryTakeover 是"拒绝接管"的失败出口:目标可能已被上一次尝试写过, // 不能像 failLibraryJob 那样写"无需恢复"。维护占用保持现状(held/acquired 就是 // 安全状态),scope 行显式标 needs_restore,作业终态失败并指明可回滚的预备份。 // // 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知; // 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放 // 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。 func failLibraryTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, scope, jobID, backupSnapshotID, message string) error { recorder.Stage(ctx, "prebackup", "failed", message) recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理" _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "prebackup", message, recovery, backupSnapshotID, "", time.Now().UTC()) _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", message, time.Now().UTC()) return errors.New(message) } func kindLabel(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "恢复" } return "同步" } func digestKey(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "restore_snapshot_id" } return "target_snapshot_id" }