package main import ( "context" "encoding/json" "errors" "log" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryfreeze" "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryrelease" postgresstore "knowledge-governance/internal/platform/postgres" ) // libraryBaselinePayload 是"建立目标权威基线"的队列载荷。 type libraryBaselinePayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` } // classifyBaselineVersions 回答"这个 scope 现在有没有生效版本、是不是本作业登记的"。 // // 输入是**当前生效版本**这一个精确查询的结果(每 scope 至多一行),不是"最近 N 条": // 窗口一满,已登记过的基线会被读成"没有",于是重复捕获、覆盖别人的版本。 // // 三种情况必须分开,否则崩溃续作会被自己的版本挡住: // - own:本作业 + 同一快照已经登记过 → 只收尾,绝不重复捕获或重写版本; // - foreign:别人登记的生效版本 → 一律拒绝(不能用基线覆盖别人的版本); // - none:还没有生效版本 → 正常建基线。 type baselineVersionState struct { Own bool Foreign *governance.LibraryVersion } func classifyBaselineVersions(current governance.LibraryVersion, hasCurrent bool, jobID, snapshotID string) baselineVersionState { if !hasCurrent { return baselineVersionState{} } if effectiveOwnedBy(current, jobID, snapshotID) { return baselineVersionState{Own: true} } foreign := current return baselineVersionState{Foreign: &foreign} } // settleBaselineWindow 是基线窗口的**唯一**收尾入口,负责让治理状态与实际入口一致: // // - 版本已登记(本次刚写,或崩溃前由本作业写过)→ 走与同步一样的成功边界: // 目标健康、入口开放、两把维护锁释放之后才暴露 succeeded; // - 没有登记且窗口仍是 held(服务没回来)→ 保持维护并标 needs_restore, // 只有"按预备份恢复"能把它收掉; // - 没有登记但窗口已按原状态恢复 → 释放占用并把作业标失败,明确写"未对目标写入", // 绝不留着"显示维护中、实际已开放"的状态。 func settleBaselineWindow(ctx context.Context, store libraryMaintenanceStore, health healthChecker, jobID, scope, instance, baselineID string, recorded bool, windowState string, windowErr error) error { if recorded { finishedAt := time.Now().UTC() return finishLibrarySuccess(ctx, store, health, instance, scope, jobID, successBoundary{Ready: true, Backup: baselineID, FinishedAt: finishedAt}) } now := time.Now().UTC() message := "未建立基线:维护窗口没有完成捕获与校验" if windowErr != nil { message += ":" + windowErr.Error() } if windowState == "held" { note := message + "(目标仍被停住、入口未恢复,保持维护)" _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", note, now) _ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "needs_restore", note, now) _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note, "目标仍在维护:先按预备份恢复并校验", baselineID, "", now) return errors.New(note) } // 窗口已按原状态开放(本流程对目标内容只读):占用必须跟着释放, // 不能留下"界面说维护中、实际入口已开"的记录。 if releaseErr := releaseWindowLocks(ctx, store, instance, scope, jobID, "未登记基线,目标内容未被写入,窗口已按原状态恢复", now); releaseErr != nil { note := message + ";维护占用释放失败:" + releaseErr.Error() _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note, "维护记录与实际入口不一致:先人工核对再重试", baselineID, "", now) return errors.New(note) } _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", message, "未对目标执行任何写入;可直接重试建立基线", baselineID, "", now) return errors.New(message) } // runLibraryBaseline 给"非空但没有权威版本"的目标建立基线。 // // 真实链路(全部在同一个维护窗口内):关入口/停写入者/证明排空 → 把目标整库捕获成 // 归档快照 → 用同一套 Verify 证明目标与这份快照逐类一致 → 读目标索引映射 → // 登记 production_effective → 关窗口(恢复服务、开入口)→ 释放两把占用 → 作业成功。 // // 崩溃语义: // - 捕获后、登记前被杀:作业行里已有快照 ID,重入时**不重新捕获**,直接在窗口内 // 复核这份快照再登记; // - 登记后、收尾前被杀:重入识别出"本作业+同一快照"已生效,只做收尾 // (恢复服务/开入口/释放占用),不重复捕获、不重写版本;别人的版本仍然拒绝。 func runLibraryBaseline(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryBaselinePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || (payload.Scope != "internal" && payload.Scope != "external") { return errors.New("invalid library baseline payload") } // 准入与代次:基线也会停写入者、取整库快照,同样是"动目标"的执行, // 必须和 sync/restore 用同一套准入与 fencing(F14/F13)。 if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, governance.LibraryJobBaseline, true); admitErr != nil { return admitErr } writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID)) if attemptErr != nil { return attemptErr } store := writer now := time.Now().UTC() recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, operationID: writer.attempt.OperationID} fail := func(message, stage string) error { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, message, stage) } existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID) actor := "" persistedSnapshot := "" if jobErr == nil { actor = existing.Actor persistedSnapshot = existing.SnapshotID } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", "", "", "", "", now) // 版本判定:别人的生效版本一律拒绝;本作业已登记过的快照只收尾。 // 用精确查询(当前生效版本),不用"最近 N 条":窗口一满会把已登记读成没登记。 current, hasCurrent, versionErr := store.LibraryCurrentVersion(ctx, payload.Scope) if versionErr != nil { return fail("读不到生效版本记录,无法确认是否需要建立基线;未对目标执行任何操作", "capture") } state := classifyBaselineVersions(current, hasCurrent, payload.LibraryJobID, persistedSnapshot) if state.Foreign != nil { return fail("该目标已有生效版本 "+shortSnapshotID(state.Foreign.SnapshotID)+ "(作业 "+state.Foreign.SourceJobID+"),不需要再建基线;换版本请走同步或恢复", "capture") } finalizeOnly := state.Own if finalizeOnly { recorder.Stage(ctx, "finalize", "running", "续作:本作业已登记同一份基线快照,只做收尾(不重新捕获)") } if !finalizeOnly && !config.ProductionReleaseRequestsEnabled { return fail("生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):建立基线要动目标运行状态,闸口关闭时不执行", "capture") } target, backups, wireErr := buildLibraryBaselineStack(ctx, config, payload.Scope) if wireErr != nil { return fail(wireErr.Error(), "capture") } defer target.Close() manager, managerErr := openLibraryMaintenance(config) if managerErr != nil { return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下捕获目标:"+managerErr.Error(), "capture") } defer manager.Close() if topoErr := manager.VerifyTopology(ctx); topoErr != nil { return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "capture") } originalState, stateErr := manager.OriginalState(ctx) if stateErr != nil { return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "capture") } identity := manager.Target() // 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。 // 同一作业重入时 store 保留第一次记下的原运行状态(否则会把"已停"当原状)。 if _, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope, TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复", }, now); instanceErr != nil { return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "capture") } if _, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: identity.Fingerprint, Level: librarymaintenance.LevelInstance, TargetFingerprint: identity.Fingerprint, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复", }, now); acquireErr != nil { _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", "scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now) return fail("目标正在维护中:"+acquireErr.Error(), "capture") } coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) if openErr != nil { _ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) return fail("维护协调器不可用:"+openErr.Error(), "capture") } defer control.Close() defer coordinator.Close() var ( baselineID = persistedSnapshot counts map[string]int64 recorded = finalizeOnly ) work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) { if finalizeOnly { // 版本已在窗口内校验并登记过:不再捕获、不再重写,收尾由下方统一处理。 return baselineID, nil } if baselineID == "" { current, currentErr := target.Current(ctx) if currentErr != nil { return "", errors.New("读取目标当前整库状态失败:" + currentErr.Error()) } counts = map[string]int64{} for table, count := range current.Counts { counts[table] = count } // 预备份存储在导出后用同一份指纹复核"导出期间内容没变", // 所以这里把窗口内读到的 state 交进去,而不是事后再读一次。 backup, backupErr := backups.Prepare(ctx, "", current) if backupErr != nil { return "", errors.New("捕获目标整库失败:" + backupErr.Error()) } baselineID = backup.SnapshotID // 先把"这次窗口的基线快照"落盘:进程在这里被杀时,恢复器知道该回到哪一份, // 重入也据此跳过重新捕获。 if markErr := window.SetBackupSnapshot(ctx, baselineID); markErr != nil { return "", errors.New("记录基线快照身份失败:" + markErr.Error()) } if persistErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "", "", "", baselineID, "", time.Now().UTC()); persistErr != nil { return "", errors.New("持久化基线快照引用失败:" + persistErr.Error()) } recorder.Stage(ctx, "capture", "succeeded", "已封存目标整库快照 "+baselineID) } else { recorder.Stage(ctx, "capture", "skipped", "复用作业已持久化的基线快照 "+baselineID) } if baselineID == "" { return "", errors.New("没有可校验的基线快照") } recorder.Stage(ctx, "verify", "running", "逐表/索引/资源比对目标与快照 "+shortSnapshotID(baselineID)) if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: baselineID}); verifyErr != nil { return "", errors.New("目标内容与基线快照不一致,拒绝登记基线:" + verifyErr.Error()) } recorder.Stage(ctx, "verify", "succeeded", "行身份/逐行内容/索引文档/资源摘要与快照一致") // 索引映射是基线证据的一部分:读不到就不登记,绝不用空摘要冒充。 mapping, mapErr := target.Search().MappingSHA256(ctx) if mapErr != nil { return "", errors.New("读不到目标索引映射,拒绝用空摘要登记基线:" + mapErr.Error()) } if counts == nil { if current, currentErr := target.Current(ctx); currentErr == nil { counts = current.Counts } } // 版本登记放在窗口内:捕获、校验、登记三件事要么一起成立,要么目标原样恢复。 recorder.Stage(ctx, "finalize", "running", "登记目标权威基线") if recordErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, baselineID, writer.attempt.OperationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: baselineID, SourceJobID: payload.LibraryJobID, CreatedBy: actor, Evidence: map[string]any{ "action": "target_baseline_init", "origin": "target_baseline", "baseline_snapshot": baselineID, "index_mapping_sha256": mapping, "scope_content_counts": counts, "verification": "目标与快照逐表行身份/逐行内容/索引文档/资源摘要一致", "note": "基线是目标在维护窗口内的真实整库捕获,不是手写版本;发布仍需独立 sync 授权", }, CreatedAt: time.Now().UTC(), }); recordErr != nil { return "", errors.New("登记基线版本失败:" + recordErr.Error()) } recorded = true recorder.Stage(ctx, "finalize", "succeeded", "已登记目标权威基线 "+shortSnapshotID(baselineID)) return baselineID, nil } window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work, // 本流程对目标内容只读:失败时按原状态重新开放永远安全。 func(error) bool { return true }) if settleErr := settleBaselineWindow(ctx, store, manager, payload.LibraryJobID, payload.Scope, identity.Instance, baselineID, recorded, window.State, windowErr); settleErr != nil { return settleErr } log.Printf("library baseline succeeded job=%s scope=%s snapshot=%s actor=%s finalize_only=%v", payload.LibraryJobID, payload.Scope, baselineID, actor, finalizeOnly) return nil } // shortSnapshotID 只用于人看的日志/阶段说明:完整摘要留在版本记录与接口里。 func shortSnapshotID(id string) string { if len(id) <= 12 { return id } return id[:12] }