Files
knowledge-governance/cmd/worker/library_baseline.go
姚章浩 a42b101a7b
build / build (push) Successful in 1m25s
fix(library): 基线崩溃续作与维护原状态保留
父审发现三处会让目标卡死或状态对不上实际入口,逐条修:

1. 登记成功但没来得及收尾:崩溃重入会撞上"已有生效版本"直接失败,
   目标留在维护里、占用也不释放。现在按「本作业+同一快照」识别为自己的
   版本,只做收尾(恢复服务/开入口/释放两把占用),不重新捕获、不重写;
   别人的版本仍然拒绝。捕获完成但登记失败则明确失败并写"未对目标写入"。
2. 重入覆盖原运行状态:崩溃时服务已被停住,再读一次只会读到 stopped,
   写回去会让恢复阶段永远不启动写入者。AcquireInstance/LibraryMaintenance
   在 owner 相同的重入时保留已持久化原状态;新作业接管已释放窗口用自己的观察。
   真实 PostgreSQL 集成用例覆盖(先红后绿)。
3. 版本登记移进维护窗口:捕获/校验/登记要么一起成立、要么目标原样恢复;
   记录失败时窗口已开放,治理状态同步释放,不再"显示维护中、实际已开"。
   索引映射读不到时拒绝登记,不再用空摘要写证据。

顺带:
- 基线窗口统一收尾抽成 settleBaselineWindow,四种结局各有测试(成功/健康失败/
  held 保持维护/已恢复则释放),并用真实 DB 用例证明 baseline 与 sync/restore
  共用写互斥、幂等重试不产生第二个作业。
- 新增 cmd/librarydrill:维护隔离 → 整库预备份 → 一致性校验 → canary 漂移检测
  → 按预备份恢复 → 逐类复核 → 按原状态恢复服务。用与 worker 相同的 LIBRARY_*
  配置,绝不触碰封存源。
- 目标/预备份/维护栈装配收成 internal/librarystack 一份(worker 与演练共用)。

验证:go build/vet/test 全过;TEST_DATABASE_URL 指向临时 PostgreSQL 跑
TestMaintenanceReentryKeepsOriginalServiceState(修复前失败:got app=stopped)
与 TestLibraryWriteMutexCoversBaselineJobs;本地门禁 15/15。
2026-10-29 19:08:58 +08:00

299 lines
15 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryBaselinePayload 是"建立目标权威基线"的队列载荷。
type libraryBaselinePayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
}
// baselineVersionState 回答"这个 scope 现在有没有生效版本、是不是本作业登记的"。
//
// 三种情况必须分开,否则崩溃续作会被自己的版本挡住:
// - own:本作业 + 同一快照已经登记过 → 只收尾,绝不重复捕获或重写版本;
// - foreign:别人登记的生效版本 → 一律拒绝(不能用基线覆盖别人的版本);
// - none:还没有生效版本 → 正常建基线。
type baselineVersionState struct {
Own bool
Foreign *governance.LibraryVersion
}
func classifyBaselineVersions(versions []governance.LibraryVersion, jobID, snapshotID string) baselineVersionState {
for index := range versions {
item := versions[index]
if item.Kind != governance.LibraryVersionProductionEffective || item.SupersededAt != nil {
continue
}
if jobID != "" && snapshotID != "" && item.SourceJobID == jobID && item.SnapshotID == snapshotID {
return baselineVersionState{Own: true}
}
foreign := item
return baselineVersionState{Foreign: &foreign}
}
return baselineVersionState{}
}
// settleBaselineWindow 是基线窗口的**唯一**收尾入口,负责让治理状态与实际入口一致:
//
// - 版本已登记(本次刚写,或崩溃前由本作业写过)→ 走与同步一样的成功边界:
// 目标健康、入口开放、两把维护锁释放之后才暴露 succeeded;
// - 没有登记且窗口仍是 held(服务没回来)→ 保持维护并标 needs_restore,
// 只有"按预备份恢复"能把它收掉;
// - 没有登记但窗口已按原状态恢复 → 释放占用并把作业标失败,明确写"未对目标写入",
// 绝不留着"显示维护中、实际已开放"的状态。
func settleBaselineWindow(ctx context.Context, store libraryMaintenanceStore, health healthChecker,
jobID, scope, instance, baselineID string, recorded bool, windowState string, windowErr error) error {
if recorded {
finishedAt := time.Now().UTC()
return finishLibrarySuccess(ctx, store, health, instance, scope, jobID,
successBoundary{Ready: true, Backup: baselineID, FinishedAt: finishedAt})
}
now := time.Now().UTC()
message := "未建立基线:维护窗口没有完成捕获与校验"
if windowErr != nil {
message += ":" + windowErr.Error()
}
if windowState == "held" {
note := message + "(目标仍被停住、入口未恢复,保持维护)"
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", note, now)
_ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "needs_restore", note, now)
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note,
"目标仍在维护:先按预备份恢复并校验", baselineID, "", now)
return errors.New(note)
}
// 窗口已按原状态开放(本流程对目标内容只读):占用必须跟着释放,
// 不能留下"界面说维护中、实际入口已开"的记录。
if releaseErr := releaseWindowLocks(ctx, store, instance, scope, jobID,
"未登记基线,目标内容未被写入,窗口已按原状态恢复", now); releaseErr != nil {
note := message + ";维护占用释放失败:" + releaseErr.Error()
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note,
"维护记录与实际入口不一致:先人工核对再重试", baselineID, "", now)
return errors.New(note)
}
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", message,
"未对目标执行任何写入;可直接重试建立基线", baselineID, "", now)
return errors.New(message)
}
// runLibraryBaseline 给"非空但没有权威版本"的目标建立基线。
//
// 真实链路(全部在同一个维护窗口内):关入口/停写入者/证明排空 → 把目标整库捕获成
// 归档快照 → 用同一套 Verify 证明目标与这份快照逐类一致 → 读目标索引映射 →
// 登记 production_effective → 关窗口(恢复服务、开入口)→ 释放两把占用 → 作业成功。
//
// 崩溃语义:
// - 捕获后、登记前被杀:作业行里已有快照 ID,重入时**不重新捕获**,直接在窗口内
// 复核这份快照再登记;
// - 登记后、收尾前被杀:重入识别出"本作业+同一快照"已生效,只做收尾
// (恢复服务/开入口/释放占用),不重复捕获、不重写版本;别人的版本仍然拒绝。
func runLibraryBaseline(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryBaselinePayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
(payload.Scope != "internal" && payload.Scope != "external") {
return errors.New("invalid library baseline payload")
}
now := time.Now().UTC()
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
fail := func(message, stage string) error {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, message, stage)
}
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
actor := ""
persistedSnapshot := ""
if jobErr == nil {
actor = existing.Actor
persistedSnapshot = existing.SnapshotID
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", "", "", "", "", now)
// 版本判定:别人的生效版本一律拒绝;本作业已登记过的快照只收尾。
versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50)
if versionErr != nil {
return fail("读不到版本记录,无法确认是否需要建立基线;未对目标执行任何操作", "capture")
}
state := classifyBaselineVersions(versions, payload.LibraryJobID, persistedSnapshot)
if state.Foreign != nil {
return fail("该目标已有生效版本 "+shortSnapshotID(state.Foreign.SnapshotID)+
"(作业 "+state.Foreign.SourceJobID+"),不需要再建基线;换版本请走同步或恢复", "capture")
}
finalizeOnly := state.Own
if finalizeOnly {
recorder.Stage(ctx, "finalize", "running", "续作:本作业已登记同一份基线快照,只做收尾(不重新捕获)")
}
if !finalizeOnly && !config.ProductionReleaseRequestsEnabled {
return fail("生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):建立基线要动目标运行状态,闸口关闭时不执行", "capture")
}
target, backups, wireErr := buildLibraryBaselineStack(ctx, config, payload.Scope)
if wireErr != nil {
return fail(wireErr.Error(), "capture")
}
defer target.Close()
manager, managerErr := openLibraryMaintenance(config)
if managerErr != nil {
return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下捕获目标:"+managerErr.Error(), "capture")
}
defer manager.Close()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "capture")
}
originalState, stateErr := manager.OriginalState(ctx)
if stateErr != nil {
return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "capture")
}
identity := manager.Target()
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
// 同一作业重入时 store 保留第一次记下的原运行状态(否则会把"已停"当原状)。
if _, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: identity.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复",
}, now); instanceErr != nil {
return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "capture")
}
if _, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: identity.Fingerprint,
Level: librarymaintenance.LevelInstance,
TargetFingerprint: identity.Fingerprint,
Services: identity.Writers,
IngressState: "unknown",
State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复",
}, now); acquireErr != nil {
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
return fail("目标正在维护中:"+acquireErr.Error(), "capture")
}
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
return fail("维护协调器不可用:"+openErr.Error(), "capture")
}
defer control.Close()
defer coordinator.Close()
var (
baselineID = persistedSnapshot
counts map[string]int64
recorded = finalizeOnly
)
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
if finalizeOnly {
// 版本已在窗口内校验并登记过:不再捕获、不再重写,收尾由下方统一处理。
return baselineID, nil
}
if baselineID == "" {
current, currentErr := target.Current(ctx)
if currentErr != nil {
return "", errors.New("读取目标当前整库状态失败:" + currentErr.Error())
}
counts = map[string]int64{}
for table, count := range current.Counts {
counts[table] = count
}
// 预备份存储在导出后用同一份指纹复核"导出期间内容没变",
// 所以这里把窗口内读到的 state 交进去,而不是事后再读一次。
backup, backupErr := backups.Prepare(ctx, "", current)
if backupErr != nil {
return "", errors.New("捕获目标整库失败:" + backupErr.Error())
}
baselineID = backup.SnapshotID
// 先把"这次窗口的基线快照"落盘:进程在这里被杀时,恢复器知道该回到哪一份,
// 重入也据此跳过重新捕获。
if markErr := window.SetBackupSnapshot(ctx, baselineID); markErr != nil {
return "", errors.New("记录基线快照身份失败:" + markErr.Error())
}
if persistErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "",
"", "", baselineID, "", time.Now().UTC()); persistErr != nil {
return "", errors.New("持久化基线快照引用失败:" + persistErr.Error())
}
recorder.Stage(ctx, "capture", "succeeded", "已封存目标整库快照 "+baselineID)
} else {
recorder.Stage(ctx, "capture", "skipped", "复用作业已持久化的基线快照 "+baselineID)
}
if baselineID == "" {
return "", errors.New("没有可校验的基线快照")
}
recorder.Stage(ctx, "verify", "running", "逐表/索引/资源比对目标与快照 "+shortSnapshotID(baselineID))
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: baselineID}); verifyErr != nil {
return "", errors.New("目标内容与基线快照不一致,拒绝登记基线:" + verifyErr.Error())
}
recorder.Stage(ctx, "verify", "succeeded", "行身份/逐行内容/索引文档/资源摘要与快照一致")
// 索引映射是基线证据的一部分:读不到就不登记,绝不用空摘要冒充。
mapping, mapErr := target.Search().MappingSHA256(ctx)
if mapErr != nil {
return "", errors.New("读不到目标索引映射,拒绝用空摘要登记基线:" + mapErr.Error())
}
if counts == nil {
if current, currentErr := target.Current(ctx); currentErr == nil {
counts = current.Counts
}
}
// 版本登记放在窗口内:捕获、校验、登记三件事要么一起成立,要么目标原样恢复。
recorder.Stage(ctx, "finalize", "running", "登记目标权威基线")
if recordErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, baselineID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: baselineID, SourceJobID: payload.LibraryJobID, CreatedBy: actor,
Evidence: map[string]any{
"action": "target_baseline_init",
"origin": "target_baseline",
"baseline_snapshot": baselineID,
"index_mapping_sha256": mapping,
"scope_content_counts": counts,
"verification": "目标与快照逐表行身份/逐行内容/索引文档/资源摘要一致",
"note": "基线是目标在维护窗口内的真实整库捕获,不是手写版本;发布仍需独立 sync 授权",
},
CreatedAt: time.Now().UTC(),
}); recordErr != nil {
return "", errors.New("登记基线版本失败:" + recordErr.Error())
}
recorded = true
recorder.Stage(ctx, "finalize", "succeeded", "已登记目标权威基线 "+shortSnapshotID(baselineID))
return baselineID, nil
}
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID,
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
// 本流程对目标内容只读:失败时按原状态重新开放永远安全。
func(error) bool { return true })
if settleErr := settleBaselineWindow(ctx, store, manager, payload.LibraryJobID, payload.Scope,
identity.Instance, baselineID, recorded, window.State, windowErr); settleErr != nil {
return settleErr
}
log.Printf("library baseline succeeded job=%s scope=%s snapshot=%s actor=%s finalize_only=%v",
payload.LibraryJobID, payload.Scope, baselineID, actor, finalizeOnly)
return nil
}
// shortSnapshotID 只用于人看的日志/阶段说明:完整摘要留在版本记录与接口里。
func shortSnapshotID(id string) string {
if len(id) <= 12 {
return id
}
return id[:12]
}