b4add6c9b0
B1-R1: 演练写集改为受限 restore-drill(V2 信封/contract 1/schema 摘要,零对象、零索引、零本地映射,来源身份必填)。B2-R1: verify 作业先建不可变 operation 并携带 operation_id,同 job recover 的网关 claim 同时携带原 operation 与 recovery 身份。B5-R1: 前端按 scope 世代与动作序号丢弃迟到响应,不污染新 scope 或父级旧回调。harness 的 recovery 改由生产 jobs.Worker.RunOnce 消费并读回 jobs 终态;原失败窗口作业保持 failed,不改写历史。
446 lines
20 KiB
Go
446 lines
20 KiB
Go
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
|
||
//
|
||
// 维护隔离(关入口/停写入者/证明排空)
|
||
// → 捕获目标整库(真预备份,落归档)
|
||
// → 校验目标 == 预备份(基线一致)
|
||
// → 写入一个 canary(证明"校验确实能发现内容变化")
|
||
// → 按预备份恢复
|
||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||
//
|
||
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
|
||
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
|
||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
|
||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||
package libraryverify
|
||
|
||
import (
|
||
"context"
|
||
"crypto/rand"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarystack"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
type Verification struct {
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Verified bool `json:"verified"`
|
||
Error string `json:"error,omitempty"`
|
||
At time.Time `json:"at"`
|
||
}
|
||
|
||
type Report struct {
|
||
Status string `json:"status"` // passed | failed | maintenance_held
|
||
Scope string `json:"scope"`
|
||
Target string `json:"target"`
|
||
SnapshotID string `json:"snapshot_id"` // 写前镜像:恢复实际回放的那份预备份
|
||
SourceSnapshot string `json:"source_snapshot_id,omitempty"`
|
||
WriteSetDigest string `json:"writeset_digest,omitempty"`
|
||
WindowState string `json:"window_state,omitempty"`
|
||
Project string `json:"project"`
|
||
Counts map[string]int64 `json:"counts"`
|
||
BaselineCheck *Verification `json:"baseline_check"`
|
||
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||
RestoreCheck *Verification `json:"restore_check"`
|
||
Canary string `json:"canary"`
|
||
Failure string `json:"failure,omitempty"`
|
||
Stage string `json:"stage,omitempty"`
|
||
StartedAt time.Time `json:"started_at"`
|
||
FinishedAt time.Time `json:"finished_at"`
|
||
Until map[string]string `json:"until,omitempty"`
|
||
}
|
||
|
||
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
|
||
type Progress func(stage, state, detail string)
|
||
|
||
func (p Progress) report(stage, state, detail string) {
|
||
if p != nil {
|
||
p(stage, state, detail)
|
||
}
|
||
}
|
||
|
||
// TargetIdentity 是治理侧维护记录必须钉住的目标事实(与 librarymaintenance.Target 同源)。
|
||
type TargetIdentity struct {
|
||
Instance string
|
||
Fingerprint string
|
||
Writers []string
|
||
}
|
||
|
||
// Governance 是演练必须持有的**治理侧**维护生命周期:实例/scope 占用、入口真实状态、
|
||
// 释放。为什么不能让调用方只写磁盘 journal:页面与恢复入口读的是治理行,只写 journal
|
||
// 时"正在维护"不会出现在 UI,失败后也没有同 job 的恢复锚点(2026-11 审核 B2)。
|
||
type Governance interface {
|
||
Acquire(ctx context.Context, jobID, scope string, target TargetIdentity, originalServiceState string) error
|
||
Ingress(ctx context.Context, jobID, scope, state, note string) error
|
||
State(ctx context.Context, jobID, scope, state, note string) error
|
||
Release(ctx context.Context, jobID, scope, note string) error
|
||
}
|
||
|
||
// Ledger 把预备份身份与写集合摘要持久化到**同一个作业**(worker 直接传
|
||
// libraryStageRecorder:它已经是 BackupObserver + WriteSetObserver)。
|
||
// 恢复入口按 (scope, job, 写前镜像) 精确读这条记录,所以两件事必须发生在写第一笔之前。
|
||
type Ledger interface {
|
||
BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error
|
||
WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error
|
||
}
|
||
|
||
// Options 是演练的可选接线。Governance/Ledger 为 nil 只允许主机侧 CLI 使用:
|
||
// 页面发起的作业必须两者都在,否则演练结论不会出现在治理记录里。
|
||
type Options struct {
|
||
Progress Progress
|
||
Governance Governance
|
||
Ledger Ledger
|
||
}
|
||
|
||
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
|
||
// 否则崩溃恢复时窗口与作业对不上。
|
||
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, opts Options) (*Report, error) {
|
||
if scope != "internal" && scope != "external" {
|
||
return nil, errors.New("scope must be internal or external")
|
||
}
|
||
progress := opts.Progress
|
||
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||
fail := func(stage string, cause error, held bool) (*Report, error) {
|
||
report.Stage = stage
|
||
report.Failure = cause.Error()
|
||
report.FinishedAt = time.Now().UTC()
|
||
if held {
|
||
report.Status = "maintenance_held"
|
||
}
|
||
progress.report(stage, "failed", cause.Error())
|
||
return report, cause
|
||
}
|
||
|
||
progress.report("capture", "running", "打开目标栈与维护管理器")
|
||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||
if err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
defer target.Close()
|
||
manager, err := librarystack.OpenMaintenance(config)
|
||
if err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
defer manager.Close()
|
||
if err := manager.VerifyTopology(ctx); err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
originalState, err := manager.OriginalState(ctx)
|
||
if err != nil {
|
||
return fail("capture", errors.New("读不到目标写入者的原运行状态,不能开维护窗口:"+err.Error()), false)
|
||
}
|
||
identity := manager.Target()
|
||
// 治理占用必须在任何目标动作之前:拿不到就没有"入口已关/写入者已停"的权威记录,
|
||
// 失败后也没有可恢复的同 job 锚点。
|
||
if opts.Governance != nil {
|
||
targetIdentity := TargetIdentity{Instance: identity.Instance, Fingerprint: identity.Fingerprint, Writers: identity.Writers}
|
||
if err := opts.Governance.Acquire(ctx, jobID, scope, targetIdentity, originalState); err != nil {
|
||
return fail("capture", errors.New("目标实例正在维护中或无法登记维护占用:"+err.Error()), false)
|
||
}
|
||
}
|
||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||
if err != nil {
|
||
if opts.Governance != nil {
|
||
// 还没进窗口、目标未被停用:释放占用是诚实的;释放失败就报告出来。
|
||
_ = opts.Governance.Release(ctx, jobID, scope, "维护协调器不可用,未进入窗口")
|
||
}
|
||
return fail("capture", err, false)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
var (
|
||
snapshotID string
|
||
sourceID string
|
||
writesetDigest string
|
||
counts map[string]int64
|
||
canary string
|
||
contentTouched bool
|
||
baseline Verification
|
||
drift Verification
|
||
restored Verification
|
||
)
|
||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||
start := time.Now()
|
||
// 窗口已关入口、已停写入者并证明排空:此时才敢把 UI 的入口状态写成 closed。
|
||
if opts.Governance != nil {
|
||
if err := opts.Governance.Ingress(ctx, jobID, scope, "closed",
|
||
"恢复验证已关闭目标入口并停止写入者"); err != nil {
|
||
return "", errors.New("登记入口关闭失败:" + err.Error())
|
||
}
|
||
}
|
||
current, err := target.Current(ctx)
|
||
if err != nil {
|
||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||
}
|
||
counts = current.Counts
|
||
report.Counts = counts
|
||
// 1) 写前镜像:真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||
// 它是恢复阶段实际回放的唯一内容来源。
|
||
backup, err := backups.Prepare(ctx, "", current)
|
||
if err != nil {
|
||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||
}
|
||
snapshotID = backup.SnapshotID
|
||
report.SnapshotID = snapshotID
|
||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||
}
|
||
// 2) 来源身份:DS01 合同要求"来源快照 ≠ 写前镜像",演练不能用同一份快照冒充
|
||
// 两个身份(真实 Restore 会在任何写之前拒绝)。这里再做一次**真实的**独立捕获,
|
||
// 只作为这次写入的来源身份;两次捕获都在同一维护切点内、由同一备份存储密封,
|
||
// 不伪造快照、也不放宽 Restore 的校验。
|
||
source, err := backups.Prepare(ctx, "", current)
|
||
if err != nil {
|
||
return "", errors.New("捕获来源身份失败:" + err.Error())
|
||
}
|
||
if source.SnapshotID == "" || source.SnapshotID == snapshotID {
|
||
return "", errors.New("来源身份与写前镜像相同:拒绝用一份快照冒充两个恢复身份")
|
||
}
|
||
sourceID = source.SnapshotID
|
||
report.SourceSnapshot = sourceID
|
||
// 预备份引用必须在任何写入之前落进同一个作业:恢复入口按它取写前镜像。
|
||
if opts.Ledger != nil {
|
||
if err := opts.Ledger.BackupPrepared(ctx, backup); err != nil {
|
||
return "", errors.New("预备份身份无法持久化,未对目标执行任何写入:" + err.Error())
|
||
}
|
||
}
|
||
progress.report("capture", "succeeded",
|
||
fmt.Sprintf("已捕获写前镜像 %s 与来源身份 %s(%d 字节,用时 %s)",
|
||
short(snapshotID), short(sourceID), backup.Bytes, time.Since(start).Round(time.Second)))
|
||
log.Printf("verify: snapshot captured preimage=%s source=%s bytes=%d in=%s",
|
||
snapshotID, sourceID, backup.Bytes, time.Since(start))
|
||
|
||
// 3) 基线校验:预备份必须与目标完全一致。
|
||
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
|
||
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||
baseline.Error = err.Error()
|
||
report.BaselineCheck = &baseline
|
||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||
}
|
||
baseline.Verified = true
|
||
report.BaselineCheck = &baseline
|
||
progress.report("verify", "succeeded", "预备份与目标逐类一致")
|
||
|
||
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
|
||
var random [16]byte
|
||
if _, err := rand.Read(random[:]); err != nil {
|
||
return "", err
|
||
}
|
||
canary = fmt.Sprintf("%x", random[:])
|
||
identityValue := target.Identity()
|
||
var exists bool
|
||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||
return "", err
|
||
}
|
||
if exists {
|
||
return "", errors.New("canary identity collision; no write attempted")
|
||
}
|
||
ledger, planErr := PlanWrite(jobID, scope, identityValue.TenantID, identityValue.KnowledgeBaseID, backup, source)
|
||
if planErr != nil {
|
||
return "", planErr
|
||
}
|
||
report.Canary = canary
|
||
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
|
||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||
func(set *libraryrelease.WriteSet) error {
|
||
// 计划必须在 Seal/落盘/登记之前定型:对象网关控制器按登记摘要加载,
|
||
// 这里补 V2 信封、目标 schema、来源身份与当前 owned attempt 绑定。
|
||
return target.PrepareDrillPlan(ctx, set)
|
||
},
|
||
func(ctx context.Context) error {
|
||
if err := window.MarkContentRisk(ctx); err != nil {
|
||
return err
|
||
}
|
||
contentTouched = true
|
||
return nil
|
||
}, func(ctx context.Context) error {
|
||
if opts.Ledger == nil {
|
||
return nil
|
||
}
|
||
// 写集合摘要必须在第一笔写入之前登记到同一个作业的预备份证据里:
|
||
// 恢复入口按 (scope, job, 写前镜像) 精确读它,读不到就保持维护。
|
||
return opts.Ledger.WriteSetPlanned(ctx, *ledgerClone(&ledger))
|
||
}, func(ctx context.Context) error {
|
||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||
return err
|
||
}); err != nil {
|
||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||
}
|
||
writesetDigest = ledger.Digest
|
||
report.WriteSetDigest = writesetDigest
|
||
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||
report.DriftCheck = &drift
|
||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||
} else {
|
||
drift.Error = verifyErr.Error()
|
||
}
|
||
report.DriftCheck = &drift
|
||
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
|
||
log.Printf("verify: drift detected as expected (%s)", drift.Error)
|
||
|
||
// 4) 按写前镜像恢复:四个阶段整段回放(与真实恢复同一批 Apply 原语),
|
||
// 而不是只删 canary。发布计划形态的 Restore 不能接受"只写过一行 canary"的
|
||
// 写集合(v2 执行计划合同),所以演练走限定的安全恢复入口;它复用同一份
|
||
// DS01 身份合同与整实例指纹校验。
|
||
progress.report("restore", "running", "按预备份恢复目标整库")
|
||
if err := target.RestoreDrill(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||
WriteSet: &ledger}); err != nil {
|
||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||
}
|
||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||
restored.Error = err.Error()
|
||
report.RestoreCheck = &restored
|
||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||
}
|
||
restored.Verified = true
|
||
report.RestoreCheck = &restored
|
||
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
|
||
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
|
||
return snapshotID, nil
|
||
}
|
||
|
||
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
|
||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||
func(error) bool { return !contentTouched || restored.Verified })
|
||
|
||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||
report.WindowState = window.State
|
||
// 服务真的回来、入口验证过(succeeded / 未写内容的安全失败)才释放治理占用;
|
||
// held 时保留行,页面才能显示真实入口状态并从同一 job 走恢复入口。
|
||
released := false
|
||
if opts.Governance != nil {
|
||
switch {
|
||
case window.State == "succeeded" || (window.State == "failed" && !contentTouched):
|
||
if err := opts.Governance.Ingress(ctx, jobID, scope, "open",
|
||
"维护窗口已按原运行状态恢复,入口已验证可用"); err != nil {
|
||
return fail("finalize", errors.New("恢复后登记入口状态失败:"+err.Error()), false)
|
||
}
|
||
if err := opts.Governance.Release(ctx, jobID, scope,
|
||
"恢复验证结束,服务与入口已按原状态恢复"); err != nil {
|
||
return fail("finalize", errors.New("恢复验证结束但维护占用释放失败:"+err.Error()), false)
|
||
}
|
||
released = true
|
||
default:
|
||
cause := "维护窗口保持关闭"
|
||
if windowErr != nil {
|
||
cause = windowErr.Error()
|
||
}
|
||
_ = opts.Governance.Ingress(ctx, jobID, scope, "closed", "目标内容状态未验证,入口保持关闭")
|
||
_ = opts.Governance.State(ctx, jobID, scope, "needs_restore",
|
||
"恢复验证失败且目标内容未验证:"+cause)
|
||
}
|
||
}
|
||
if windowErr != nil || window.State != "succeeded" {
|
||
cause := windowErr
|
||
if cause == nil {
|
||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||
}
|
||
held := window.State == "held"
|
||
if held && released {
|
||
held = false
|
||
}
|
||
return fail("finalize", cause, held)
|
||
}
|
||
report.Status = "passed"
|
||
report.FinishedAt = time.Now().UTC()
|
||
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
|
||
return report, nil
|
||
}
|
||
|
||
// PlanWrite 构造演练的写前镜像/来源身份与写集合。Run 与定向回归共用这一份:
|
||
// 只有来源身份 ≠ 写前镜像、且两者都是同一次维护切点里密封的真实捕获时,
|
||
// 恢复入口(同一份 DS01 身份合同)才接受这次演练。
|
||
func PlanWrite(jobID, scope string, tenantID uint64, knowledgeBaseID string,
|
||
preimage, source libraryrelease.BackupRef) (libraryrelease.WriteSet, error) {
|
||
if preimage.SnapshotID == "" || source.SnapshotID == "" {
|
||
return libraryrelease.WriteSet{}, errors.New("演练缺少写前镜像或来源身份快照")
|
||
}
|
||
if preimage.SnapshotID == source.SnapshotID {
|
||
return libraryrelease.WriteSet{}, errors.New("来源身份与写前镜像相同:拒绝用一份快照冒充两个恢复身份")
|
||
}
|
||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||
TenantID: tenantID, KnowledgeBaseID: knowledgeBaseID,
|
||
BackupSnapshotID: preimage.SnapshotID, TargetSnapshotID: source.SnapshotID,
|
||
CreatedAt: time.Now().UTC(), Universe: preimage.Universe}
|
||
if err := set.ValidateRestoreBinding(preimage.SnapshotID, tenantID, knowledgeBaseID); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
return set, nil
|
||
}
|
||
|
||
func short(id string) string {
|
||
if len(id) <= 12 {
|
||
return id
|
||
}
|
||
return id[:12]
|
||
}
|
||
|
||
// ledgerClone 只用于把台账交给登记方:Seal 会重排 Rows,调用方随后还要用同一份
|
||
// 内容去恢复,不能让登记方持有可变的同一指针。
|
||
func ledgerClone(set *libraryrelease.WriteSet) *libraryrelease.WriteSet {
|
||
copy := *set
|
||
copy.Rows = append([]libraryrelease.WriteSetRow(nil), set.Rows...)
|
||
return ©
|
||
}
|
||
|
||
// writeDrillCanary 是"写入前先留痕"的唯一实现:seal → 落盘 → 登记 → 标风险 → 写。
|
||
// 任何一步失败都不写目标;登记在标风险之前,恢复入口才拿得到这份写集合。
|
||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string,
|
||
prepare func(*libraryrelease.WriteSet) error,
|
||
mark func(context.Context) error, record func(context.Context) error, write func(context.Context) error) error {
|
||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||
if prepare != nil {
|
||
if err := prepare(ledger); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if err := ledger.Seal(); err != nil {
|
||
return err
|
||
}
|
||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||
return err
|
||
}
|
||
if record != nil {
|
||
if err := record(ctx); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if err := mark(ctx); err != nil {
|
||
return err
|
||
}
|
||
return write(ctx)
|
||
}
|
||
|
||
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
||
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
|
||
func RecoverWindows(config appconfig.Config) error {
|
||
manager, err := librarystack.OpenMaintenance(config)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer manager.Close()
|
||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
return coordinator.Recover(context.Background())
|
||
}
|