Files
knowledge-governance/internal/libraryverify/verify.go
T
姚章浩 5e1482e349
build / build (push) Successful in 1m40s
feat(library): 恢复验证成为可页面发起的真实作业
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
2026-11-03 10:14:08 +08:00

267 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
//
// 维护隔离(关入口/停写入者/证明排空)
// → 捕获目标整库(真预备份,落归档)
// → 校验目标 == 预备份(基线一致)
// → 写入一个 canary(证明"校验确实能发现内容变化")
// → 按预备份恢复
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
//
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
package libraryverify
import (
"context"
"crypto/rand"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarystack"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
type Verification struct {
SnapshotID string `json:"snapshot_id"`
Verified bool `json:"verified"`
Error string `json:"error,omitempty"`
At time.Time `json:"at"`
}
type Report struct {
Status string `json:"status"` // passed | failed | maintenance_held
Scope string `json:"scope"`
Target string `json:"target"`
SnapshotID string `json:"snapshot_id"`
Project string `json:"project"`
Counts map[string]int64 `json:"counts"`
BaselineCheck *Verification `json:"baseline_check"`
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
RestoreCheck *Verification `json:"restore_check"`
Canary string `json:"canary"`
Failure string `json:"failure,omitempty"`
Stage string `json:"stage,omitempty"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
Until map[string]string `json:"until,omitempty"`
}
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
type Progress func(stage, state, detail string)
func (p Progress) report(stage, state, detail string) {
if p != nil {
p(stage, state, detail)
}
}
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
// 否则崩溃恢复时窗口与作业对不上。
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, progress Progress) (*Report, error) {
if scope != "internal" && scope != "external" {
return nil, errors.New("scope must be internal or external")
}
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
fail := func(stage string, cause error, held bool) (*Report, error) {
report.Stage = stage
report.Failure = cause.Error()
report.FinishedAt = time.Now().UTC()
if held {
report.Status = "maintenance_held"
}
progress.report(stage, "failed", cause.Error())
return report, cause
}
progress.report("capture", "running", "打开目标栈与维护管理器")
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
if err != nil {
return fail("capture", err, false)
}
defer target.Close()
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return fail("capture", err, false)
}
defer manager.Close()
if err := manager.VerifyTopology(ctx); err != nil {
return fail("capture", err, false)
}
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return fail("capture", err, false)
}
defer control.Close()
defer coordinator.Close()
var (
snapshotID string
counts map[string]int64
canary string
contentTouched bool
baseline Verification
drift Verification
restored Verification
)
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
start := time.Now()
current, err := target.Current(ctx)
if err != nil {
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
}
counts = current.Counts
report.Counts = counts
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
backup, err := backups.Prepare(ctx, "", current)
if err != nil {
return "", errors.New("捕获目标整库失败:" + err.Error())
}
snapshotID = backup.SnapshotID
report.SnapshotID = snapshotID
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
return "", errors.New("记录预备份身份失败:" + err.Error())
}
progress.report("capture", "succeeded",
fmt.Sprintf("已捕获目标整库快照 %s(%d 字节,用时 %s)", snapshotID, backup.Bytes, time.Since(start).Round(time.Second)))
log.Printf("verify: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
// 2) 基线校验:预备份必须与目标完全一致。
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
baseline.Error = err.Error()
report.BaselineCheck = &baseline
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
}
baseline.Verified = true
report.BaselineCheck = &baseline
progress.report("verify", "succeeded", "预备份与目标逐类一致")
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
var random [16]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
canary = fmt.Sprintf("%x", random[:])
identityValue := target.Identity()
var exists bool
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
return "", err
}
if exists {
return "", errors.New("canary identity collision; no write attempted")
}
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
report.Canary = canary
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
func(ctx context.Context) error {
if err := window.MarkContentRisk(ctx); err != nil {
return err
}
contentTouched = true
return nil
}, func(ctx context.Context) error {
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
return err
}); err != nil {
return "", fmt.Errorf("canary preparation/write failed: %w", err)
}
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
report.DriftCheck = &drift
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
} else {
drift.Error = verifyErr.Error()
}
report.DriftCheck = &drift
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
log.Printf("verify: drift detected as expected (%s)", drift.Error)
// 4) 按预备份恢复。恢复只删掉持久写集合里那个新 canary 身份。
progress.report("restore", "running", "按预备份恢复目标整库")
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
WriteSet: &ledger}); err != nil {
return "", errors.New("按预备份恢复失败:" + err.Error())
}
// 5) 恢复后逐类复核:必须与预备份完全一致。
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
restored.Error = err.Error()
report.RestoreCheck = &restored
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
}
restored.Verified = true
report.RestoreCheck = &restored
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
return snapshotID, nil
}
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
window, windowErr := coordinator.RunWindow(ctx, jobID,
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
func(error) bool { return !contentTouched || restored.Verified })
report.Until = map[string]string{"job": jobID, "window_state": window.State}
if windowErr != nil || window.State != "succeeded" {
cause := windowErr
if cause == nil {
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
}
if window.State == "held" {
return fail("finalize", cause, true)
}
return fail("finalize", cause, false)
}
report.Status = "passed"
report.FinishedAt = time.Now().UTC()
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
return report, nil
}
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
if err := ledger.Seal(); err != nil {
return err
}
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
return err
}
if err := mark(ctx); err != nil {
return err
}
return write(ctx)
}
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
func RecoverWindows(config appconfig.Config) error {
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return err
}
defer manager.Close()
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return err
}
defer control.Close()
defer coordinator.Close()
return coordinator.Recover(context.Background())
}