5e1482e349
build / build (push) Successful in 1m40s
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
267 lines
11 KiB
Go
267 lines
11 KiB
Go
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
|
||
//
|
||
// 维护隔离(关入口/停写入者/证明排空)
|
||
// → 捕获目标整库(真预备份,落归档)
|
||
// → 校验目标 == 预备份(基线一致)
|
||
// → 写入一个 canary(证明"校验确实能发现内容变化")
|
||
// → 按预备份恢复
|
||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||
//
|
||
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
|
||
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
|
||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
|
||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||
package libraryverify
|
||
|
||
import (
|
||
"context"
|
||
"crypto/rand"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarystack"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
type Verification struct {
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Verified bool `json:"verified"`
|
||
Error string `json:"error,omitempty"`
|
||
At time.Time `json:"at"`
|
||
}
|
||
|
||
type Report struct {
|
||
Status string `json:"status"` // passed | failed | maintenance_held
|
||
Scope string `json:"scope"`
|
||
Target string `json:"target"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Project string `json:"project"`
|
||
Counts map[string]int64 `json:"counts"`
|
||
BaselineCheck *Verification `json:"baseline_check"`
|
||
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||
RestoreCheck *Verification `json:"restore_check"`
|
||
Canary string `json:"canary"`
|
||
Failure string `json:"failure,omitempty"`
|
||
Stage string `json:"stage,omitempty"`
|
||
StartedAt time.Time `json:"started_at"`
|
||
FinishedAt time.Time `json:"finished_at"`
|
||
Until map[string]string `json:"until,omitempty"`
|
||
}
|
||
|
||
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
|
||
type Progress func(stage, state, detail string)
|
||
|
||
func (p Progress) report(stage, state, detail string) {
|
||
if p != nil {
|
||
p(stage, state, detail)
|
||
}
|
||
}
|
||
|
||
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
|
||
// 否则崩溃恢复时窗口与作业对不上。
|
||
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, progress Progress) (*Report, error) {
|
||
if scope != "internal" && scope != "external" {
|
||
return nil, errors.New("scope must be internal or external")
|
||
}
|
||
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||
fail := func(stage string, cause error, held bool) (*Report, error) {
|
||
report.Stage = stage
|
||
report.Failure = cause.Error()
|
||
report.FinishedAt = time.Now().UTC()
|
||
if held {
|
||
report.Status = "maintenance_held"
|
||
}
|
||
progress.report(stage, "failed", cause.Error())
|
||
return report, cause
|
||
}
|
||
|
||
progress.report("capture", "running", "打开目标栈与维护管理器")
|
||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||
if err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
defer target.Close()
|
||
manager, err := librarystack.OpenMaintenance(config)
|
||
if err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
defer manager.Close()
|
||
if err := manager.VerifyTopology(ctx); err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||
if err != nil {
|
||
return fail("capture", err, false)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
var (
|
||
snapshotID string
|
||
counts map[string]int64
|
||
canary string
|
||
contentTouched bool
|
||
baseline Verification
|
||
drift Verification
|
||
restored Verification
|
||
)
|
||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||
start := time.Now()
|
||
current, err := target.Current(ctx)
|
||
if err != nil {
|
||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||
}
|
||
counts = current.Counts
|
||
report.Counts = counts
|
||
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||
backup, err := backups.Prepare(ctx, "", current)
|
||
if err != nil {
|
||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||
}
|
||
snapshotID = backup.SnapshotID
|
||
report.SnapshotID = snapshotID
|
||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||
}
|
||
progress.report("capture", "succeeded",
|
||
fmt.Sprintf("已捕获目标整库快照 %s(%d 字节,用时 %s)", snapshotID, backup.Bytes, time.Since(start).Round(time.Second)))
|
||
log.Printf("verify: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
||
|
||
// 2) 基线校验:预备份必须与目标完全一致。
|
||
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
|
||
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||
baseline.Error = err.Error()
|
||
report.BaselineCheck = &baseline
|
||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||
}
|
||
baseline.Verified = true
|
||
report.BaselineCheck = &baseline
|
||
progress.report("verify", "succeeded", "预备份与目标逐类一致")
|
||
|
||
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
|
||
var random [16]byte
|
||
if _, err := rand.Read(random[:]); err != nil {
|
||
return "", err
|
||
}
|
||
canary = fmt.Sprintf("%x", random[:])
|
||
identityValue := target.Identity()
|
||
var exists bool
|
||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||
return "", err
|
||
}
|
||
if exists {
|
||
return "", errors.New("canary identity collision; no write attempted")
|
||
}
|
||
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
||
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
||
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
||
report.Canary = canary
|
||
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
|
||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||
func(ctx context.Context) error {
|
||
if err := window.MarkContentRisk(ctx); err != nil {
|
||
return err
|
||
}
|
||
contentTouched = true
|
||
return nil
|
||
}, func(ctx context.Context) error {
|
||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||
return err
|
||
}); err != nil {
|
||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||
}
|
||
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||
report.DriftCheck = &drift
|
||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||
} else {
|
||
drift.Error = verifyErr.Error()
|
||
}
|
||
report.DriftCheck = &drift
|
||
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
|
||
log.Printf("verify: drift detected as expected (%s)", drift.Error)
|
||
|
||
// 4) 按预备份恢复。恢复只删掉持久写集合里那个新 canary 身份。
|
||
progress.report("restore", "running", "按预备份恢复目标整库")
|
||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||
WriteSet: &ledger}); err != nil {
|
||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||
}
|
||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||
restored.Error = err.Error()
|
||
report.RestoreCheck = &restored
|
||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||
}
|
||
restored.Verified = true
|
||
report.RestoreCheck = &restored
|
||
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
|
||
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
|
||
return snapshotID, nil
|
||
}
|
||
|
||
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
|
||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||
func(error) bool { return !contentTouched || restored.Verified })
|
||
|
||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||
if windowErr != nil || window.State != "succeeded" {
|
||
cause := windowErr
|
||
if cause == nil {
|
||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||
}
|
||
if window.State == "held" {
|
||
return fail("finalize", cause, true)
|
||
}
|
||
return fail("finalize", cause, false)
|
||
}
|
||
report.Status = "passed"
|
||
report.FinishedAt = time.Now().UTC()
|
||
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
|
||
return report, nil
|
||
}
|
||
|
||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
|
||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||
if err := ledger.Seal(); err != nil {
|
||
return err
|
||
}
|
||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||
return err
|
||
}
|
||
if err := mark(ctx); err != nil {
|
||
return err
|
||
}
|
||
return write(ctx)
|
||
}
|
||
|
||
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
||
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
|
||
func RecoverWindows(config appconfig.Config) error {
|
||
manager, err := librarystack.OpenMaintenance(config)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer manager.Close()
|
||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
return coordinator.Recover(context.Background())
|
||
}
|