姚章浩
288 lines
11 KiB
Go
288 lines
11 KiB
Go
// command librarydrill 在目标实例上做一次**真实**的恢复演练：
|
|
//
|
|
// 维护隔离（关入口/停写入者/证明排空）
|
|
// → 捕获目标整库（真预备份，落归档）
|
|
// → 校验目标 == 预备份（基线一致）
|
|
// → 写入一个 canary（证明"校验确实能发现内容变化"）
|
|
// → 按预备份恢复
|
|
// → 逐表/索引/资源/Wiki 再校验一次，必须完全一致
|
|
// → 关窗口（按原运行状态恢复服务、开入口、验证健康）
|
|
//
|
|
// 它只读/写**被测目标自己**，绝不触碰封存源；所有凭据来自进程环境
|
|
// （与 worker 同一份 LIBRARY_* 配置），不进命令行、不打印。
|
|
// 任一步失败都不伪装成功：窗口保持维护、报告写明失败阶段。
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/appconfig"
|
|
"knowledge-governance/internal/libraryfreeze"
|
|
"knowledge-governance/internal/libraryrelease"
|
|
"knowledge-governance/internal/librarystack"
|
|
)
|
|
|
|
type drillVerification struct {
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Verified bool `json:"verified"`
|
|
Error string `json:"error,omitempty"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
type drillReport struct {
|
|
Status string `json:"status"` // passed | failed | maintenance_held
|
|
Scope string `json:"scope"`
|
|
Target string `json:"target"`
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Project string `json:"project"`
|
|
Counts map[string]int64 `json:"counts"`
|
|
BaselineCheck *drillVerification `json:"baseline_check"`
|
|
DriftCheck *drillVerification `json:"drift_check"` // 必须失败：证明确实能发现差异
|
|
RestoreCheck *drillVerification `json:"restore_check"`
|
|
Canary string `json:"canary"`
|
|
Failure string `json:"failure,omitempty"`
|
|
Stage string `json:"stage,omitempty"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinishedAt time.Time `json:"finished_at"`
|
|
Until map[string]string `json:"until,omitempty"`
|
|
}
|
|
|
|
func main() {
|
|
scope := flag.String("scope", "internal", "internal|external")
|
|
reportPath := flag.String("report", "", "write the JSON report here (0600)")
|
|
jobID := flag.String("job", "", "maintenance job id (default drill-<scope>-<timestamp>)")
|
|
recoverOnly := flag.Bool("recover", false, "only recover non-terminal maintenance windows (restore services), then exit")
|
|
flag.Parse()
|
|
if *recoverOnly {
|
|
if err := recoverWindows(); err != nil {
|
|
log.Fatalf("recover failed: %v", err)
|
|
}
|
|
return
|
|
}
|
|
report, err := run(*scope, *jobID)
|
|
if *reportPath != "" && report != nil {
|
|
if writeErr := writeReport(*reportPath, report); writeErr != nil {
|
|
log.Printf("drill report write failed: %v", writeErr)
|
|
}
|
|
}
|
|
if err != nil {
|
|
log.Fatalf("drill failed: %v", err)
|
|
}
|
|
log.Printf("drill passed: snapshot=%s", report.SnapshotID)
|
|
}
|
|
|
|
// recoverWindows 只做一件事：把非终态的维护窗口按原运行状态恢复（服务起来、入口验证）。
|
|
// 演练失败但还没写目标内容时，用它把目标从"停住"状态救回来；它不改治理记录，
|
|
// 治理记录由作业/接口那一侧负责。
|
|
func recoverWindows() error {
|
|
config, err := appconfig.Load(os.Getenv)
|
|
if err != nil {
|
|
return fmt.Errorf("load config: %w", err)
|
|
}
|
|
manager, err := librarystack.OpenMaintenance(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer manager.Close()
|
|
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer control.Close()
|
|
defer coordinator.Close()
|
|
if err := coordinator.Recover(context.Background()); err != nil {
|
|
return err
|
|
}
|
|
log.Printf("maintenance windows recovered; target should be back to its original service state")
|
|
return nil
|
|
}
|
|
|
|
func run(scope, jobID string) (*drillReport, error) {
|
|
if scope != "internal" && scope != "external" {
|
|
return nil, errors.New("scope must be internal or external")
|
|
}
|
|
config, err := appconfig.Load(os.Getenv)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load config: %w", err)
|
|
}
|
|
ctx := context.Background()
|
|
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
|
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
|
fail := func(stage string, cause error, held bool) (*drillReport, error) {
|
|
report.Stage = stage
|
|
report.Failure = cause.Error()
|
|
report.FinishedAt = time.Now().UTC()
|
|
if held {
|
|
report.Status = "maintenance_held"
|
|
}
|
|
return report, cause
|
|
}
|
|
|
|
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
|
if err != nil {
|
|
return fail("open_target", err, false)
|
|
}
|
|
defer target.Close()
|
|
manager, err := librarystack.OpenMaintenance(config)
|
|
if err != nil {
|
|
return fail("open_maintenance", err, false)
|
|
}
|
|
defer manager.Close()
|
|
if err := manager.VerifyTopology(ctx); err != nil {
|
|
return fail("topology", err, false)
|
|
}
|
|
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
|
if err != nil {
|
|
return fail("open_coordinator", err, false)
|
|
}
|
|
defer control.Close()
|
|
defer coordinator.Close()
|
|
|
|
if jobID == "" {
|
|
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
|
}
|
|
var (
|
|
snapshotID string
|
|
counts map[string]int64
|
|
canary string
|
|
contentTouched bool
|
|
baseline drillVerification
|
|
drift drillVerification
|
|
restored drillVerification
|
|
)
|
|
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
|
start := time.Now()
|
|
current, err := target.Current(ctx)
|
|
if err != nil {
|
|
return "", errors.New("读取目标当前整库状态失败：" + err.Error())
|
|
}
|
|
counts = current.Counts
|
|
report.Counts = counts
|
|
// 1) 真预备份（落归档、内容寻址、导出后复核内容未变）。
|
|
backup, err := backups.Prepare(ctx, "", current)
|
|
if err != nil {
|
|
return "", errors.New("捕获目标整库失败：" + err.Error())
|
|
}
|
|
snapshotID = backup.SnapshotID
|
|
report.SnapshotID = snapshotID
|
|
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
|
return "", errors.New("记录预备份身份失败：" + err.Error())
|
|
}
|
|
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
|
|
|
// 2) 基线校验：预备份必须与目标完全一致。
|
|
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
|
baseline.Error = err.Error()
|
|
report.BaselineCheck = &baseline
|
|
return "", errors.New("预备份与目标不一致（预备份本身不可信）：" + err.Error())
|
|
}
|
|
baseline.Verified = true
|
|
report.BaselineCheck = &baseline
|
|
|
|
// Canary is an isolated tag, including on an empty target. Never seed
|
|
// a document or change an existing row merely to make a drill possible.
|
|
var random [16]byte
|
|
if _, err := rand.Read(random[:]); err != nil {
|
|
return "", err
|
|
}
|
|
canary = fmt.Sprintf("%x", random[:])
|
|
identityValue := target.Identity()
|
|
var exists bool
|
|
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
|
return "", err
|
|
}
|
|
if exists {
|
|
return "", errors.New("canary identity collision; no write attempted")
|
|
}
|
|
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
|
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
|
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
|
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
|
report.Canary = canary
|
|
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
|
func(ctx context.Context) error {
|
|
if err := window.MarkContentRisk(ctx); err != nil {
|
|
return err
|
|
}
|
|
contentTouched = true
|
|
return nil
|
|
}, func(ctx context.Context) error {
|
|
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
|
return err
|
|
}); err != nil {
|
|
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
|
}
|
|
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
|
drift.Error = "写入 canary 后校验仍然通过：校验没有真正比对内容"
|
|
report.DriftCheck = &drift
|
|
return "", errors.New("写入 canary 后校验仍然通过：校验没有真正比对内容，拒绝把这次演练当成通过")
|
|
} else {
|
|
drift.Error = verifyErr.Error()
|
|
}
|
|
report.DriftCheck = &drift
|
|
log.Printf("drill: drift detected as expected (%s)", drift.Error)
|
|
|
|
// 4) 按预备份恢复。
|
|
// Restore removes only the exact new canary identity in the durable write set.
|
|
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
|
WriteSet: &ledger}); err != nil {
|
|
return "", errors.New("按预备份恢复失败：" + err.Error())
|
|
}
|
|
// 5) 恢复后逐类复核：必须与预备份完全一致。
|
|
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
|
restored.Error = err.Error()
|
|
report.RestoreCheck = &restored
|
|
return "", errors.New("恢复后内容仍与预备份不一致：" + err.Error())
|
|
}
|
|
restored.Verified = true
|
|
report.RestoreCheck = &restored
|
|
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
|
|
return snapshotID, nil
|
|
}
|
|
|
|
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
|
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
|
// 没写过目标内容 ⇒ 按原状态恢复永远安全；写过 canary ⇒ 只有恢复并校验通过才开放。
|
|
func(error) bool { return !contentTouched || restored.Verified })
|
|
|
|
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
|
if windowErr != nil || window.State != "succeeded" {
|
|
cause := windowErr
|
|
if cause == nil {
|
|
cause = fmt.Errorf("维护窗口状态 %s（未成功）", window.State)
|
|
}
|
|
if window.State == "held" {
|
|
return fail("window", cause, true)
|
|
}
|
|
return fail("window", cause, false)
|
|
}
|
|
report.Status = "passed"
|
|
report.FinishedAt = time.Now().UTC()
|
|
return report, nil
|
|
}
|
|
|
|
func writeReport(path string, report *drillReport) error {
|
|
raw, err := json.MarshalIndent(report, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dir := filepath.Dir(path); dir != "" {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o600)
|
|
}
|