e0768f2caa
build / build (push) Successful in 1m34s
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它 知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可 恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例 证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。 改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据" 算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份 版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对 资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。 resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。 整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。 本地验证(已跑): - make verify 15 项全 success(identity 因本地档案 skipped) - go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES + 真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/ 历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复; 跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净 - SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变 - go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、 接管不重新规划
287 lines
11 KiB
Go
287 lines
11 KiB
Go
// command librarydrill 在目标实例上做一次**真实**的恢复演练:
|
|
//
|
|
// 维护隔离(关入口/停写入者/证明排空)
|
|
// → 捕获目标整库(真预备份,落归档)
|
|
// → 校验目标 == 预备份(基线一致)
|
|
// → 写入一个 canary(证明"校验确实能发现内容变化")
|
|
// → 按预备份恢复
|
|
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
|
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
|
//
|
|
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境
|
|
// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。
|
|
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/appconfig"
|
|
"knowledge-governance/internal/libraryfreeze"
|
|
"knowledge-governance/internal/libraryrelease"
|
|
"knowledge-governance/internal/librarystack"
|
|
v071 "knowledge-governance/internal/releaseadapter/v071"
|
|
)
|
|
|
|
type drillVerification struct {
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Verified bool `json:"verified"`
|
|
Error string `json:"error,omitempty"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
type drillReport struct {
|
|
Status string `json:"status"` // passed | failed | maintenance_held
|
|
Scope string `json:"scope"`
|
|
Target string `json:"target"`
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Project string `json:"project"`
|
|
Counts map[string]int64 `json:"counts"`
|
|
BaselineCheck *drillVerification `json:"baseline_check"`
|
|
DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
|
RestoreCheck *drillVerification `json:"restore_check"`
|
|
Canary string `json:"canary"`
|
|
Failure string `json:"failure,omitempty"`
|
|
Stage string `json:"stage,omitempty"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinishedAt time.Time `json:"finished_at"`
|
|
Until map[string]string `json:"until,omitempty"`
|
|
}
|
|
|
|
func main() {
|
|
scope := flag.String("scope", "internal", "internal|external")
|
|
reportPath := flag.String("report", "", "write the JSON report here (0600)")
|
|
jobID := flag.String("job", "", "maintenance job id (default drill-<scope>-<timestamp>)")
|
|
recoverOnly := flag.Bool("recover", false, "only recover non-terminal maintenance windows (restore services), then exit")
|
|
flag.Parse()
|
|
if *recoverOnly {
|
|
if err := recoverWindows(); err != nil {
|
|
log.Fatalf("recover failed: %v", err)
|
|
}
|
|
return
|
|
}
|
|
report, err := run(*scope, *jobID)
|
|
if *reportPath != "" && report != nil {
|
|
if writeErr := writeReport(*reportPath, report); writeErr != nil {
|
|
log.Printf("drill report write failed: %v", writeErr)
|
|
}
|
|
}
|
|
if err != nil {
|
|
log.Fatalf("drill failed: %v", err)
|
|
}
|
|
log.Printf("drill passed: snapshot=%s", report.SnapshotID)
|
|
}
|
|
|
|
// recoverWindows 只做一件事:把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
|
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;它不改治理记录,
|
|
// 治理记录由作业/接口那一侧负责。
|
|
func recoverWindows() error {
|
|
config, err := appconfig.Load(os.Getenv)
|
|
if err != nil {
|
|
return fmt.Errorf("load config: %w", err)
|
|
}
|
|
manager, err := librarystack.OpenMaintenance(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer manager.Close()
|
|
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer control.Close()
|
|
defer coordinator.Close()
|
|
if err := coordinator.Recover(context.Background()); err != nil {
|
|
return err
|
|
}
|
|
log.Printf("maintenance windows recovered; target should be back to its original service state")
|
|
return nil
|
|
}
|
|
|
|
func run(scope, jobID string) (*drillReport, error) {
|
|
if scope != "internal" && scope != "external" {
|
|
return nil, errors.New("scope must be internal or external")
|
|
}
|
|
config, err := appconfig.Load(os.Getenv)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load config: %w", err)
|
|
}
|
|
ctx := context.Background()
|
|
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
|
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
|
fail := func(stage string, cause error, held bool) (*drillReport, error) {
|
|
report.Stage = stage
|
|
report.Failure = cause.Error()
|
|
report.FinishedAt = time.Now().UTC()
|
|
if held {
|
|
report.Status = "maintenance_held"
|
|
}
|
|
return report, cause
|
|
}
|
|
|
|
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
|
if err != nil {
|
|
return fail("open_target", err, false)
|
|
}
|
|
defer target.Close()
|
|
manager, err := librarystack.OpenMaintenance(config)
|
|
if err != nil {
|
|
return fail("open_maintenance", err, false)
|
|
}
|
|
defer manager.Close()
|
|
if err := manager.VerifyTopology(ctx); err != nil {
|
|
return fail("topology", err, false)
|
|
}
|
|
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
|
if err != nil {
|
|
return fail("open_coordinator", err, false)
|
|
}
|
|
defer control.Close()
|
|
defer coordinator.Close()
|
|
|
|
if jobID == "" {
|
|
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
|
}
|
|
var (
|
|
snapshotID string
|
|
counts map[string]int64
|
|
canary string
|
|
contentTouched bool
|
|
baseline drillVerification
|
|
drift drillVerification
|
|
restored drillVerification
|
|
)
|
|
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
|
start := time.Now()
|
|
current, err := target.Current(ctx)
|
|
if err != nil {
|
|
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
|
}
|
|
counts = current.Counts
|
|
report.Counts = counts
|
|
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
|
|
backup, err := backups.Prepare(ctx, "", current)
|
|
if err != nil {
|
|
return "", errors.New("捕获目标整库失败:" + err.Error())
|
|
}
|
|
snapshotID = backup.SnapshotID
|
|
report.SnapshotID = snapshotID
|
|
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
|
return "", errors.New("记录预备份身份失败:" + err.Error())
|
|
}
|
|
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
|
|
|
// 2) 基线校验:预备份必须与目标完全一致。
|
|
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
|
baseline.Error = err.Error()
|
|
report.BaselineCheck = &baseline
|
|
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
|
}
|
|
baseline.Verified = true
|
|
report.BaselineCheck = &baseline
|
|
|
|
// 3) canary:改一行真实内容,校验**必须**失败(否则"校验"是摆设)。
|
|
canary = "drill-canary-" + time.Now().UTC().Format("20060102T150405")
|
|
identity := target.Identity()
|
|
tag, err := target.Pool().Exec(ctx, `
|
|
UPDATE knowledges SET title = title || ' [' || $3 || ']'
|
|
WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL
|
|
ORDER BY id LIMIT 1)`, identity.TenantID, identity.KnowledgeBaseID, canary)
|
|
if err != nil {
|
|
return "", errors.New("写入 canary 失败:" + err.Error())
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return "", errors.New("目标里没有可改写的文档,无法做漂移检测(目标不应为空)")
|
|
}
|
|
report.Canary = canary
|
|
// 从这一笔开始目标内容被改写过:窗口必须记录内容风险,
|
|
// 之后只有"恢复并校验通过"才允许重新开放(否则就是半写状态)。
|
|
if err := window.MarkContentRisk(ctx); err != nil {
|
|
return "", errors.New("记录内容风险失败:" + err.Error())
|
|
}
|
|
contentTouched = true
|
|
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
|
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
|
report.DriftCheck = &drift
|
|
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
|
} else {
|
|
drift.Error = verifyErr.Error()
|
|
}
|
|
report.DriftCheck = &drift
|
|
log.Printf("drill: drift detected as expected (%s)", drift.Error)
|
|
|
|
// 4) 按预备份恢复。
|
|
// 恢复必须带着写集合记录。演练只改了一行既有内容(canary),没有新建行,
|
|
// 所以写集合为空;整实例证据用预备份时刻的那一份。
|
|
identityValue := target.Identity()
|
|
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
|
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
|
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
|
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
|
if err := ledger.Seal(); err != nil {
|
|
return "", errors.New("演练写集合生成失败:" + err.Error())
|
|
}
|
|
if _, err := v071.PersistWriteSet(config.LibraryArchiveRoot, &ledger); err != nil {
|
|
return "", errors.New("演练写集合落盘失败:" + err.Error())
|
|
}
|
|
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
|
WriteSet: &ledger}); err != nil {
|
|
return "", errors.New("按预备份恢复失败:" + err.Error())
|
|
}
|
|
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
|
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
|
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
|
restored.Error = err.Error()
|
|
report.RestoreCheck = &restored
|
|
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
|
}
|
|
restored.Verified = true
|
|
report.RestoreCheck = &restored
|
|
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
|
|
return snapshotID, nil
|
|
}
|
|
|
|
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
|
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
|
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
|
func(error) bool { return !contentTouched || restored.Verified })
|
|
|
|
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
|
if windowErr != nil || window.State != "succeeded" {
|
|
cause := windowErr
|
|
if cause == nil {
|
|
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
|
}
|
|
if window.State == "held" {
|
|
return fail("window", cause, true)
|
|
}
|
|
return fail("window", cause, false)
|
|
}
|
|
report.Status = "passed"
|
|
report.FinishedAt = time.Now().UTC()
|
|
return report, nil
|
|
}
|
|
|
|
func writeReport(path string, report *drillReport) error {
|
|
raw, err := json.MarshalIndent(report, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dir := filepath.Dir(path); dir != "" {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o600)
|
|
}
|