knowledge-governance/cmd/librarydrill/main.go

// command librarydrill 在目标实例上做一次**真实**的恢复演练：
//
//	维护隔离（关入口/停写入者/证明排空）
//	  → 捕获目标整库（真预备份，落归档）
//	  → 校验目标 == 预备份（基线一致）
//	  → 写入一个 canary（证明"校验确实能发现内容变化"）
//	  → 按预备份恢复
//	  → 逐表/索引/资源/Wiki 再校验一次，必须完全一致
//	  → 关窗口（按原运行状态恢复服务、开入口、验证健康）
//
// 它只读/写**被测目标自己**，绝不触碰封存源；所有凭据来自进程环境
// （与 worker 同一份 LIBRARY_* 配置），不进命令行、不打印。
// 任一步失败都不伪装成功：窗口保持维护、报告写明失败阶段。
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"knowledge-governance/internal/appconfig"
	"knowledge-governance/internal/libraryfreeze"
	"knowledge-governance/internal/libraryrelease"
	"knowledge-governance/internal/librarystack"
)

type drillVerification struct {
	SnapshotID string    `json:"snapshot_id"`
	Verified   bool      `json:"verified"`
	Error      string    `json:"error,omitempty"`
	At         time.Time `json:"at"`
}

type drillReport struct {
	Status        string             `json:"status"` // passed | failed | maintenance_held
	Scope         string             `json:"scope"`
	Target        string             `json:"target"`
	SnapshotID    string             `json:"snapshot_id"`
	Project       string             `json:"project"`
	Counts        map[string]int64   `json:"counts"`
	BaselineCheck *drillVerification `json:"baseline_check"`
	DriftCheck    *drillVerification `json:"drift_check"` // 必须失败：证明确实能发现差异
	RestoreCheck  *drillVerification `json:"restore_check"`
	Canary        string             `json:"canary"`
	Failure       string             `json:"failure,omitempty"`
	Stage         string             `json:"stage,omitempty"`
	StartedAt     time.Time          `json:"started_at"`
	FinishedAt    time.Time          `json:"finished_at"`
	Until         map[string]string  `json:"until,omitempty"`
}

func main() {
	scope := flag.String("scope", "internal", "internal|external")
	reportPath := flag.String("report", "", "write the JSON report here (0600)")
	jobID := flag.String("job", "", "maintenance job id (default drill-<scope>-<timestamp>)")
	recoverOnly := flag.Bool("recover", false, "only recover non-terminal maintenance windows (restore services), then exit")
	flag.Parse()
	if *recoverOnly {
		if err := recoverWindows(); err != nil {
			log.Fatalf("recover failed: %v", err)
		}
		return
	}
	report, err := run(*scope, *jobID)
	if *reportPath != "" && report != nil {
		if writeErr := writeReport(*reportPath, report); writeErr != nil {
			log.Printf("drill report write failed: %v", writeErr)
		}
	}
	if err != nil {
		log.Fatalf("drill failed: %v", err)
	}
	log.Printf("drill passed: snapshot=%s", report.SnapshotID)
}

// recoverWindows 只做一件事：把非终态的维护窗口按原运行状态恢复（服务起来、入口验证）。
// 演练失败但还没写目标内容时，用它把目标从"停住"状态救回来；它不改治理记录，
// 治理记录由作业/接口那一侧负责。
func recoverWindows() error {
	config, err := appconfig.Load(os.Getenv)
	if err != nil {
		return fmt.Errorf("load config: %w", err)
	}
	manager, err := librarystack.OpenMaintenance(config)
	if err != nil {
		return err
	}
	defer manager.Close()
	coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
	if err != nil {
		return err
	}
	defer control.Close()
	defer coordinator.Close()
	if err := coordinator.Recover(context.Background()); err != nil {
		return err
	}
	log.Printf("maintenance windows recovered; target should be back to its original service state")
	return nil
}

func run(scope, jobID string) (*drillReport, error) {
	if scope != "internal" && scope != "external" {
		return nil, errors.New("scope must be internal or external")
	}
	config, err := appconfig.Load(os.Getenv)
	if err != nil {
		return nil, fmt.Errorf("load config: %w", err)
	}
	ctx := context.Background()
	report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
		Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
	fail := func(stage string, cause error, held bool) (*drillReport, error) {
		report.Stage = stage
		report.Failure = cause.Error()
		report.FinishedAt = time.Now().UTC()
		if held {
			report.Status = "maintenance_held"
		}
		return report, cause
	}

	target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
	if err != nil {
		return fail("open_target", err, false)
	}
	defer target.Close()
	manager, err := librarystack.OpenMaintenance(config)
	if err != nil {
		return fail("open_maintenance", err, false)
	}
	defer manager.Close()
	if err := manager.VerifyTopology(ctx); err != nil {
		return fail("topology", err, false)
	}
	coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
	if err != nil {
		return fail("open_coordinator", err, false)
	}
	defer control.Close()
	defer coordinator.Close()

	if jobID == "" {
		jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
	}
	var (
		snapshotID     string
		counts         map[string]int64
		canary         string
		contentTouched bool
		baseline       drillVerification
		drift          drillVerification
		restored       drillVerification
	)
	work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
		start := time.Now()
		current, err := target.Current(ctx)
		if err != nil {
			return "", errors.New("读取目标当前整库状态失败：" + err.Error())
		}
		counts = current.Counts
		report.Counts = counts
		// 1) 真预备份（落归档、内容寻址、导出后复核内容未变）。
		backup, err := backups.Prepare(ctx, "", current)
		if err != nil {
			return "", errors.New("捕获目标整库失败：" + err.Error())
		}
		snapshotID = backup.SnapshotID
		report.SnapshotID = snapshotID
		if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
			return "", errors.New("记录预备份身份失败：" + err.Error())
		}
		log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))

		// 2) 基线校验：预备份必须与目标完全一致。
		baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
		if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
			baseline.Error = err.Error()
			report.BaselineCheck = &baseline
			return "", errors.New("预备份与目标不一致（预备份本身不可信）：" + err.Error())
		}
		baseline.Verified = true
		report.BaselineCheck = &baseline

		// 3) canary：改一行真实内容，校验**必须**失败（否则"校验"是摆设）。
		canary = "drill-canary-" + time.Now().UTC().Format("20060102T150405")
		identity := target.Identity()
		tag, err := target.Pool().Exec(ctx, `
UPDATE knowledges SET title = title || ' [' || $3 || ']'
WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL
            ORDER BY id LIMIT 1)`, identity.TenantID, identity.KnowledgeBaseID, canary)
		if err != nil {
			return "", errors.New("写入 canary 失败：" + err.Error())
		}
		if tag.RowsAffected() == 0 {
			return "", errors.New("目标里没有可改写的文档，无法做漂移检测（目标不应为空）")
		}
		report.Canary = canary
		// 从这一笔开始目标内容被改写过：窗口必须记录内容风险，
		// 之后只有"恢复并校验通过"才允许重新开放（否则就是半写状态）。
		if err := window.MarkContentRisk(ctx); err != nil {
			return "", errors.New("记录内容风险失败：" + err.Error())
		}
		contentTouched = true
		drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
		if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
			drift.Error = "写入 canary 后校验仍然通过：校验没有真正比对内容"
			report.DriftCheck = &drift
			return "", errors.New("写入 canary 后校验仍然通过：校验没有真正比对内容，拒绝把这次演练当成通过")
		} else {
			drift.Error = verifyErr.Error()
		}
		report.DriftCheck = &drift
		log.Printf("drill: drift detected as expected (%s)", drift.Error)

		// 4) 按预备份恢复。
		if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID}); err != nil {
			return "", errors.New("按预备份恢复失败：" + err.Error())
		}
		// 5) 恢复后逐类复核：必须与预备份完全一致。
		restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
		if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
			restored.Error = err.Error()
			report.RestoreCheck = &restored
			return "", errors.New("恢复后内容仍与预备份不一致：" + err.Error())
		}
		restored.Verified = true
		report.RestoreCheck = &restored
		log.Printf("drill: restored and verified snapshot=%s", snapshotID)
		return snapshotID, nil
	}

	window, windowErr := coordinator.RunWindow(ctx, jobID,
		func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
		// 没写过目标内容 ⇒ 按原状态恢复永远安全；写过 canary ⇒ 只有恢复并校验通过才开放。
		func(error) bool { return !contentTouched || restored.Verified })

	report.Until = map[string]string{"job": jobID, "window_state": window.State}
	if windowErr != nil || window.State != "succeeded" {
		cause := windowErr
		if cause == nil {
			cause = fmt.Errorf("维护窗口状态 %s（未成功）", window.State)
		}
		if window.State == "held" {
			return fail("window", cause, true)
		}
		return fail("window", cause, false)
	}
	report.Status = "passed"
	report.FinishedAt = time.Now().UTC()
	return report, nil
}

func writeReport(path string, report *drillReport) error {
	raw, err := json.MarshalIndent(report, "", "  ")
	if err != nil {
		return err
	}
	if dir := filepath.Dir(path); dir != "" {
		if err := os.MkdirAll(dir, 0o700); err != nil {
			return err
		}
	}
	return os.WriteFile(path, append(raw, '\n'), 0o600)
}