feat(library): 恢复验证成为可页面发起的真实作业
build / build (push) Successful in 1m40s

根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
姚章浩
2026-11-03 10:14:08 +08:00
parent 4421dd38ce
commit 5e1482e349
47 changed files with 2502 additions and 353 deletions
+2
View File
@@ -232,6 +232,8 @@ func run(ctx context.Context, getenv func(string) string) error {
LibraryProductionInternalKB: config.LibraryProductionInternalKB,
LibraryProductionExternalKB: config.LibraryProductionExternalKB,
LibraryProductionTenantID: config.LibraryProductionTenantID,
LibraryProductionIndex: config.LibraryProductionIndex,
LibraryProductionS3Bucket: config.LibraryProductionS3Bucket,
LibraryRetentionVersions: config.LibraryRetentionVersions,
LibraryRetentionDays: config.LibraryRetentionDays,
BackupHost: backuphost.New(backuphost.Config{
-21
View File
@@ -1,21 +0,0 @@
package main
import (
"context"
"knowledge-governance/internal/libraryrelease"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
if err := ledger.Seal(); err != nil {
return err
}
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
return err
}
if err := mark(ctx); err != nil {
return err
}
return write(ctx)
}
-60
View File
@@ -1,60 +0,0 @@
package main
import (
"context"
"errors"
"knowledge-governance/internal/libraryrelease"
"os"
"path/filepath"
"testing"
"time"
)
func TestCanaryPersistsRecoveryEvidenceBeforeWrite(t *testing.T) {
root := t.TempDir()
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
marked := false
written := false
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", func(context.Context) error { marked = true; return nil }, func(context.Context) error {
if !marked {
t.Fatal("write preceded risk marker")
}
if len(ledger.CreatedIDs("knowledge_tags")) != 1 {
t.Fatal("canary absent from write set")
}
matches, e := filepath.Glob(filepath.Join(root, "writesets", "*.json"))
if e != nil || len(matches) != 1 {
t.Fatalf("durable ledger missing: %v %v", matches, e)
}
if _, e := os.Stat(matches[0]); e != nil {
t.Fatal(e)
}
written = true
return nil
})
if err != nil || !written {
t.Fatalf("err=%v written=%v", err, written)
}
}
func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) {
root := filepath.Join(t.TempDir(), "blocked")
if err := os.WriteFile(root, []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
marked, written := false, false
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", func(context.Context) error { marked = true; return nil }, func(context.Context) error { written = true; return nil })
if err == nil || marked || written {
t.Fatalf("err=%v marked=%v written=%v", err, marked, written)
}
}
func TestCanaryNeverWritesWhenRiskMarkerFails(t *testing.T) {
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
written := false
err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", func(context.Context) error { return errors.New("journal unavailable") }, func(context.Context) error { written = true; return nil })
if err == nil || written {
t.Fatalf("err=%v written=%v", err, written)
}
}
+12 -203
View File
@@ -1,4 +1,4 @@
// command librarydrill 在目标实例上做一次**真实**的恢复演练:
// command librarydrill 是恢复演练的**命令行入口**,实现在 internal/libraryverify:
//
// 维护隔离(关入口/停写入者/证明排空)
// → 捕获目标整库(真预备份,落归档)
@@ -8,16 +8,14 @@
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
//
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境
// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
// 为什么只剩薄封装:页面要能点一次"恢复验证",worker 必须在进程内跑同一份实现。
// 两份实现意味着"CLI 通过、页面失败"或反过来,演练结论就不可信了。
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境。
package main
import (
"context"
"crypto/rand"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
@@ -26,35 +24,12 @@ import (
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarystack"
"knowledge-governance/internal/libraryverify"
)
type drillVerification struct {
SnapshotID string `json:"snapshot_id"`
Verified bool `json:"verified"`
Error string `json:"error,omitempty"`
At time.Time `json:"at"`
}
type drillReport struct {
Status string `json:"status"` // passed | failed | maintenance_held
Scope string `json:"scope"`
Target string `json:"target"`
SnapshotID string `json:"snapshot_id"`
Project string `json:"project"`
Counts map[string]int64 `json:"counts"`
BaselineCheck *drillVerification `json:"baseline_check"`
DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异
RestoreCheck *drillVerification `json:"restore_check"`
Canary string `json:"canary"`
Failure string `json:"failure,omitempty"`
Stage string `json:"stage,omitempty"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
Until map[string]string `json:"until,omitempty"`
}
// 报告类型直接复用共享实现:命令行与页面/worker 的证据字段完全一致。
type drillVerification = libraryverify.Verification
type drillReport = libraryverify.Report
func main() {
scope := flag.String("scope", "internal", "internal|external")
@@ -88,18 +63,7 @@ func recoverWindows() error {
if err != nil {
return fmt.Errorf("load config: %w", err)
}
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return err
}
defer manager.Close()
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return err
}
defer control.Close()
defer coordinator.Close()
if err := coordinator.Recover(context.Background()); err != nil {
if err := libraryverify.RecoverWindows(config); err != nil {
return err
}
log.Printf("maintenance windows recovered; target should be back to its original service state")
@@ -107,170 +71,15 @@ func recoverWindows() error {
}
func run(scope, jobID string) (*drillReport, error) {
if scope != "internal" && scope != "external" {
return nil, errors.New("scope must be internal or external")
}
config, err := appconfig.Load(os.Getenv)
if err != nil {
return nil, fmt.Errorf("load config: %w", err)
}
ctx := context.Background()
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
fail := func(stage string, cause error, held bool) (*drillReport, error) {
report.Stage = stage
report.Failure = cause.Error()
report.FinishedAt = time.Now().UTC()
if held {
report.Status = "maintenance_held"
}
return report, cause
}
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
if err != nil {
return fail("open_target", err, false)
}
defer target.Close()
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return fail("open_maintenance", err, false)
}
defer manager.Close()
if err := manager.VerifyTopology(ctx); err != nil {
return fail("topology", err, false)
}
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return fail("open_coordinator", err, false)
}
defer control.Close()
defer coordinator.Close()
if jobID == "" {
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
jobID = fmt.Sprintf("drill-%s-%d", scope, time.Now().UTC().UnixNano())
}
var (
snapshotID string
counts map[string]int64
canary string
contentTouched bool
baseline drillVerification
drift drillVerification
restored drillVerification
)
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
start := time.Now()
current, err := target.Current(ctx)
if err != nil {
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
}
counts = current.Counts
report.Counts = counts
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
backup, err := backups.Prepare(ctx, "", current)
if err != nil {
return "", errors.New("捕获目标整库失败:" + err.Error())
}
snapshotID = backup.SnapshotID
report.SnapshotID = snapshotID
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
return "", errors.New("记录预备份身份失败:" + err.Error())
}
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
// 2) 基线校验:预备份必须与目标完全一致。
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
baseline.Error = err.Error()
report.BaselineCheck = &baseline
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
}
baseline.Verified = true
report.BaselineCheck = &baseline
// Canary is an isolated tag, including on an empty target. Never seed
// a document or change an existing row merely to make a drill possible.
var random [16]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
canary = fmt.Sprintf("%x", random[:])
identityValue := target.Identity()
var exists bool
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
return "", err
}
if exists {
return "", errors.New("canary identity collision; no write attempted")
}
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
report.Canary = canary
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
func(ctx context.Context) error {
if err := window.MarkContentRisk(ctx); err != nil {
return err
}
contentTouched = true
return nil
}, func(ctx context.Context) error {
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
return err
}); err != nil {
return "", fmt.Errorf("canary preparation/write failed: %w", err)
}
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
report.DriftCheck = &drift
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
} else {
drift.Error = verifyErr.Error()
}
report.DriftCheck = &drift
log.Printf("drill: drift detected as expected (%s)", drift.Error)
// 4) 按预备份恢复。
// Restore removes only the exact new canary identity in the durable write set.
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
WriteSet: &ledger}); err != nil {
return "", errors.New("按预备份恢复失败:" + err.Error())
}
// 5) 恢复后逐类复核:必须与预备份完全一致。
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
restored.Error = err.Error()
report.RestoreCheck = &restored
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
}
restored.Verified = true
report.RestoreCheck = &restored
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
return snapshotID, nil
}
window, windowErr := coordinator.RunWindow(ctx, jobID,
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
func(error) bool { return !contentTouched || restored.Verified })
report.Until = map[string]string{"job": jobID, "window_state": window.State}
if windowErr != nil || window.State != "succeeded" {
cause := windowErr
if cause == nil {
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
}
if window.State == "held" {
return fail("window", cause, true)
}
return fail("window", cause, false)
}
report.Status = "passed"
report.FinishedAt = time.Now().UTC()
return report, nil
return libraryverify.Run(context.TODO(), config, scope, jobID,
func(stage, state, detail string) { log.Printf("stage=%s state=%s %s", stage, state, detail) })
}
func writeReport(path string, report *drillReport) error {
+6 -2
View File
@@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
gateReasons = append(gateReasons, reason)
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
@@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore,
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
queueType = libraryBaselineJobType
case governance.LibraryJobVerify:
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
// 否则目标会一直停在维护里而作业显示失败。
queueType = libraryVerifyJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
+5
View File
@@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos
handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
return runLibraryBaseline(ctx, store, config, job)
})
// 恢复验证:真实"备份→canary→恢复→复核",通过后写入与当前目标绑定的可恢复证据。
// 它是同步闸口的证据来源,所以与上面三类共用同一把执行锁与准入。
handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
return runLibraryVerify(ctx, store, config, job)
})
}
+4 -2
View File
@@ -15,7 +15,8 @@ import (
func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
handlers := map[string]jobs.Handler{}
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil)
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType}
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
libraryBaselineJobType, libraryVerifyJobType}
if len(handlers) != len(want) {
t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers)
}
@@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) {
handlers := map[string]jobs.Handler{}
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{})
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} {
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
libraryBaselineJobType, libraryVerifyJobType} {
err := handlers[jobType](context.Background(), jobs.Job{})
if err == nil || !strings.Contains(err.Error(), "准入未通过") {
t.Fatalf("handler %q error = %v, want admission refusal", jobType, err)
+188
View File
@@ -0,0 +1,188 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryverify"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryVerifyPayload 是"恢复验证"的队列载荷。
//
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
type libraryVerifyPayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
}
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
//
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryVerifyPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
(payload.Scope != "internal" && payload.Scope != "external") {
return errors.New("invalid library verify payload")
}
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
governance.LibraryJobVerify, true); admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
func(runCtx context.Context, writer *libraryAttemptWriter) error {
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
})
}
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
payload libraryVerifyPayload) error {
store := writer
now := time.Now().UTC()
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: writer.attempt.OperationID}
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
if jobErr != nil {
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
}
if existing.State == governance.LibraryJobSucceeded {
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
return nil
}
if !config.ProductionReleaseRequestsEnabled {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
"capture")
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
"", "", "", "", now)
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
evidence := governance.LibraryRestoreVerification{
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
Scope: payload.Scope,
JobID: payload.LibraryJobID,
Status: governance.LibraryVerificationFailed,
StartedAt: now,
}
if report != nil {
evidence.StartedAt = report.StartedAt
evidence.SnapshotID = report.SnapshotID
evidence.Report = verificationReportObject(report)
}
if runErr == nil && report != nil && report.Status == "passed" {
evidence.Status = governance.LibraryVerificationPassed
evidence.FinishedAt = report.FinishedAt
} else {
evidence.FinishedAt = time.Now().UTC()
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
evidence.FinishedAt = report.FinishedAt
}
switch {
case runErr != nil:
evidence.Failure = runErr.Error()
case report != nil && report.Status != "passed":
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
default:
evidence.Failure = "恢复验证未产生报告"
}
}
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
}
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
held := report != nil && report.Status == "maintenance_held"
message := "恢复验证未通过:" + evidence.Failure
if held {
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
}
recovery := "未对目标执行任何写入,无需恢复"
if report != nil && report.SnapshotID != "" {
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
}
snapshot := ""
if report != nil {
snapshot = report.SnapshotID
}
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, recovery, snapshot, snapshot, time.Now().UTC())
return errors.New(message)
}
finishedAt := evidence.FinishedAt
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
}
recorder.Stage(ctx, "finalize", "succeeded",
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
return nil
}
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
func verificationReportObject(report *libraryverify.Report) map[string]any {
if report == nil {
return nil
}
raw, err := json.Marshal(report)
if err != nil {
return map[string]any{"encode_error": err.Error()}
}
var object map[string]any
if err := json.Unmarshal(raw, &object); err != nil {
return map[string]any{"encode_error": err.Error()}
}
return object
}
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
//
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
}, config appconfig.Config, scope string) (bool, string) {
if store != nil {
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
switch {
case err != nil:
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "读取恢复验证证据失败:" + err.Error()
case found:
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
return true, ""
} else {
return false, reason
}
}
}
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
}
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"context"
"strings"
"testing"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
)
// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准:
// 没有证据 ⇒ 拒绝并说清怎么补;有 passed 证据 ⇒ 放行;只有失败证据 ⇒ 拒绝并带失败原因。
// 历史 env 覆盖只在证据缺失时兜底(页面不再写它)。
type verifyEvidenceStore struct {
evidence governance.LibraryRestoreVerification
found bool
}
func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
return s.evidence, s.found, nil
}
func TestLibraryRestoreEvidenceSatisfied(t *testing.T) {
now := time.Now().UTC()
passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok {
t.Fatalf("passed evidence must satisfy gate: %q", reason)
}
failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1",
Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok ||
!strings.Contains(reason, "恢复后内容仍与预备份不一致") {
t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason)
}
missing := verifyEvidenceStore{}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok ||
!strings.Contains(reason, "恢复验证") {
t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason)
}
// 历史部署的 env 覆盖:证据缺失时才生效,证据存在时不再起作用。
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing,
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok {
t.Fatalf("legacy env override must still allow when no evidence: %q", reason)
}
if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed,
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok {
t.Fatal("evidence must win over the legacy env override")
}
}
+4
View File
@@ -110,6 +110,10 @@ const libraryRestoreJobType = "library.restore"
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
const libraryBaselineJobType = "library.baseline"
// libraryVerifyJobType 是"恢复验证"的队列类型:真实备份→canary→恢复→复核,
// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。
const libraryVerifyJobType = "library.verify"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()