feat(library): 恢复验证成为可页面发起的真实作业
build / build (push) Successful in 1m40s

根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
姚章浩
2026-11-03 10:14:08 +08:00
parent 4421dd38ce
commit 5e1482e349
47 changed files with 2502 additions and 353 deletions
@@ -33,7 +33,8 @@ func (s *Store) createLibraryDispatch(ctx context.Context, job governance.Librar
return created, true, nil
}
kind := "library." + string(created.Kind)
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore &&
created.Kind != governance.LibraryJobBaseline && created.Kind != governance.LibraryJobVerify {
return created, reused, errors.New("unsupported library dispatch kind")
}
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
@@ -0,0 +1,98 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// 恢复验证证据:只由 worker 在真实演练结束后写入;这里只做落库与按当前目标读取。
//
// 读取永远带 target_id = 当前部署目标:换库/换索引/换桶之后旧证据自动失效,
// 不需要任何人去"记得把开关关掉"。
// SaveLibraryRestoreVerification 幂等登记一次演练结果(同一 target+scope+job 只留一条)。
func (s *Store) SaveLibraryRestoreVerification(ctx context.Context, verification governance.LibraryRestoreVerification) error {
if verification.ID == "" || verification.JobID == "" {
return fmt.Errorf("restore verification requires id and job id")
}
if verification.Scope != "internal" && verification.Scope != "external" {
return fmt.Errorf("restore verification scope %q is invalid", verification.Scope)
}
if verification.Status != governance.LibraryVerificationPassed && verification.Status != governance.LibraryVerificationFailed {
return fmt.Errorf("restore verification status %q is invalid", verification.Status)
}
report, err := json.Marshal(verification.Report)
if err != nil {
return fmt.Errorf("encode restore verification report: %w", err)
}
_, err = s.db.Exec(ctx, `
INSERT INTO library_restore_verifications (id, target_id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (target_id, scope, job_id) DO UPDATE
SET status = EXCLUDED.status, snapshot_id = EXCLUDED.snapshot_id, failure = EXCLUDED.failure,
report = EXCLUDED.report, started_at = EXCLUDED.started_at, finished_at = EXCLUDED.finished_at`,
verification.ID, s.libraryTargetID, verification.Scope, verification.Status,
verification.SnapshotID, verification.JobID, verification.Failure, report,
verification.StartedAt.UTC(), verification.FinishedAt.UTC())
if err != nil {
return fmt.Errorf("save restore verification: %w", err)
}
return nil
}
// LatestLibraryRestoreVerification 按完成时间取当前目标该 scope 的最近一条证据。
// 没有记录返回 (zero,false,nil):那不是错误,页面据此说"还没有验证过"。
func (s *Store) LatestLibraryRestoreVerification(ctx context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
var (
item governance.LibraryRestoreVerification
report []byte
)
err := s.db.QueryRow(ctx, `
SELECT id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at
FROM library_restore_verifications
WHERE target_id=$1 AND scope=$2
ORDER BY finished_at DESC, created_at DESC
LIMIT 1`, s.libraryTargetID, scope).Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID,
&item.JobID, &item.Failure, &report, &item.StartedAt, &item.FinishedAt)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryRestoreVerification{}, false, nil
}
return governance.LibraryRestoreVerification{}, false, fmt.Errorf("load restore verification: %w", err)
}
item.Report = decodeEvidenceObject(report)
return item, true, nil
}
// LibraryRestoreVerifications 是页面用的轻量清单(最近 N 条),按目标过滤。
func (s *Store) LibraryRestoreVerifications(ctx context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
if limit <= 0 {
limit = 10
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, status, snapshot_id, job_id, failure, started_at, finished_at
FROM library_restore_verifications
WHERE target_id=$1 AND scope=$2
ORDER BY finished_at DESC, created_at DESC
LIMIT $3`, s.libraryTargetID, scope, limit)
if err != nil {
return nil, fmt.Errorf("list restore verifications: %w", err)
}
defer rows.Close()
items := make([]governance.LibraryRestoreVerification, 0, limit)
for rows.Next() {
var item governance.LibraryRestoreVerification
if err := rows.Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, &item.JobID,
&item.Failure, &item.StartedAt, &item.FinishedAt); err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
@@ -0,0 +1,13 @@
DROP INDEX IF EXISTS library_restore_verifications_latest;
DROP TABLE IF EXISTS library_restore_verifications;
-- 先把 verify 作业清掉(它没有历史数据,只可能来自本版本),再恢复旧的 kind 约束。
DELETE FROM library_jobs WHERE kind = 'verify';
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
CHECK (kind IN ('seal','sync','restore','baseline'));
DROP INDEX IF EXISTS library_jobs_write_target;
CREATE UNIQUE INDEX library_jobs_write_target
ON library_jobs(scope)
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline');
@@ -0,0 +1,37 @@
-- 恢复验证成为一种独立业务作业,并且它的证据只能由真实演练写入。
--
-- 为什么需要:闸门原来读 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个纯 env 开关,
-- 谁把它写成 true 都能"证明"生产可恢复;页面也没有入口去真跑一次恢复。
-- 现在 verified 只能来自 library_restore_verifications 里 passed 的真实记录。
--
-- verify 与 sync/restore/baseline 共用同一把写互斥:它同样要停写入者、开维护窗口、
-- 在目标上写 canary 再恢复,不能和别的写作业并发。
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
CHECK (kind IN ('seal','sync','restore','baseline','verify'));
DROP INDEX IF EXISTS library_jobs_write_target;
CREATE UNIQUE INDEX library_jobs_write_target
ON library_jobs(scope)
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline','verify');
-- target_id 是 appconfig.LibraryTargetIdentity():换目标(库/索引/桶/实例)后旧证据不再适用,
-- 这是"证据必须对得上当前目标"的落库形态。
CREATE TABLE IF NOT EXISTS library_restore_verifications (
id text PRIMARY KEY,
target_id text NOT NULL CHECK (target_id <> ''),
scope text NOT NULL CHECK (scope IN ('internal','external')),
status text NOT NULL CHECK (status IN ('passed','failed')),
snapshot_id text NOT NULL DEFAULT '',
job_id text NOT NULL CHECK (job_id <> ''),
failure text NOT NULL DEFAULT '',
report jsonb,
started_at timestamptz NOT NULL,
finished_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
-- 同一作业只留一条;重复上报是幂等重放,不是新证据。
CONSTRAINT library_restore_verifications_job UNIQUE (target_id, scope, job_id)
);
CREATE INDEX IF NOT EXISTS library_restore_verifications_latest
ON library_restore_verifications (target_id, scope, status, finished_at DESC);
@@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 78 {
t.Fatalf("migration count = %d, want 78", len(migrations))
if len(migrations) != 79 {
t.Fatalf("migration count = %d, want 79", len(migrations))
}
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
// 证据按 target_id 落库(换目标旧证据自动失效)。
if verification := migrations[78]; verification.Version != 79 ||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
!strings.Contains(verification.UpSQL, "'verify'") {
t.Fatalf("migration 79 = %#v", verification)
}
// 78 是 DS01 恢复标记派发台账:唯一 key、单调状态与激活登记;只动治理库。
if dispatch := migrations[77]; dispatch.Version != 78 ||