根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
@@ -33,7 +33,8 @@ func (s *Store) createLibraryDispatch(ctx context.Context, job governance.Librar
|
||||
return created, true, nil
|
||||
}
|
||||
kind := "library." + string(created.Kind)
|
||||
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
|
||||
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore &&
|
||||
created.Kind != governance.LibraryJobBaseline && created.Kind != governance.LibraryJobVerify {
|
||||
return created, reused, errors.New("unsupported library dispatch kind")
|
||||
}
|
||||
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证证据:只由 worker 在真实演练结束后写入;这里只做落库与按当前目标读取。
|
||||
//
|
||||
// 读取永远带 target_id = 当前部署目标:换库/换索引/换桶之后旧证据自动失效,
|
||||
// 不需要任何人去"记得把开关关掉"。
|
||||
|
||||
// SaveLibraryRestoreVerification 幂等登记一次演练结果(同一 target+scope+job 只留一条)。
|
||||
func (s *Store) SaveLibraryRestoreVerification(ctx context.Context, verification governance.LibraryRestoreVerification) error {
|
||||
if verification.ID == "" || verification.JobID == "" {
|
||||
return fmt.Errorf("restore verification requires id and job id")
|
||||
}
|
||||
if verification.Scope != "internal" && verification.Scope != "external" {
|
||||
return fmt.Errorf("restore verification scope %q is invalid", verification.Scope)
|
||||
}
|
||||
if verification.Status != governance.LibraryVerificationPassed && verification.Status != governance.LibraryVerificationFailed {
|
||||
return fmt.Errorf("restore verification status %q is invalid", verification.Status)
|
||||
}
|
||||
report, err := json.Marshal(verification.Report)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode restore verification report: %w", err)
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
INSERT INTO library_restore_verifications (id, target_id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (target_id, scope, job_id) DO UPDATE
|
||||
SET status = EXCLUDED.status, snapshot_id = EXCLUDED.snapshot_id, failure = EXCLUDED.failure,
|
||||
report = EXCLUDED.report, started_at = EXCLUDED.started_at, finished_at = EXCLUDED.finished_at`,
|
||||
verification.ID, s.libraryTargetID, verification.Scope, verification.Status,
|
||||
verification.SnapshotID, verification.JobID, verification.Failure, report,
|
||||
verification.StartedAt.UTC(), verification.FinishedAt.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("save restore verification: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestLibraryRestoreVerification 按完成时间取当前目标该 scope 的最近一条证据。
|
||||
// 没有记录返回 (zero,false,nil):那不是错误,页面据此说"还没有验证过"。
|
||||
func (s *Store) LatestLibraryRestoreVerification(ctx context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
var (
|
||||
item governance.LibraryRestoreVerification
|
||||
report []byte
|
||||
)
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at
|
||||
FROM library_restore_verifications
|
||||
WHERE target_id=$1 AND scope=$2
|
||||
ORDER BY finished_at DESC, created_at DESC
|
||||
LIMIT 1`, s.libraryTargetID, scope).Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID,
|
||||
&item.JobID, &item.Failure, &report, &item.StartedAt, &item.FinishedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, fmt.Errorf("load restore verification: %w", err)
|
||||
}
|
||||
item.Report = decodeEvidenceObject(report)
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
// LibraryRestoreVerifications 是页面用的轻量清单(最近 N 条),按目标过滤。
|
||||
func (s *Store) LibraryRestoreVerifications(ctx context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, scope, status, snapshot_id, job_id, failure, started_at, finished_at
|
||||
FROM library_restore_verifications
|
||||
WHERE target_id=$1 AND scope=$2
|
||||
ORDER BY finished_at DESC, created_at DESC
|
||||
LIMIT $3`, s.libraryTargetID, scope, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list restore verifications: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]governance.LibraryRestoreVerification, 0, limit)
|
||||
for rows.Next() {
|
||||
var item governance.LibraryRestoreVerification
|
||||
if err := rows.Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, &item.JobID,
|
||||
&item.Failure, &item.StartedAt, &item.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
DROP INDEX IF EXISTS library_restore_verifications_latest;
|
||||
DROP TABLE IF EXISTS library_restore_verifications;
|
||||
|
||||
-- 先把 verify 作业清掉(它没有历史数据,只可能来自本版本),再恢复旧的 kind 约束。
|
||||
DELETE FROM library_jobs WHERE kind = 'verify';
|
||||
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
|
||||
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
|
||||
CHECK (kind IN ('seal','sync','restore','baseline'));
|
||||
|
||||
DROP INDEX IF EXISTS library_jobs_write_target;
|
||||
CREATE UNIQUE INDEX library_jobs_write_target
|
||||
ON library_jobs(scope)
|
||||
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline');
|
||||
@@ -0,0 +1,37 @@
|
||||
-- 恢复验证成为一种独立业务作业,并且它的证据只能由真实演练写入。
|
||||
--
|
||||
-- 为什么需要:闸门原来读 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个纯 env 开关,
|
||||
-- 谁把它写成 true 都能"证明"生产可恢复;页面也没有入口去真跑一次恢复。
|
||||
-- 现在 verified 只能来自 library_restore_verifications 里 passed 的真实记录。
|
||||
--
|
||||
-- verify 与 sync/restore/baseline 共用同一把写互斥:它同样要停写入者、开维护窗口、
|
||||
-- 在目标上写 canary 再恢复,不能和别的写作业并发。
|
||||
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
|
||||
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
|
||||
CHECK (kind IN ('seal','sync','restore','baseline','verify'));
|
||||
|
||||
DROP INDEX IF EXISTS library_jobs_write_target;
|
||||
CREATE UNIQUE INDEX library_jobs_write_target
|
||||
ON library_jobs(scope)
|
||||
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline','verify');
|
||||
|
||||
-- target_id 是 appconfig.LibraryTargetIdentity():换目标(库/索引/桶/实例)后旧证据不再适用,
|
||||
-- 这是"证据必须对得上当前目标"的落库形态。
|
||||
CREATE TABLE IF NOT EXISTS library_restore_verifications (
|
||||
id text PRIMARY KEY,
|
||||
target_id text NOT NULL CHECK (target_id <> ''),
|
||||
scope text NOT NULL CHECK (scope IN ('internal','external')),
|
||||
status text NOT NULL CHECK (status IN ('passed','failed')),
|
||||
snapshot_id text NOT NULL DEFAULT '',
|
||||
job_id text NOT NULL CHECK (job_id <> ''),
|
||||
failure text NOT NULL DEFAULT '',
|
||||
report jsonb,
|
||||
started_at timestamptz NOT NULL,
|
||||
finished_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
-- 同一作业只留一条;重复上报是幂等重放,不是新证据。
|
||||
CONSTRAINT library_restore_verifications_job UNIQUE (target_id, scope, job_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS library_restore_verifications_latest
|
||||
ON library_restore_verifications (target_id, scope, status, finished_at DESC);
|
||||
@@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 78 {
|
||||
t.Fatalf("migration count = %d, want 78", len(migrations))
|
||||
if len(migrations) != 79 {
|
||||
t.Fatalf("migration count = %d, want 79", len(migrations))
|
||||
}
|
||||
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
|
||||
// 证据按 target_id 落库(换目标旧证据自动失效)。
|
||||
if verification := migrations[78]; verification.Version != 79 ||
|
||||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
|
||||
!strings.Contains(verification.UpSQL, "'verify'") {
|
||||
t.Fatalf("migration 79 = %#v", verification)
|
||||
}
|
||||
// 78 是 DS01 恢复标记派发台账:唯一 key、单调状态与激活登记;只动治理库。
|
||||
if dispatch := migrations[77]; dispatch.Version != 78 ||
|
||||
|
||||
Reference in New Issue
Block a user