根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
@@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
|
||||
gateReasons = append(gateReasons, reason)
|
||||
}
|
||||
if len(gateReasons) > 0 {
|
||||
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
|
||||
@@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore,
|
||||
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
|
||||
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
|
||||
queueType = libraryBaselineJobType
|
||||
case governance.LibraryJobVerify:
|
||||
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
|
||||
// 否则目标会一直停在维护里而作业显示失败。
|
||||
queueType = libraryVerifyJobType
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||||
"target_snapshot_id": job.TargetSnapshotID})
|
||||
|
||||
@@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos
|
||||
handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryBaseline(ctx, store, config, job)
|
||||
})
|
||||
// 恢复验证:真实"备份→canary→恢复→复核",通过后写入与当前目标绑定的可恢复证据。
|
||||
// 它是同步闸口的证据来源,所以与上面三类共用同一把执行锁与准入。
|
||||
handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryVerify(ctx, store, config, job)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil)
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType}
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType}
|
||||
if len(handlers) != len(want) {
|
||||
t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers)
|
||||
}
|
||||
@@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{})
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} {
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType} {
|
||||
err := handlers[jobType](context.Background(), jobs.Job{})
|
||||
if err == nil || !strings.Contains(err.Error(), "准入未通过") {
|
||||
t.Fatalf("handler %q error = %v, want admission refusal", jobType, err)
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/libraryverify"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// libraryVerifyPayload 是"恢复验证"的队列载荷。
|
||||
//
|
||||
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
|
||||
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
|
||||
type libraryVerifyPayload struct {
|
||||
LibraryJobID string `json:"library_job_id"`
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
|
||||
//
|
||||
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
|
||||
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
|
||||
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
|
||||
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||||
var payload libraryVerifyPayload
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
|
||||
(payload.Scope != "internal" && payload.Scope != "external") {
|
||||
return errors.New("invalid library verify payload")
|
||||
}
|
||||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
|
||||
governance.LibraryJobVerify, true); admitErr != nil {
|
||||
return admitErr
|
||||
}
|
||||
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
|
||||
func(runCtx context.Context, writer *libraryAttemptWriter) error {
|
||||
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
|
||||
})
|
||||
}
|
||||
|
||||
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
|
||||
payload libraryVerifyPayload) error {
|
||||
store := writer
|
||||
now := time.Now().UTC()
|
||||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||||
operationID: writer.attempt.OperationID}
|
||||
|
||||
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
|
||||
if jobErr != nil {
|
||||
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
|
||||
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
|
||||
}
|
||||
if existing.State == governance.LibraryJobSucceeded {
|
||||
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
|
||||
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
|
||||
return nil
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
|
||||
"capture")
|
||||
}
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
|
||||
"", "", "", "", now)
|
||||
|
||||
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
|
||||
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
|
||||
|
||||
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
|
||||
evidence := governance.LibraryRestoreVerification{
|
||||
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
|
||||
Scope: payload.Scope,
|
||||
JobID: payload.LibraryJobID,
|
||||
Status: governance.LibraryVerificationFailed,
|
||||
StartedAt: now,
|
||||
}
|
||||
if report != nil {
|
||||
evidence.StartedAt = report.StartedAt
|
||||
evidence.SnapshotID = report.SnapshotID
|
||||
evidence.Report = verificationReportObject(report)
|
||||
}
|
||||
if runErr == nil && report != nil && report.Status == "passed" {
|
||||
evidence.Status = governance.LibraryVerificationPassed
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
} else {
|
||||
evidence.FinishedAt = time.Now().UTC()
|
||||
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
}
|
||||
switch {
|
||||
case runErr != nil:
|
||||
evidence.Failure = runErr.Error()
|
||||
case report != nil && report.Status != "passed":
|
||||
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
|
||||
default:
|
||||
evidence.Failure = "恢复验证未产生报告"
|
||||
}
|
||||
}
|
||||
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
|
||||
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
|
||||
}
|
||||
|
||||
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
|
||||
held := report != nil && report.Status == "maintenance_held"
|
||||
message := "恢复验证未通过:" + evidence.Failure
|
||||
if held {
|
||||
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
|
||||
}
|
||||
recovery := "未对目标执行任何写入,无需恢复"
|
||||
if report != nil && report.SnapshotID != "" {
|
||||
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
|
||||
}
|
||||
snapshot := ""
|
||||
if report != nil {
|
||||
snapshot = report.SnapshotID
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "failed", message)
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||||
message, recovery, snapshot, snapshot, time.Now().UTC())
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
finishedAt := evidence.FinishedAt
|
||||
if finishedAt.IsZero() {
|
||||
finishedAt = time.Now().UTC()
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "succeeded",
|
||||
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
|
||||
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
|
||||
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
|
||||
func verificationReportObject(report *libraryverify.Report) map[string]any {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(raw, &object); err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
|
||||
//
|
||||
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
|
||||
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
|
||||
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
|
||||
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
}, config appconfig.Config, scope string) (bool, string) {
|
||||
if store != nil {
|
||||
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
switch {
|
||||
case err != nil:
|
||||
if config.ProductionBackupRestoreVerified {
|
||||
return true, ""
|
||||
}
|
||||
return false, "读取恢复验证证据失败:" + err.Error()
|
||||
case found:
|
||||
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
|
||||
return true, ""
|
||||
} else {
|
||||
return false, reason
|
||||
}
|
||||
}
|
||||
}
|
||||
if config.ProductionBackupRestoreVerified {
|
||||
return true, ""
|
||||
}
|
||||
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准:
|
||||
// 没有证据 ⇒ 拒绝并说清怎么补;有 passed 证据 ⇒ 放行;只有失败证据 ⇒ 拒绝并带失败原因。
|
||||
// 历史 env 覆盖只在证据缺失时兜底(页面不再写它)。
|
||||
type verifyEvidenceStore struct {
|
||||
evidence governance.LibraryRestoreVerification
|
||||
found bool
|
||||
}
|
||||
|
||||
func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
return s.evidence, s.found, nil
|
||||
}
|
||||
|
||||
func TestLibraryRestoreEvidenceSatisfied(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok {
|
||||
t.Fatalf("passed evidence must satisfy gate: %q", reason)
|
||||
}
|
||||
failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1",
|
||||
Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
missing := verifyEvidenceStore{}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复验证") {
|
||||
t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
// 历史部署的 env 覆盖:证据缺失时才生效,证据存在时不再起作用。
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok {
|
||||
t.Fatalf("legacy env override must still allow when no evidence: %q", reason)
|
||||
}
|
||||
if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok {
|
||||
t.Fatal("evidence must win over the legacy env override")
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,10 @@ const libraryRestoreJobType = "library.restore"
|
||||
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
|
||||
const libraryBaselineJobType = "library.baseline"
|
||||
|
||||
// libraryVerifyJobType 是"恢复验证"的队列类型:真实备份→canary→恢复→复核,
|
||||
// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
Reference in New Issue
Block a user