5e1482e349
build / build (push) Successful in 1m40s
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
189 lines
7.9 KiB
Go
189 lines
7.9 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryverify"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// libraryVerifyPayload 是"恢复验证"的队列载荷。
|
||
//
|
||
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
|
||
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
|
||
type libraryVerifyPayload struct {
|
||
LibraryJobID string `json:"library_job_id"`
|
||
Scope string `json:"scope"`
|
||
}
|
||
|
||
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
|
||
//
|
||
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
|
||
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
|
||
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
|
||
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryVerifyPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") {
|
||
return errors.New("invalid library verify payload")
|
||
}
|
||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
|
||
governance.LibraryJobVerify, true); admitErr != nil {
|
||
return admitErr
|
||
}
|
||
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
|
||
func(runCtx context.Context, writer *libraryAttemptWriter) error {
|
||
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
|
||
})
|
||
}
|
||
|
||
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
|
||
payload libraryVerifyPayload) error {
|
||
store := writer
|
||
now := time.Now().UTC()
|
||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||
operationID: writer.attempt.OperationID}
|
||
|
||
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
|
||
if jobErr != nil {
|
||
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
|
||
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
|
||
}
|
||
if existing.State == governance.LibraryJobSucceeded {
|
||
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
|
||
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
|
||
return nil
|
||
}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
|
||
"capture")
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
|
||
"", "", "", "", now)
|
||
|
||
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
|
||
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
|
||
|
||
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
|
||
evidence := governance.LibraryRestoreVerification{
|
||
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
|
||
Scope: payload.Scope,
|
||
JobID: payload.LibraryJobID,
|
||
Status: governance.LibraryVerificationFailed,
|
||
StartedAt: now,
|
||
}
|
||
if report != nil {
|
||
evidence.StartedAt = report.StartedAt
|
||
evidence.SnapshotID = report.SnapshotID
|
||
evidence.Report = verificationReportObject(report)
|
||
}
|
||
if runErr == nil && report != nil && report.Status == "passed" {
|
||
evidence.Status = governance.LibraryVerificationPassed
|
||
evidence.FinishedAt = report.FinishedAt
|
||
} else {
|
||
evidence.FinishedAt = time.Now().UTC()
|
||
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
|
||
evidence.FinishedAt = report.FinishedAt
|
||
}
|
||
switch {
|
||
case runErr != nil:
|
||
evidence.Failure = runErr.Error()
|
||
case report != nil && report.Status != "passed":
|
||
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
|
||
default:
|
||
evidence.Failure = "恢复验证未产生报告"
|
||
}
|
||
}
|
||
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
|
||
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
|
||
}
|
||
|
||
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
|
||
held := report != nil && report.Status == "maintenance_held"
|
||
message := "恢复验证未通过:" + evidence.Failure
|
||
if held {
|
||
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
|
||
}
|
||
recovery := "未对目标执行任何写入,无需恢复"
|
||
if report != nil && report.SnapshotID != "" {
|
||
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
|
||
}
|
||
snapshot := ""
|
||
if report != nil {
|
||
snapshot = report.SnapshotID
|
||
}
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||
message, recovery, snapshot, snapshot, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
finishedAt := evidence.FinishedAt
|
||
if finishedAt.IsZero() {
|
||
finishedAt = time.Now().UTC()
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded",
|
||
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
|
||
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
|
||
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
|
||
return nil
|
||
}
|
||
|
||
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
|
||
func verificationReportObject(report *libraryverify.Report) map[string]any {
|
||
if report == nil {
|
||
return nil
|
||
}
|
||
raw, err := json.Marshal(report)
|
||
if err != nil {
|
||
return map[string]any{"encode_error": err.Error()}
|
||
}
|
||
var object map[string]any
|
||
if err := json.Unmarshal(raw, &object); err != nil {
|
||
return map[string]any{"encode_error": err.Error()}
|
||
}
|
||
return object
|
||
}
|
||
|
||
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
|
||
//
|
||
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
|
||
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
|
||
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
|
||
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
}, config appconfig.Config, scope string) (bool, string) {
|
||
if store != nil {
|
||
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||
switch {
|
||
case err != nil:
|
||
if config.ProductionBackupRestoreVerified {
|
||
return true, ""
|
||
}
|
||
return false, "读取恢复验证证据失败:" + err.Error()
|
||
case found:
|
||
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
|
||
return true, ""
|
||
} else {
|
||
return false, reason
|
||
}
|
||
}
|
||
}
|
||
if config.ProductionBackupRestoreVerified {
|
||
return true, ""
|
||
}
|
||
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
|
||
}
|