Files
knowledge-governance/cmd/worker/library_verify.go
T
姚章浩 5e1482e349
build / build (push) Successful in 1m40s
feat(library): 恢复验证成为可页面发起的真实作业
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
2026-11-03 10:14:08 +08:00

189 lines
7.9 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryverify"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryVerifyPayload 是"恢复验证"的队列载荷。
//
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
type libraryVerifyPayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
}
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
//
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryVerifyPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
(payload.Scope != "internal" && payload.Scope != "external") {
return errors.New("invalid library verify payload")
}
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
governance.LibraryJobVerify, true); admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
func(runCtx context.Context, writer *libraryAttemptWriter) error {
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
})
}
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
payload libraryVerifyPayload) error {
store := writer
now := time.Now().UTC()
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: writer.attempt.OperationID}
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
if jobErr != nil {
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
}
if existing.State == governance.LibraryJobSucceeded {
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
return nil
}
if !config.ProductionReleaseRequestsEnabled {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
"capture")
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
"", "", "", "", now)
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
evidence := governance.LibraryRestoreVerification{
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
Scope: payload.Scope,
JobID: payload.LibraryJobID,
Status: governance.LibraryVerificationFailed,
StartedAt: now,
}
if report != nil {
evidence.StartedAt = report.StartedAt
evidence.SnapshotID = report.SnapshotID
evidence.Report = verificationReportObject(report)
}
if runErr == nil && report != nil && report.Status == "passed" {
evidence.Status = governance.LibraryVerificationPassed
evidence.FinishedAt = report.FinishedAt
} else {
evidence.FinishedAt = time.Now().UTC()
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
evidence.FinishedAt = report.FinishedAt
}
switch {
case runErr != nil:
evidence.Failure = runErr.Error()
case report != nil && report.Status != "passed":
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
default:
evidence.Failure = "恢复验证未产生报告"
}
}
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
}
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
held := report != nil && report.Status == "maintenance_held"
message := "恢复验证未通过:" + evidence.Failure
if held {
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
}
recovery := "未对目标执行任何写入,无需恢复"
if report != nil && report.SnapshotID != "" {
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
}
snapshot := ""
if report != nil {
snapshot = report.SnapshotID
}
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, recovery, snapshot, snapshot, time.Now().UTC())
return errors.New(message)
}
finishedAt := evidence.FinishedAt
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
}
recorder.Stage(ctx, "finalize", "succeeded",
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
return nil
}
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
func verificationReportObject(report *libraryverify.Report) map[string]any {
if report == nil {
return nil
}
raw, err := json.Marshal(report)
if err != nil {
return map[string]any{"encode_error": err.Error()}
}
var object map[string]any
if err := json.Unmarshal(raw, &object); err != nil {
return map[string]any{"encode_error": err.Error()}
}
return object
}
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
//
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
}, config appconfig.Config, scope string) (bool, string) {
if store != nil {
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
switch {
case err != nil:
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "读取恢复验证证据失败:" + err.Error()
case found:
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
return true, ""
} else {
return false, reason
}
}
}
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
}