Files
knowledge-governance/cmd/worker/library_verify.go
T
姚章浩 f9457fab52
build / build (push) Successful in 1m35s
fix(library): 打通恢复验证的正常恢复与页面联动
B1/B2:演练在同一维护切点做两次真实捕获(写前镜像与来源身份),
写集合在写第一笔之前落盘并登记;恢复走限定范围的 RestoreDrill,
复用同一份 DS01 身份合同与 Apply 原语整段回放,不再让同快照必被拒绝。
worker 接治理维护生命周期(实例+scope 行、真实入口状态、成功才释放),
held 保留行并由同 job 的 library.recover 收口(kind 过滤贯通 verify)。
B3:证据优先,failed/invalid 与读取错误即使 env=true 也拒绝。
B4/B5:GET 投影在飞作业,子面板回调父页面轮询;观测绑定 scope 与代次,
刷新失败明确过期,不再显示旧的"已验证"。
迁移 79 不自动 down;未部署、未调用业务目标。
2026-11-03 11:14:27 +08:00

263 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryverify"
"knowledge-governance/internal/pgdsn"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryVerifyPayload 是"恢复验证"的队列载荷。
//
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
type libraryVerifyPayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
}
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
//
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryVerifyPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
(payload.Scope != "internal" && payload.Scope != "external") {
return errors.New("invalid library verify payload")
}
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
governance.LibraryJobVerify, true); admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
func(runCtx context.Context, writer *libraryAttemptWriter) error {
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
})
}
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
payload libraryVerifyPayload) error {
store := writer
now := time.Now().UTC()
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: writer.attempt.OperationID}
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
if jobErr != nil {
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
}
if existing.State == governance.LibraryJobSucceeded {
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
return nil
}
if !config.ProductionReleaseRequestsEnabled {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
"capture")
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
"", "", "", "", now)
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
libraryverify.Options{
Progress: func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) },
Governance: verifyGovernance{store: store, config: config},
Ledger: recorder,
})
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
evidence := governance.LibraryRestoreVerification{
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
Scope: payload.Scope,
JobID: payload.LibraryJobID,
Status: governance.LibraryVerificationFailed,
StartedAt: now,
}
if report != nil {
evidence.StartedAt = report.StartedAt
evidence.SnapshotID = report.SnapshotID
evidence.Report = verificationReportObject(report)
}
if runErr == nil && report != nil && report.Status == "passed" {
evidence.Status = governance.LibraryVerificationPassed
evidence.FinishedAt = report.FinishedAt
} else {
evidence.FinishedAt = time.Now().UTC()
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
evidence.FinishedAt = report.FinishedAt
}
switch {
case runErr != nil:
evidence.Failure = runErr.Error()
case report != nil && report.Status != "passed":
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
default:
evidence.Failure = "恢复验证未产生报告"
}
}
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
}
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
held := report != nil && report.Status == "maintenance_held"
message := "恢复验证未通过:" + evidence.Failure
if held {
message += "(目标仍在维护窗口,入口关闭、写入者已停)"
}
// 失败出口只写真实结果:报告里的 restore_check 与窗口终态说明目标此刻是
// 仍在维护,还是已经按原运行状态安全收尾。不能拿"有快照"冒充"已恢复并复核"。
recovery := "未对目标执行任何写入,无需恢复"
switch {
case held:
recovery = "目标仍在维护窗口:在同一作业上执行「按预备份恢复」(library.recover),恢复与逐类校验通过后才会开入口并释放占用"
case report != nil && report.WindowState != "":
recovery = "维护窗口已按原运行状态收尾(window=" + report.WindowState + "):目标内容未写入或已恢复并复核,无需再恢复"
}
snapshot := ""
if report != nil {
snapshot = report.SnapshotID
}
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, recovery, snapshot, snapshot, time.Now().UTC())
return errors.New(message)
}
finishedAt := evidence.FinishedAt
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
}
recorder.Stage(ctx, "finalize", "succeeded",
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
return nil
}
// verifyGovernance 把演练接进**既有**维护生命周期:实例行 + scope 行都由 worker 的
// 真实 attempt 写入,入口状态与状态文字来自实际执行。页面读的维护接口只认这些行,
// 所以只写磁盘 journal 的窗口在 UI 上等于不存在(审核 B2 的真实缺口)。
type verifyGovernance struct {
store *libraryAttemptWriter
config appconfig.Config
}
func (g verifyGovernance) targetIdentity(scope string) string {
return fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", scope,
pgdsn.Endpoint(g.config.LibraryProductionDatabaseURL), g.config.LibraryProductionIndex, g.config.LibraryProductionS3Bucket)
}
func (g verifyGovernance) Acquire(ctx context.Context, jobID, scope string, target libraryverify.TargetIdentity, originalServiceState string) error {
now := time.Now().UTC()
note := "实例级维护:关闭目标入口并停止全部写入者,恢复验证完成并复核后开放"
if _, err := g.store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: target.Instance, OwnerJobID: jobID, OwnerScope: scope,
TargetFingerprint: target.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: target.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalServiceState, Note: note,
}, now); err != nil {
return err
}
if _, err := g.store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: scope, OwnerJobID: jobID, TargetIdentity: g.targetIdentity(scope),
Level: librarymaintenance.LevelInstance, TargetFingerprint: target.Fingerprint,
Services: target.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalServiceState, Note: note,
}, now); err != nil {
// scope 行没拿到时目标还没被停用:释放实例占用,避免留下一个空窗口。
_ = g.store.ReleaseInstanceMaintenance(ctx, target.Instance, jobID, now)
return err
}
return nil
}
func (g verifyGovernance) Ingress(ctx context.Context, jobID, scope, state, note string) error {
now := time.Now().UTC()
if err := g.store.SetLibraryMaintenanceIngress(ctx, scope, jobID, state, note, now); err != nil {
return err
}
return g.store.SetInstanceMaintenanceIngress(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
}
func (g verifyGovernance) State(ctx context.Context, jobID, scope, state, note string) error {
now := time.Now().UTC()
if err := g.store.UpdateLibraryMaintenanceState(ctx, scope, jobID, state, note, now); err != nil {
return err
}
return g.store.UpdateInstanceMaintenanceState(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
}
func (g verifyGovernance) Release(ctx context.Context, jobID, scope, note string) error {
now := time.Now().UTC()
if err := g.store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil {
return err
}
return g.store.ReleaseInstanceMaintenance(ctx, g.config.LibraryTargetInstance, jobID, now)
}
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
func verificationReportObject(report *libraryverify.Report) map[string]any {
if report == nil {
return nil
}
raw, err := json.Marshal(report)
if err != nil {
return map[string]any{"encode_error": err.Error()}
}
var object map[string]any
if err := json.Unmarshal(raw, &object); err != nil {
return map[string]any{"encode_error": err.Error()}
}
return object
}
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
//
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
// 仍作为显式覆盖保留(不删除旧部署能力),但只允许在**明确没有证据行**时兜底;
// 明确 failed/invalid 与读取错误一律拒绝(审核 B3:不能把"不知道"当"满足")。
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
}, config appconfig.Config, scope string) (bool, string) {
// 存储没有证据能力(nil/迁移前形状)等同于"没有证据行":历史 env 覆盖可兜底;
// 但只要读得出证据行,failed/invalid 与读取错误都优先拒绝。
missing := true
if store != nil {
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
switch {
case err != nil:
return false, "读取恢复验证证据失败:" + err.Error()
case found:
missing = false
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
return true, ""
} else {
return false, reason
}
}
}
if missing && config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
}