f9457fab52
build / build (push) Successful in 1m35s
B1/B2:演练在同一维护切点做两次真实捕获(写前镜像与来源身份), 写集合在写第一笔之前落盘并登记;恢复走限定范围的 RestoreDrill, 复用同一份 DS01 身份合同与 Apply 原语整段回放,不再让同快照必被拒绝。 worker 接治理维护生命周期(实例+scope 行、真实入口状态、成功才释放), held 保留行并由同 job 的 library.recover 收口(kind 过滤贯通 verify)。 B3:证据优先,failed/invalid 与读取错误即使 env=true 也拒绝。 B4/B5:GET 投影在飞作业,子面板回调父页面轮询;观测绑定 scope 与代次, 刷新失败明确过期,不再显示旧的"已验证"。 迁移 79 不自动 down;未部署、未调用业务目标。
263 lines
12 KiB
Go
263 lines
12 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
"knowledge-governance/internal/libraryverify"
|
||
"knowledge-governance/internal/pgdsn"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// libraryVerifyPayload 是"恢复验证"的队列载荷。
|
||
//
|
||
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
|
||
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
|
||
type libraryVerifyPayload struct {
|
||
LibraryJobID string `json:"library_job_id"`
|
||
Scope string `json:"scope"`
|
||
}
|
||
|
||
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
|
||
//
|
||
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
|
||
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
|
||
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
|
||
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryVerifyPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") {
|
||
return errors.New("invalid library verify payload")
|
||
}
|
||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
|
||
governance.LibraryJobVerify, true); admitErr != nil {
|
||
return admitErr
|
||
}
|
||
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
|
||
func(runCtx context.Context, writer *libraryAttemptWriter) error {
|
||
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
|
||
})
|
||
}
|
||
|
||
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
|
||
payload libraryVerifyPayload) error {
|
||
store := writer
|
||
now := time.Now().UTC()
|
||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||
operationID: writer.attempt.OperationID}
|
||
|
||
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
|
||
if jobErr != nil {
|
||
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
|
||
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
|
||
}
|
||
if existing.State == governance.LibraryJobSucceeded {
|
||
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
|
||
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
|
||
return nil
|
||
}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
|
||
"capture")
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
|
||
"", "", "", "", now)
|
||
|
||
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
|
||
libraryverify.Options{
|
||
Progress: func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) },
|
||
Governance: verifyGovernance{store: store, config: config},
|
||
Ledger: recorder,
|
||
})
|
||
|
||
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
|
||
evidence := governance.LibraryRestoreVerification{
|
||
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
|
||
Scope: payload.Scope,
|
||
JobID: payload.LibraryJobID,
|
||
Status: governance.LibraryVerificationFailed,
|
||
StartedAt: now,
|
||
}
|
||
if report != nil {
|
||
evidence.StartedAt = report.StartedAt
|
||
evidence.SnapshotID = report.SnapshotID
|
||
evidence.Report = verificationReportObject(report)
|
||
}
|
||
if runErr == nil && report != nil && report.Status == "passed" {
|
||
evidence.Status = governance.LibraryVerificationPassed
|
||
evidence.FinishedAt = report.FinishedAt
|
||
} else {
|
||
evidence.FinishedAt = time.Now().UTC()
|
||
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
|
||
evidence.FinishedAt = report.FinishedAt
|
||
}
|
||
switch {
|
||
case runErr != nil:
|
||
evidence.Failure = runErr.Error()
|
||
case report != nil && report.Status != "passed":
|
||
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
|
||
default:
|
||
evidence.Failure = "恢复验证未产生报告"
|
||
}
|
||
}
|
||
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
|
||
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
|
||
}
|
||
|
||
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
|
||
held := report != nil && report.Status == "maintenance_held"
|
||
message := "恢复验证未通过:" + evidence.Failure
|
||
if held {
|
||
message += "(目标仍在维护窗口,入口关闭、写入者已停)"
|
||
}
|
||
// 失败出口只写真实结果:报告里的 restore_check 与窗口终态说明目标此刻是
|
||
// 仍在维护,还是已经按原运行状态安全收尾。不能拿"有快照"冒充"已恢复并复核"。
|
||
recovery := "未对目标执行任何写入,无需恢复"
|
||
switch {
|
||
case held:
|
||
recovery = "目标仍在维护窗口:在同一作业上执行「按预备份恢复」(library.recover),恢复与逐类校验通过后才会开入口并释放占用"
|
||
case report != nil && report.WindowState != "":
|
||
recovery = "维护窗口已按原运行状态收尾(window=" + report.WindowState + "):目标内容未写入或已恢复并复核,无需再恢复"
|
||
}
|
||
snapshot := ""
|
||
if report != nil {
|
||
snapshot = report.SnapshotID
|
||
}
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||
message, recovery, snapshot, snapshot, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
finishedAt := evidence.FinishedAt
|
||
if finishedAt.IsZero() {
|
||
finishedAt = time.Now().UTC()
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded",
|
||
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
|
||
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
|
||
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
|
||
return nil
|
||
}
|
||
|
||
// verifyGovernance 把演练接进**既有**维护生命周期:实例行 + scope 行都由 worker 的
|
||
// 真实 attempt 写入,入口状态与状态文字来自实际执行。页面读的维护接口只认这些行,
|
||
// 所以只写磁盘 journal 的窗口在 UI 上等于不存在(审核 B2 的真实缺口)。
|
||
type verifyGovernance struct {
|
||
store *libraryAttemptWriter
|
||
config appconfig.Config
|
||
}
|
||
|
||
func (g verifyGovernance) targetIdentity(scope string) string {
|
||
return fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", scope,
|
||
pgdsn.Endpoint(g.config.LibraryProductionDatabaseURL), g.config.LibraryProductionIndex, g.config.LibraryProductionS3Bucket)
|
||
}
|
||
|
||
func (g verifyGovernance) Acquire(ctx context.Context, jobID, scope string, target libraryverify.TargetIdentity, originalServiceState string) error {
|
||
now := time.Now().UTC()
|
||
note := "实例级维护:关闭目标入口并停止全部写入者,恢复验证完成并复核后开放"
|
||
if _, err := g.store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
|
||
Instance: target.Instance, OwnerJobID: jobID, OwnerScope: scope,
|
||
TargetFingerprint: target.Fingerprint, Level: librarymaintenance.LevelInstance,
|
||
Services: target.Writers, IngressState: "unknown", State: "acquired",
|
||
OriginalServiceState: originalServiceState, Note: note,
|
||
}, now); err != nil {
|
||
return err
|
||
}
|
||
if _, err := g.store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||
Scope: scope, OwnerJobID: jobID, TargetIdentity: g.targetIdentity(scope),
|
||
Level: librarymaintenance.LevelInstance, TargetFingerprint: target.Fingerprint,
|
||
Services: target.Writers, IngressState: "unknown", State: "acquired",
|
||
OriginalServiceState: originalServiceState, Note: note,
|
||
}, now); err != nil {
|
||
// scope 行没拿到时目标还没被停用:释放实例占用,避免留下一个空窗口。
|
||
_ = g.store.ReleaseInstanceMaintenance(ctx, target.Instance, jobID, now)
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (g verifyGovernance) Ingress(ctx context.Context, jobID, scope, state, note string) error {
|
||
now := time.Now().UTC()
|
||
if err := g.store.SetLibraryMaintenanceIngress(ctx, scope, jobID, state, note, now); err != nil {
|
||
return err
|
||
}
|
||
return g.store.SetInstanceMaintenanceIngress(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
|
||
}
|
||
|
||
func (g verifyGovernance) State(ctx context.Context, jobID, scope, state, note string) error {
|
||
now := time.Now().UTC()
|
||
if err := g.store.UpdateLibraryMaintenanceState(ctx, scope, jobID, state, note, now); err != nil {
|
||
return err
|
||
}
|
||
return g.store.UpdateInstanceMaintenanceState(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
|
||
}
|
||
|
||
func (g verifyGovernance) Release(ctx context.Context, jobID, scope, note string) error {
|
||
now := time.Now().UTC()
|
||
if err := g.store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil {
|
||
return err
|
||
}
|
||
return g.store.ReleaseInstanceMaintenance(ctx, g.config.LibraryTargetInstance, jobID, now)
|
||
}
|
||
|
||
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
|
||
func verificationReportObject(report *libraryverify.Report) map[string]any {
|
||
if report == nil {
|
||
return nil
|
||
}
|
||
raw, err := json.Marshal(report)
|
||
if err != nil {
|
||
return map[string]any{"encode_error": err.Error()}
|
||
}
|
||
var object map[string]any
|
||
if err := json.Unmarshal(raw, &object); err != nil {
|
||
return map[string]any{"encode_error": err.Error()}
|
||
}
|
||
return object
|
||
}
|
||
|
||
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
|
||
//
|
||
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
|
||
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
|
||
// 仍作为显式覆盖保留(不删除旧部署能力),但只允许在**明确没有证据行**时兜底;
|
||
// 明确 failed/invalid 与读取错误一律拒绝(审核 B3:不能把"不知道"当"满足")。
|
||
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
}, config appconfig.Config, scope string) (bool, string) {
|
||
// 存储没有证据能力(nil/迁移前形状)等同于"没有证据行":历史 env 覆盖可兜底;
|
||
// 但只要读得出证据行,failed/invalid 与读取错误都优先拒绝。
|
||
missing := true
|
||
if store != nil {
|
||
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||
switch {
|
||
case err != nil:
|
||
return false, "读取恢复验证证据失败:" + err.Error()
|
||
case found:
|
||
missing = false
|
||
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
|
||
return true, ""
|
||
} else {
|
||
return false, reason
|
||
}
|
||
}
|
||
}
|
||
if missing && config.ProductionBackupRestoreVerified {
|
||
return true, ""
|
||
}
|
||
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
|
||
}
|