package main import ( "context" "encoding/json" "errors" "fmt" "log" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryverify" "knowledge-governance/internal/pgdsn" postgresstore "knowledge-governance/internal/platform/postgres" ) // libraryVerifyPayload 是"恢复验证"的队列载荷。 // // 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方 // 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。 type libraryVerifyPayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` } // runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。 // // 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary // 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。 // 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。 func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryVerifyPayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || (payload.Scope != "internal" && payload.Scope != "external") { return errors.New("invalid library verify payload") } if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, governance.LibraryJobVerify, true); admitErr != nil { return admitErr } return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, func(runCtx context.Context, writer *libraryAttemptWriter) error { return runLibraryVerifyAttempt(runCtx, writer, config, payload) }) } func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config, payload libraryVerifyPayload) error { store := writer now := time.Now().UTC() recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, operationID: writer.attempt.OperationID} existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID) if jobErr != nil { // 作业行读不到就没有可登记的终态:直接失败,不落任何证据。 return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr) } if existing.State == governance.LibraryJobSucceeded { // 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。 recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行") return nil } if !config.ProductionReleaseRequestsEnabled { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行", "capture") } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", "", "", "", "", now) report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID, libraryverify.Options{ Progress: func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) }, Governance: verifyGovernance{store: store, config: config}, Ledger: recorder, }) // 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。 evidence := governance.LibraryRestoreVerification{ ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID, Scope: payload.Scope, JobID: payload.LibraryJobID, Status: governance.LibraryVerificationFailed, StartedAt: now, } if report != nil { evidence.StartedAt = report.StartedAt evidence.SnapshotID = report.SnapshotID evidence.Report = verificationReportObject(report) } if runErr == nil && report != nil && report.Status == "passed" { evidence.Status = governance.LibraryVerificationPassed evidence.FinishedAt = report.FinishedAt } else { evidence.FinishedAt = time.Now().UTC() if report != nil && report.FinishedAt.After(evidence.StartedAt) { evidence.FinishedAt = report.FinishedAt } switch { case runErr != nil: evidence.Failure = runErr.Error() case report != nil && report.Status != "passed": evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")" default: evidence.Failure = "恢复验证未产生报告" } } if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil { // 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。 return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize") } if runErr != nil || evidence.Status != governance.LibraryVerificationPassed { held := report != nil && report.Status == "maintenance_held" message := "恢复验证未通过:" + evidence.Failure if held { message += "(目标仍在维护窗口,入口关闭、写入者已停)" } // 失败出口只写真实结果:报告里的 restore_check 与窗口终态说明目标此刻是 // 仍在维护,还是已经按原运行状态安全收尾。不能拿"有快照"冒充"已恢复并复核"。 recovery := "未对目标执行任何写入,无需恢复" switch { case held: recovery = "目标仍在维护窗口:在同一作业上执行「按预备份恢复」(library.recover),恢复与逐类校验通过后才会开入口并释放占用" case report != nil && report.WindowState != "": recovery = "维护窗口已按原运行状态收尾(window=" + report.WindowState + "):目标内容未写入或已恢复并复核,无需再恢复" } snapshot := "" if report != nil { snapshot = report.SnapshotID } recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, recovery, snapshot, snapshot, time.Now().UTC()) return errors.New(message) } finishedAt := evidence.FinishedAt if finishedAt.IsZero() { finishedAt = time.Now().UTC() } recorder.Stage(ctx, "finalize", "succeeded", "恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记") _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt) log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID) return nil } // verifyGovernance 把演练接进**既有**维护生命周期:实例行 + scope 行都由 worker 的 // 真实 attempt 写入,入口状态与状态文字来自实际执行。页面读的维护接口只认这些行, // 所以只写磁盘 journal 的窗口在 UI 上等于不存在(审核 B2 的真实缺口)。 type verifyGovernance struct { store *libraryAttemptWriter config appconfig.Config } func (g verifyGovernance) targetIdentity(scope string) string { return fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", scope, pgdsn.Endpoint(g.config.LibraryProductionDatabaseURL), g.config.LibraryProductionIndex, g.config.LibraryProductionS3Bucket) } func (g verifyGovernance) Acquire(ctx context.Context, jobID, scope string, target libraryverify.TargetIdentity, originalServiceState string) error { now := time.Now().UTC() note := "实例级维护:关闭目标入口并停止全部写入者,恢复验证完成并复核后开放" if _, err := g.store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ Instance: target.Instance, OwnerJobID: jobID, OwnerScope: scope, TargetFingerprint: target.Fingerprint, Level: librarymaintenance.LevelInstance, Services: target.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalServiceState, Note: note, }, now); err != nil { return err } if _, err := g.store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ Scope: scope, OwnerJobID: jobID, TargetIdentity: g.targetIdentity(scope), Level: librarymaintenance.LevelInstance, TargetFingerprint: target.Fingerprint, Services: target.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalServiceState, Note: note, }, now); err != nil { // scope 行没拿到时目标还没被停用:释放实例占用,避免留下一个空窗口。 _ = g.store.ReleaseInstanceMaintenance(ctx, target.Instance, jobID, now) return err } return nil } func (g verifyGovernance) Ingress(ctx context.Context, jobID, scope, state, note string) error { now := time.Now().UTC() if err := g.store.SetLibraryMaintenanceIngress(ctx, scope, jobID, state, note, now); err != nil { return err } return g.store.SetInstanceMaintenanceIngress(ctx, g.config.LibraryTargetInstance, jobID, state, note, now) } func (g verifyGovernance) State(ctx context.Context, jobID, scope, state, note string) error { now := time.Now().UTC() if err := g.store.UpdateLibraryMaintenanceState(ctx, scope, jobID, state, note, now); err != nil { return err } return g.store.UpdateInstanceMaintenanceState(ctx, g.config.LibraryTargetInstance, jobID, state, note, now) } func (g verifyGovernance) Release(ctx context.Context, jobID, scope, note string) error { now := time.Now().UTC() if err := g.store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil { return err } return g.store.ReleaseInstanceMaintenance(ctx, g.config.LibraryTargetInstance, jobID, now) } // verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。 func verificationReportObject(report *libraryverify.Report) map[string]any { if report == nil { return nil } raw, err := json.Marshal(report) if err != nil { return map[string]any{"encode_error": err.Error()} } var object map[string]any if err := json.Unmarshal(raw, &object); err != nil { return map[string]any{"encode_error": err.Error()} } return object } // libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。 // // 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶 // 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED // 仍作为显式覆盖保留(不删除旧部署能力),但只允许在**明确没有证据行**时兜底; // 明确 failed/invalid 与读取错误一律拒绝(审核 B3:不能把"不知道"当"满足")。 func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface { LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) }, config appconfig.Config, scope string) (bool, string) { // 存储没有证据能力(nil/迁移前形状)等同于"没有证据行":历史 env 覆盖可兜底; // 但只要读得出证据行,failed/invalid 与读取错误都优先拒绝。 missing := true if store != nil { evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope) switch { case err != nil: return false, "读取恢复验证证据失败:" + err.Error() case found: missing = false if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok { return true, "" } else { return false, reason } } } if missing && config.ProductionBackupRestoreVerified { return true, "" } return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)" }