package main import ( "context" "encoding/json" "errors" "fmt" "log" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryverify" postgresstore "knowledge-governance/internal/platform/postgres" ) // libraryVerifyPayload 是"恢复验证"的队列载荷。 // // 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方 // 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。 type libraryVerifyPayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` } // runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。 // // 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary // 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。 // 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。 func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryVerifyPayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || (payload.Scope != "internal" && payload.Scope != "external") { return errors.New("invalid library verify payload") } if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, governance.LibraryJobVerify, true); admitErr != nil { return admitErr } return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, func(runCtx context.Context, writer *libraryAttemptWriter) error { return runLibraryVerifyAttempt(runCtx, writer, config, payload) }) } func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config, payload libraryVerifyPayload) error { store := writer now := time.Now().UTC() recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, operationID: writer.attempt.OperationID} existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID) if jobErr != nil { // 作业行读不到就没有可登记的终态:直接失败,不落任何证据。 return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr) } if existing.State == governance.LibraryJobSucceeded { // 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。 recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行") return nil } if !config.ProductionReleaseRequestsEnabled { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行", "capture") } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", "", "", "", "", now) report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID, func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) }) // 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。 evidence := governance.LibraryRestoreVerification{ ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID, Scope: payload.Scope, JobID: payload.LibraryJobID, Status: governance.LibraryVerificationFailed, StartedAt: now, } if report != nil { evidence.StartedAt = report.StartedAt evidence.SnapshotID = report.SnapshotID evidence.Report = verificationReportObject(report) } if runErr == nil && report != nil && report.Status == "passed" { evidence.Status = governance.LibraryVerificationPassed evidence.FinishedAt = report.FinishedAt } else { evidence.FinishedAt = time.Now().UTC() if report != nil && report.FinishedAt.After(evidence.StartedAt) { evidence.FinishedAt = report.FinishedAt } switch { case runErr != nil: evidence.Failure = runErr.Error() case report != nil && report.Status != "passed": evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")" default: evidence.Failure = "恢复验证未产生报告" } } if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil { // 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。 return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize") } if runErr != nil || evidence.Status != governance.LibraryVerificationPassed { held := report != nil && report.Status == "maintenance_held" message := "恢复验证未通过:" + evidence.Failure if held { message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)" } recovery := "未对目标执行任何写入,无需恢复" if report != nil && report.SnapshotID != "" { recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获" } snapshot := "" if report != nil { snapshot = report.SnapshotID } recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, recovery, snapshot, snapshot, time.Now().UTC()) return errors.New(message) } finishedAt := evidence.FinishedAt if finishedAt.IsZero() { finishedAt = time.Now().UTC() } recorder.Stage(ctx, "finalize", "succeeded", "恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记") _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt) log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID) return nil } // verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。 func verificationReportObject(report *libraryverify.Report) map[string]any { if report == nil { return nil } raw, err := json.Marshal(report) if err != nil { return map[string]any{"encode_error": err.Error()} } var object map[string]any if err := json.Unmarshal(raw, &object); err != nil { return map[string]any{"encode_error": err.Error()} } return object } // libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。 // // 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶 // 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED // 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。 func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface { LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) }, config appconfig.Config, scope string) (bool, string) { if store != nil { evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope) switch { case err != nil: if config.ProductionBackupRestoreVerified { return true, "" } return false, "读取恢复验证证据失败:" + err.Error() case found: if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok { return true, "" } else { return false, reason } } } if config.ProductionBackupRestoreVerified { return true, "" } return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)" }