3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
164 lines
6.5 KiB
Go
164 lines
6.5 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||
type libraryRecoveryPayload struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
}
|
||
|
||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryRecoveryPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||
return errors.New("invalid library recovery payload")
|
||
}
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||
}
|
||
defer manager.Close()
|
||
identity := manager.Target()
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||
}
|
||
|
||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||
}
|
||
if lock == nil || lock.ReleasedAt != nil {
|
||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||
}
|
||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||
}
|
||
if lock.TargetFingerprint != identity.Fingerprint {
|
||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||
}
|
||
|
||
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
|
||
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
|
||
if err != nil || windowJob.SnapshotID == "" {
|
||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||
}
|
||
prebackup := windowJob.SnapshotID
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||
if wireErr != nil {
|
||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||
}
|
||
defer target.Close()
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
restorer := recoveryRestorer{target: target, expected: prebackup}
|
||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||
now := time.Now().UTC()
|
||
if recoverErr != nil {
|
||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||
return recoverErr
|
||
}
|
||
if result.State != "recovered" {
|
||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
return errors.New(note)
|
||
}
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
|
||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||
}
|
||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
|
||
return nil
|
||
}
|
||
|
||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||
type recoveryRestorer struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||
if backupID != r.expected {
|
||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||
}
|
||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
|
||
}
|
||
|
||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||
type recoveryVerifier struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||
if backupID != v.expected {
|
||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||
}
|
||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||
return "", err
|
||
}
|
||
state, err := v.target.Current(ctx)
|
||
if err != nil {
|
||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||
}
|
||
raw, err := json.Marshal(state)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
|
||
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
|
||
versions, err := store.LibraryVersions(ctx, scope, 50)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, item := range versions {
|
||
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
return raw
|
||
}
|
||
}
|
||
}
|
||
for _, item := range versions {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
return raw
|
||
}
|
||
}
|
||
return ""
|
||
}
|