Files
knowledge-governance/cmd/worker/library_recovery.go
T
姚章浩 3643e9bf82
build / build (push) Successful in 1m20s
fix(library): 维护收尾必须按预备份恢复并校验
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
2026-10-29 00:50:32 +08:00

164 lines
6.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
if err != nil || windowJob.SnapshotID == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
prebackup := windowJob.SnapshotID
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
return nil
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
versions, err := store.LibraryVersions(ctx, scope, 50)
if err != nil {
return ""
}
for _, item := range versions {
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
}
for _, item := range versions {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
return ""
}