0088004cd6
接管分类、基线判定、已知索引映射、同步闸口的创建判据与归档保留的保护标记, 原来都用 LibraryVersions(scope, 50/100) 这个分页窗口再在内存里筛。生效版本每 scope 只有一行、可能很久以前登记,历史一长就被挤出窗口 —— 于是"已生效"被读成 "没有生效":接管会把已经上线的目标重写一遍,基线入口会给已有权威版本的目标 再建一次基线。 现在统一走两条精确查询:LibraryCurrentVersion(kind='production_effective' AND superseded_at IS NULL,每 scope 至多一行)与 LibraryVersionOfSnapshot(按快照 身份取回"已顶替"证据)。多行、自相矛盾、读不到一律 fail closed;同一含义只留 一份实现(knownIndexMapping 同步与恢复共用)。LibraryVersions 仍是 1..100 的 展示窗口,只用于列表。 验证:真实 PG 上先复现(窗口实现 ⇒ found=false),改后 7 条 store 契约用例与 worker 端到端判定用例全绿;本机 make verify 15 项 success(用例 1264 通过 / 0 失败 / 93 已声明可选跳过)。
163 lines
7.1 KiB
Go
163 lines
7.1 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||
type libraryRecoveryPayload struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
}
|
||
|
||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryRecoveryPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||
return errors.New("invalid library recovery payload")
|
||
}
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||
}
|
||
defer manager.Close()
|
||
identity := manager.Target()
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||
}
|
||
|
||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||
}
|
||
if lock == nil || lock.ReleasedAt != nil {
|
||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||
}
|
||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||
}
|
||
if lock.TargetFingerprint != identity.Fingerprint {
|
||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||
}
|
||
|
||
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
|
||
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
|
||
if err != nil || windowJob.SnapshotID == "" {
|
||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||
}
|
||
prebackup := windowJob.SnapshotID
|
||
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
|
||
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
|
||
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||
payload.WindowJobID, prebackup)
|
||
if ledgerErr != nil {
|
||
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||
if wireErr != nil {
|
||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||
}
|
||
defer target.Close()
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
|
||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||
now := time.Now().UTC()
|
||
if recoverErr != nil {
|
||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||
return recoverErr
|
||
}
|
||
if result.State != "recovered" {
|
||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
return errors.New(note)
|
||
}
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
|
||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||
}
|
||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
|
||
return nil
|
||
}
|
||
|
||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||
type recoveryRestorer struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
writeSet *libraryrelease.WriteSet
|
||
}
|
||
|
||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||
if backupID != r.expected {
|
||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||
}
|
||
if r.writeSet == nil {
|
||
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
|
||
}
|
||
if r.writeSet.BackupSnapshotID != backupID {
|
||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||
}
|
||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
|
||
WriteSet: r.writeSet})
|
||
}
|
||
|
||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||
type recoveryVerifier struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||
if backupID != v.expected {
|
||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||
}
|
||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||
return "", err
|
||
}
|
||
state, err := v.target.Current(ctx)
|
||
if err != nil {
|
||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||
}
|
||
raw, err := json.Marshal(state)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
|
||
// 取不到映射才退回最近记录)。这里不再复制第二份。
|