Files
knowledge-governance/cmd/worker/library_recovery.go
T
姚章浩 0088004cd6 fix(library): 当前生效版本改按精确查询
接管分类、基线判定、已知索引映射、同步闸口的创建判据与归档保留的保护标记,
原来都用 LibraryVersions(scope, 50/100) 这个分页窗口再在内存里筛。生效版本每
scope 只有一行、可能很久以前登记,历史一长就被挤出窗口 —— 于是"已生效"被读成
"没有生效":接管会把已经上线的目标重写一遍,基线入口会给已有权威版本的目标
再建一次基线。

现在统一走两条精确查询:LibraryCurrentVersion(kind='production_effective' AND
superseded_at IS NULL,每 scope 至多一行)与 LibraryVersionOfSnapshot(按快照
身份取回"已顶替"证据)。多行、自相矛盾、读不到一律 fail closed;同一含义只留
一份实现(knownIndexMapping 同步与恢复共用)。LibraryVersions 仍是 1..100 的
展示窗口,只用于列表。

验证:真实 PG 上先复现(窗口实现 ⇒ found=false),改后 7 条 store 契约用例与
worker 端到端判定用例全绿;本机 make verify 15 项 success(用例 1264 通过 /
0 失败 / 93 已声明可选跳过)。
2026-10-30 03:09:21 +08:00

163 lines
7.1 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
if err != nil || windowJob.SnapshotID == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
prebackup := windowJob.SnapshotID
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.WindowJobID, prebackup)
if ledgerErr != nil {
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
return nil
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
writeSet *libraryrelease.WriteSet
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
if r.writeSet == nil {
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
}
if r.writeSet.BackupSnapshotID != backupID {
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
WriteSet: r.writeSet})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
// 取不到映射才退回最近记录)。这里不再复制第二份。