Files
knowledge-governance/cmd/worker/library_recovery.go
T
姚章浩 e0768f2caa
build / build (push) Successful in 1m34s
fix(library): 写集合台账与跨范围冲突预检
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它
知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可
恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例
证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。

改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据"
算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份
版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对
资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。
resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。
整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。

本地验证(已跑):
- make verify 15 项全 success(identity 因本地档案 skipped)
- go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES +
  真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/
  历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复;
  跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净
- SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变
- go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、
  接管不重新规划
2026-10-29 23:55:13 +08:00

182 lines
7.6 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
if err != nil || windowJob.SnapshotID == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
prebackup := windowJob.SnapshotID
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.WindowJobID, prebackup)
if ledgerErr != nil {
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
return nil
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
writeSet *libraryrelease.WriteSet
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
if r.writeSet == nil {
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
}
if r.writeSet.BackupSnapshotID != backupID {
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
WriteSet: r.writeSet})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
versions, err := store.LibraryVersions(ctx, scope, 50)
if err != nil {
return ""
}
for _, item := range versions {
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
}
for _, item := range versions {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
return ""
}