e0768f2caa
build / build (push) Successful in 1m34s
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它 知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可 恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例 证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。 改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据" 算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份 版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对 资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。 resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。 整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。 本地验证(已跑): - make verify 15 项全 success(identity 因本地档案 skipped) - go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES + 真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/ 历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复; 跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净 - SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变 - go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、 接管不重新规划
182 lines
7.6 KiB
Go
182 lines
7.6 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||
type libraryRecoveryPayload struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
}
|
||
|
||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryRecoveryPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||
return errors.New("invalid library recovery payload")
|
||
}
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||
}
|
||
defer manager.Close()
|
||
identity := manager.Target()
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||
}
|
||
|
||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||
}
|
||
if lock == nil || lock.ReleasedAt != nil {
|
||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||
}
|
||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||
}
|
||
if lock.TargetFingerprint != identity.Fingerprint {
|
||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||
}
|
||
|
||
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
|
||
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
|
||
if err != nil || windowJob.SnapshotID == "" {
|
||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||
}
|
||
prebackup := windowJob.SnapshotID
|
||
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
|
||
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
|
||
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||
payload.WindowJobID, prebackup)
|
||
if ledgerErr != nil {
|
||
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||
if wireErr != nil {
|
||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||
}
|
||
defer target.Close()
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
|
||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||
now := time.Now().UTC()
|
||
if recoverErr != nil {
|
||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||
return recoverErr
|
||
}
|
||
if result.State != "recovered" {
|
||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
return errors.New(note)
|
||
}
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
|
||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||
}
|
||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
|
||
return nil
|
||
}
|
||
|
||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||
type recoveryRestorer struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
writeSet *libraryrelease.WriteSet
|
||
}
|
||
|
||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||
if backupID != r.expected {
|
||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||
}
|
||
if r.writeSet == nil {
|
||
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
|
||
}
|
||
if r.writeSet.BackupSnapshotID != backupID {
|
||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||
}
|
||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
|
||
WriteSet: r.writeSet})
|
||
}
|
||
|
||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||
type recoveryVerifier struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||
if backupID != v.expected {
|
||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||
}
|
||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||
return "", err
|
||
}
|
||
state, err := v.target.Current(ctx)
|
||
if err != nil {
|
||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||
}
|
||
raw, err := json.Marshal(state)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
|
||
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
|
||
versions, err := store.LibraryVersions(ctx, scope, 50)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, item := range versions {
|
||
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
return raw
|
||
}
|
||
}
|
||
}
|
||
for _, item := range versions {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
return raw
|
||
}
|
||
}
|
||
return ""
|
||
}
|