Files
knowledge-governance/cmd/worker/library_recovery.go
T
姚章浩 f9457fab52
build / build (push) Successful in 1m35s
fix(library): 打通恢复验证的正常恢复与页面联动
B1/B2:演练在同一维护切点做两次真实捕获(写前镜像与来源身份),
写集合在写第一笔之前落盘并登记;恢复走限定范围的 RestoreDrill,
复用同一份 DS01 身份合同与 Apply 原语整段回放,不再让同快照必被拒绝。
worker 接治理维护生命周期(实例+scope 行、真实入口状态、成功才释放),
held 保留行并由同 job 的 library.recover 收口(kind 过滤贯通 verify)。
B3:证据优先,failed/invalid 与读取错误即使 env=true 也拒绝。
B4/B5:GET 投影在飞作业,子面板回调父页面轮询;观测绑定 scope 与代次,
刷新失败明确过期,不再显示旧的"已验证"。
迁移 79 不自动 down;未部署、未调用业务目标。
2026-11-03 11:14:27 +08:00

226 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"bytes"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
SourceSnapshot string `json:"source_snapshot_id,omitempty"`
TargetID string `json:"target_id,omitempty"`
OriginalOperation string `json:"original_operation_id,omitempty"`
BackupSnapshot string `json:"backup_snapshot_id,omitempty"`
PlanDigest string `json:"plan_digest,omitempty"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
if err := store.ValidateLibraryRecoveryQueue(ctx, job, payload.Scope, payload.WindowJobID); err != nil {
return err
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。
windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false)
if admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, store, config, job, payload.WindowJobID, func(ctx context.Context, writer *libraryAttemptWriter) error {
guarded := writer
prebackup := windowJob.SnapshotID
if prebackup == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.WindowJobID, prebackup)
if ledgerErr != nil {
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
for _, pair := range [][2]string{{payload.SourceSnapshot, ledger.TargetSnapshotID}, {payload.TargetID, config.LibraryTargetIdentity()}, {payload.OriginalOperation, windowJob.OperationID}, {payload.BackupSnapshot, prebackup}, {payload.PlanDigest, ledger.Digest}} {
if pair[0] != "" && pair[0] != pair[1] {
return errors.New("recovery supplied identity differs from original immutable binding")
}
}
recoveryDigest, digestErr := governance.LibraryRecoveryDigest(payload.Scope, job.ID, payload.WindowJobID, windowJob.OperationID, config.LibraryTargetIdentity(), ledger.TargetSnapshotID, prebackup, ledger.Digest)
if digestErr != nil {
return digestErr
}
recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, job.ID),
Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup,
RequestDigest: recoveryDigest, OperationKey: job.ID, Actor: job.LeaseOwner, CreatedAt: job.CreatedAt,
})
if opErr != nil {
return opErr
}
claim := objectgateway.ClaimFromContext(ctx)
if claim.JobID == "" {
return errors.New("recovery attempt context missing")
}
claim.RecoveryJobID, claim.RecoveryOperationID = job.ID, recoveryOp.ID
ctx = objectgateway.WithClaim(ctx, claim)
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, restoreDispatchGate(config, store), payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger,
// 恢复验证写的是"隔离 canary",不是发布计划:同一 job 的收尾走受限的演练
// 恢复入口(同样的身份合同 + 整段回放),不能拿发布计划合同去回放它。
drill: windowJob.Kind == governance.LibraryJobVerify}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
// 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经
// 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。
// 事件是追加的,不改写原来那条失败作业与失败版本。
digest := result.Digest
if digest == "" {
return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)")
}
if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID,
digest, "按预备份恢复并逐类校验通过"); err != nil {
return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err)
}
// 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本"
// 的窗口——那正是页面把恢复成功显示成待恢复的状态。
if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s",
payload.WindowJobID, payload.Scope, prebackup, digest)
return nil
})
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
writeSet *libraryrelease.WriteSet
// drill 表示窗口属于恢复演练:它的写集合是 canary,不是发布执行计划。
drill bool
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
if r.writeSet == nil {
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
}
if r.writeSet.BackupSnapshotID != backupID {
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
}
ref := libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID, WriteSet: r.writeSet}
if r.drill {
return r.target.RestoreDrill(ctx, ref)
}
return r.target.Restore(ctx, ref)
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
// 取不到映射才退回最近记录)。这里不再复制第二份。