Files
knowledge-governance/cmd/worker/library_recovery.go
T
姚章浩 a7d643635a
build / build (push) Successful in 1m27s
fix(library): 守卫按行取作业身份、同作业多代次、原作业续作与执行域准入
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2,
导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的
owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。

R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引
(job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与
operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、
新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律
ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。

R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式
结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。

R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release
路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。

同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点
改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、
恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。

测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例;
go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
2026-10-30 17:20:40 +08:00

204 lines
9.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。
windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false)
if admitErr != nil {
return admitErr
}
// 恢复是一次**新执行代次**:它接管这个窗口,旧代次从此不能写业务状态/释放锁(F13)。
writer, attemptErr := beginLibraryWriter(ctx, store, payload.WindowJobID, libraryQueueOwner(config.WorkerID, job, payload.WindowJobID))
if attemptErr != nil {
return fmt.Errorf("无法取得恢复窗口的执行权:%w", attemptErr)
}
guarded := writer
prebackup := windowJob.SnapshotID
if prebackup == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
// 恢复是**新意图**(kind=recover):内容更旧、意图更新。把恢复事实挂在失败作业的旧
// operation 上会被 current 的次序(按意图创建时间)判成迟到事实而拒绝——F06 就治不好。
recoveryDigest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobRecover,
map[string]any{"window_job": payload.WindowJobID, "backup_snapshot": prebackup})
if digestErr != nil {
return fmt.Errorf("无法计算恢复意图摘要:%w", digestErr)
}
recoveryKey := fmt.Sprintf("%s#%d", payload.WindowJobID, writer.attempt.Epoch)
recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, recoveryKey),
Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup,
RequestDigest: recoveryDigest, OperationKey: recoveryKey, Actor: job.LeaseOwner,
CreatedAt: time.Now().UTC(),
})
if opErr != nil {
return fmt.Errorf("无法登记恢复意图:%w", opErr)
}
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.WindowJobID, prebackup)
if ledgerErr != nil {
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
// 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经
// 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。
// 事件是追加的,不改写原来那条失败作业与失败版本。
digest := result.Digest
if digest == "" {
return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)")
}
if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID,
digest, "按预备份恢复并逐类校验通过"); err != nil {
return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err)
}
// 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本"
// 的窗口——那正是页面把恢复成功显示成待恢复的状态。
if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "recovered")
log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s",
payload.WindowJobID, payload.Scope, prebackup, digest)
return nil
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
writeSet *libraryrelease.WriteSet
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
if r.writeSet == nil {
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
}
if r.writeSet.BackupSnapshotID != backupID {
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
WriteSet: r.writeSet})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
// 取不到映射才退回最近记录)。这里不再复制第二份。