a7d643635a
build / build (push) Successful in 1m27s
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2, 导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的 owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。 R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引 (job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与 operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、 新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律 ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。 R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式 结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。 R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release 路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。 同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点 改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、 恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。 测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例; go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
204 lines
9.7 KiB
Go
204 lines
9.7 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||
type libraryRecoveryPayload struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
}
|
||
|
||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryRecoveryPayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||
return errors.New("invalid library recovery payload")
|
||
}
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||
}
|
||
defer manager.Close()
|
||
identity := manager.Target()
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||
}
|
||
|
||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||
}
|
||
if lock == nil || lock.ReleasedAt != nil {
|
||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||
}
|
||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||
}
|
||
if lock.TargetFingerprint != identity.Fingerprint {
|
||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||
}
|
||
// 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。
|
||
windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false)
|
||
if admitErr != nil {
|
||
return admitErr
|
||
}
|
||
// 恢复是一次**新执行代次**:它接管这个窗口,旧代次从此不能写业务状态/释放锁(F13)。
|
||
writer, attemptErr := beginLibraryWriter(ctx, store, payload.WindowJobID, libraryQueueOwner(config.WorkerID, job, payload.WindowJobID))
|
||
if attemptErr != nil {
|
||
return fmt.Errorf("无法取得恢复窗口的执行权:%w", attemptErr)
|
||
}
|
||
guarded := writer
|
||
prebackup := windowJob.SnapshotID
|
||
if prebackup == "" {
|
||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||
}
|
||
// 恢复是**新意图**(kind=recover):内容更旧、意图更新。把恢复事实挂在失败作业的旧
|
||
// operation 上会被 current 的次序(按意图创建时间)判成迟到事实而拒绝——F06 就治不好。
|
||
recoveryDigest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobRecover,
|
||
map[string]any{"window_job": payload.WindowJobID, "backup_snapshot": prebackup})
|
||
if digestErr != nil {
|
||
return fmt.Errorf("无法计算恢复意图摘要:%w", digestErr)
|
||
}
|
||
recoveryKey := fmt.Sprintf("%s#%d", payload.WindowJobID, writer.attempt.Epoch)
|
||
recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||
ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, recoveryKey),
|
||
Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup,
|
||
RequestDigest: recoveryDigest, OperationKey: recoveryKey, Actor: job.LeaseOwner,
|
||
CreatedAt: time.Now().UTC(),
|
||
})
|
||
if opErr != nil {
|
||
return fmt.Errorf("无法登记恢复意图:%w", opErr)
|
||
}
|
||
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
|
||
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
|
||
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||
payload.WindowJobID, prebackup)
|
||
if ledgerErr != nil {
|
||
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||
if wireErr != nil {
|
||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||
}
|
||
defer target.Close()
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
|
||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||
now := time.Now().UTC()
|
||
if recoverErr != nil {
|
||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||
return recoverErr
|
||
}
|
||
if result.State != "recovered" {
|
||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
return errors.New(note)
|
||
}
|
||
// 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经
|
||
// 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。
|
||
// 事件是追加的,不改写原来那条失败作业与失败版本。
|
||
digest := result.Digest
|
||
if digest == "" {
|
||
return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)")
|
||
}
|
||
if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID,
|
||
digest, "按预备份恢复并逐类校验通过"); err != nil {
|
||
return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err)
|
||
}
|
||
// 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本"
|
||
// 的窗口——那正是页面把恢复成功显示成待恢复的状态。
|
||
if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID,
|
||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||
}
|
||
writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "recovered")
|
||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s",
|
||
payload.WindowJobID, payload.Scope, prebackup, digest)
|
||
return nil
|
||
}
|
||
|
||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||
type recoveryRestorer struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
writeSet *libraryrelease.WriteSet
|
||
}
|
||
|
||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||
if backupID != r.expected {
|
||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||
}
|
||
if r.writeSet == nil {
|
||
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
|
||
}
|
||
if r.writeSet.BackupSnapshotID != backupID {
|
||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||
}
|
||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
|
||
WriteSet: r.writeSet})
|
||
}
|
||
|
||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||
type recoveryVerifier struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||
if backupID != v.expected {
|
||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||
}
|
||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||
return "", err
|
||
}
|
||
state, err := v.target.Current(ctx)
|
||
if err != nil {
|
||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||
}
|
||
raw, err := json.Marshal(state)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
|
||
// 取不到映射才退回最近记录)。这里不再复制第二份。
|