f9457fab52
build / build (push) Successful in 1m35s
B1/B2:演练在同一维护切点做两次真实捕获(写前镜像与来源身份), 写集合在写第一笔之前落盘并登记;恢复走限定范围的 RestoreDrill, 复用同一份 DS01 身份合同与 Apply 原语整段回放,不再让同快照必被拒绝。 worker 接治理维护生命周期(实例+scope 行、真实入口状态、成功才释放), held 保留行并由同 job 的 library.recover 收口(kind 过滤贯通 verify)。 B3:证据优先,failed/invalid 与读取错误即使 env=true 也拒绝。 B4/B5:GET 投影在飞作业,子面板回调父页面轮询;观测绑定 scope 与代次, 刷新失败明确过期,不再显示旧的"已验证"。 迁移 79 不自动 down;未部署、未调用业务目标。
226 lines
11 KiB
Go
226 lines
11 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"bytes"
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/objectgateway"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||
type libraryRecoveryPayload struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
SourceSnapshot string `json:"source_snapshot_id,omitempty"`
|
||
TargetID string `json:"target_id,omitempty"`
|
||
OriginalOperation string `json:"original_operation_id,omitempty"`
|
||
BackupSnapshot string `json:"backup_snapshot_id,omitempty"`
|
||
PlanDigest string `json:"plan_digest,omitempty"`
|
||
}
|
||
|
||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||
var payload libraryRecoveryPayload
|
||
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
|
||
decoder.DisallowUnknownFields()
|
||
if err := decoder.Decode(&payload); err != nil ||
|
||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||
return errors.New("invalid library recovery payload")
|
||
}
|
||
if err := store.ValidateLibraryRecoveryQueue(ctx, job, payload.Scope, payload.WindowJobID); err != nil {
|
||
return err
|
||
}
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||
}
|
||
defer manager.Close()
|
||
identity := manager.Target()
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||
}
|
||
|
||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||
}
|
||
if lock == nil || lock.ReleasedAt != nil {
|
||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||
}
|
||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||
}
|
||
if lock.TargetFingerprint != identity.Fingerprint {
|
||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||
}
|
||
// 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。
|
||
windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false)
|
||
if admitErr != nil {
|
||
return admitErr
|
||
}
|
||
return runOwnedLibraryAttempt(ctx, store, config, job, payload.WindowJobID, func(ctx context.Context, writer *libraryAttemptWriter) error {
|
||
guarded := writer
|
||
prebackup := windowJob.SnapshotID
|
||
if prebackup == "" {
|
||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||
}
|
||
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
|
||
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
|
||
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||
payload.WindowJobID, prebackup)
|
||
if ledgerErr != nil {
|
||
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
for _, pair := range [][2]string{{payload.SourceSnapshot, ledger.TargetSnapshotID}, {payload.TargetID, config.LibraryTargetIdentity()}, {payload.OriginalOperation, windowJob.OperationID}, {payload.BackupSnapshot, prebackup}, {payload.PlanDigest, ledger.Digest}} {
|
||
if pair[0] != "" && pair[0] != pair[1] {
|
||
return errors.New("recovery supplied identity differs from original immutable binding")
|
||
}
|
||
}
|
||
recoveryDigest, digestErr := governance.LibraryRecoveryDigest(payload.Scope, job.ID, payload.WindowJobID, windowJob.OperationID, config.LibraryTargetIdentity(), ledger.TargetSnapshotID, prebackup, ledger.Digest)
|
||
if digestErr != nil {
|
||
return digestErr
|
||
}
|
||
recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||
ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, job.ID),
|
||
Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup,
|
||
RequestDigest: recoveryDigest, OperationKey: job.ID, Actor: job.LeaseOwner, CreatedAt: job.CreatedAt,
|
||
})
|
||
if opErr != nil {
|
||
return opErr
|
||
}
|
||
claim := objectgateway.ClaimFromContext(ctx)
|
||
if claim.JobID == "" {
|
||
return errors.New("recovery attempt context missing")
|
||
}
|
||
claim.RecoveryJobID, claim.RecoveryOperationID = job.ID, recoveryOp.ID
|
||
ctx = objectgateway.WithClaim(ctx, claim)
|
||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||
target, _, wireErr := buildLibraryProductionStack(ctx, config, restoreDispatchGate(config, store), payload.Scope, prebackup, knownMapping)
|
||
if wireErr != nil {
|
||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||
}
|
||
defer target.Close()
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger,
|
||
// 恢复验证写的是"隔离 canary",不是发布计划:同一 job 的收尾走受限的演练
|
||
// 恢复入口(同样的身份合同 + 整段回放),不能拿发布计划合同去回放它。
|
||
drill: windowJob.Kind == governance.LibraryJobVerify}
|
||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||
now := time.Now().UTC()
|
||
if recoverErr != nil {
|
||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
_ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||
return recoverErr
|
||
}
|
||
if result.State != "recovered" {
|
||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||
_ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||
return errors.New(note)
|
||
}
|
||
// 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经
|
||
// 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。
|
||
// 事件是追加的,不改写原来那条失败作业与失败版本。
|
||
digest := result.Digest
|
||
if digest == "" {
|
||
return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)")
|
||
}
|
||
if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID,
|
||
digest, "按预备份恢复并逐类校验通过"); err != nil {
|
||
return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err)
|
||
}
|
||
// 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本"
|
||
// 的窗口——那正是页面把恢复成功显示成待恢复的状态。
|
||
if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID,
|
||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||
}
|
||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s",
|
||
payload.WindowJobID, payload.Scope, prebackup, digest)
|
||
return nil
|
||
})
|
||
}
|
||
|
||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||
type recoveryRestorer struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
writeSet *libraryrelease.WriteSet
|
||
// drill 表示窗口属于恢复演练:它的写集合是 canary,不是发布执行计划。
|
||
drill bool
|
||
}
|
||
|
||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||
if backupID != r.expected {
|
||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||
}
|
||
if r.writeSet == nil {
|
||
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
|
||
}
|
||
if r.writeSet.BackupSnapshotID != backupID {
|
||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||
}
|
||
ref := libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID, WriteSet: r.writeSet}
|
||
if r.drill {
|
||
return r.target.RestoreDrill(ctx, ref)
|
||
}
|
||
return r.target.Restore(ctx, ref)
|
||
}
|
||
|
||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||
type recoveryVerifier struct {
|
||
target *v071.ProductionLibraryTarget
|
||
expected string
|
||
}
|
||
|
||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||
if backupID != v.expected {
|
||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||
}
|
||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||
return "", err
|
||
}
|
||
state, err := v.target.Current(ctx)
|
||
if err != nil {
|
||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||
}
|
||
raw, err := json.Marshal(state)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
// knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查,
|
||
// 取不到映射才退回最近记录)。这里不再复制第二份。
|