package main import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryrelease" postgresstore "knowledge-governance/internal/platform/postgres" v071 "knowledge-governance/internal/releaseadapter/v071" ) // libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。 // 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。 type libraryRecoveryPayload struct { Scope string `json:"scope"` WindowJobID string `json:"window_job_id"` } // runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口: // 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源), // 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。 func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryRecoveryPayload if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" { return errors.New("invalid library recovery payload") } manager, err := openLibraryMaintenance(config) if err != nil { return fmt.Errorf("目标维护未接线,不能做恢复:%w", err) } defer manager.Close() identity := manager.Target() if topoErr := manager.VerifyTopology(ctx); topoErr != nil { return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr) } lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance) if err != nil { return fmt.Errorf("读不到实例维护记录:%w", err) } if lock == nil || lock.ReleasedAt != nil { return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复") } if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope { return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope) } if lock.TargetFingerprint != identity.Fingerprint { return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标") } // 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。 windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false) if admitErr != nil { return admitErr } // 恢复是一次**新执行代次**:它接管这个窗口,旧代次从此不能写业务状态/释放锁(F13)。 writer, attemptErr := beginLibraryWriter(ctx, store, payload.WindowJobID, libraryQueueOwner(config.WorkerID, job, payload.WindowJobID)) if attemptErr != nil { return fmt.Errorf("无法取得恢复窗口的执行权:%w", attemptErr) } guarded := writer prebackup := windowJob.SnapshotID if prebackup == "" { return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") } // 恢复是**新意图**(kind=recover):内容更旧、意图更新。把恢复事实挂在失败作业的旧 // operation 上会被 current 的次序(按意图创建时间)判成迟到事实而拒绝——F06 就治不好。 recoveryDigest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobRecover, map[string]any{"window_job": payload.WindowJobID, "backup_snapshot": prebackup}) if digestErr != nil { return fmt.Errorf("无法计算恢复意图摘要:%w", digestErr) } recoveryKey := fmt.Sprintf("%s#%d", payload.WindowJobID, writer.attempt.Epoch) recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, recoveryKey), Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup, RequestDigest: recoveryDigest, OperationKey: recoveryKey, Actor: job.LeaseOwner, CreatedAt: time.Now().UTC(), }) if opErr != nil { return fmt.Errorf("无法登记恢复意图:%w", opErr) } // 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣, // 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。 ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, payload.WindowJobID, prebackup) if ledgerErr != nil { note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error() _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) return errors.New(note) } knownMapping := knownIndexMapping(ctx, store, payload.Scope) target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping) if wireErr != nil { return fmt.Errorf("恢复目标装配失败:%w", wireErr) } defer target.Close() coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) if openErr != nil { return fmt.Errorf("维护协调器不可用:%w", openErr) } defer control.Close() defer coordinator.Close() restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger} verifier := recoveryVerifier{target: target, expected: prebackup} result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier) now := time.Now().UTC() if recoverErr != nil { note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error() _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) return recoverErr } if result.State != "recovered" { note := "恢复流程结束但状态不是 recovered:" + result.State _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) return errors.New(note) } // 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经 // 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。 // 事件是追加的,不改写原来那条失败作业与失败版本。 digest := result.Digest if digest == "" { return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)") } if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID, digest, "按预备份恢复并逐类校验通过"); err != nil { return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err) } // 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本" // 的窗口——那正是页面把恢复成功显示成待恢复的状态。 if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID, "按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil { return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err) } writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "recovered") log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s", payload.WindowJobID, payload.Scope, prebackup, digest) return nil } // recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致 // 就拒绝——那是两份快照,恢复出来的东西不能算数。 type recoveryRestorer struct { target *v071.ProductionLibraryTarget expected string writeSet *libraryrelease.WriteSet } func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error { if backupID != r.expected { return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected) } if r.writeSet == nil { return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣") } if r.writeSet.BackupSnapshotID != backupID { return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID) } return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID, WriteSet: r.writeSet}) } // recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。 type recoveryVerifier struct { target *v071.ProductionLibraryTarget expected string } func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) { if backupID != v.expected { return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected) } // Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。 if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil { return "", err } state, err := v.target.Current(ctx) if err != nil { return "", fmt.Errorf("恢复后读不到目标状态:%w", err) } raw, err := json.Marshal(state) if err != nil { return "", err } sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:]), nil } // knownIndexMapping 在 library_versions.go:同步与恢复共用同一份(当前生效版本精确查, // 取不到映射才退回最近记录)。这里不再复制第二份。