Files
knowledge-governance/internal/releaseadapter/v071/library_index_restore_dispatch.go
T
姚章浩 245cefd6b0 fix(library): 收紧 DS01 派发租约/投递/治理身份与绑定冲突
F2.a:Acquire/Confirm 在取得 effect 行锁后用数据库时钟重核 job/attempt/
恢复投递租约,等锁跨过到期零许可且状态保持 prepared。

F2.b:同一 CAS 事务锁 recovery queue 与 operation,复核 window/scope、
idempotency、owner/attempt/lease,并按原 window 锚点重算恢复请求摘要。

F3.identity:激活证据 v3 携带 PG 集群 system_identifier + 库 OID、完整构建
SHA 与协议批次;激活与使用点都现场重读并拒绝同名异域/副本/unknown/不兼容 SHA。

F4.binding:区分绑定冲突与仅禁止重发的 Held,准入/确认/续作/收尾在所有状态
拒绝冲突,合法 dispatching + 精确 marker 仍可确认。
2026-11-02 20:36:36 +08:00

461 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"time"
"knowledge-governance/internal/libraryrelease"
)
// DS01:删除型恢复(Before 非空、After 为空)的标记 create 是唯一会"无中生有"的
// 恢复写:ES 里文档缺失与"从未写过"完全同形,所以它不能靠外观、stats 或内存计数
// 放行。新协议把"是否已经有人取得发送许可"放进治理库的单调台账;本文件是 adapter
// 侧的全部接入点——准入、唯一派发、确认与收尾检查。
//
// 明确不保证:ES 端 fence;已 dispatching 的旧 owner 绝不迟到一次。协议只保证
// 遵守协议的派发者不会得到第二次许可,且 dispatching 永不回退。
// restoreMarkerBinding 用**原持久锚点**构造不可变绑定:
// - scope 来自真实装配;
// - source 是发布基线(WriteSet.TargetSnapshotID),backup 是本次写前镜像;
// - target instance/fingerprint/index UUID 实时观察并与本次恢复的冻结值比较。
func (t *ProductionLibraryTarget) restoreMarkerBinding(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (libraryrelease.RestoreMarkerBinding, error) {
if t.config.RestoreDispatch == nil {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch protocol is not wired")
}
if t.activePlan == nil {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch requires the frozen plan")
}
if t.config.RestoreTargetIdentity == "" || t.config.RestoreInstance == "" ||
t.config.RestoreOperationID == "" || t.config.RestoreWindowJobID == "" ||
t.config.RestoreAttemptOwner == "" || t.config.RestoreAttemptEpoch <= 0 {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original job/attempt identity")
}
if t.restoreScope == "" || t.restoreSourceSnapshot == "" || t.restoreBackupSnapshot == "" {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original scope/snapshot anchors")
}
if t.restoreSourceSnapshot == t.restoreBackupSnapshot {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch would use one snapshot as both source and backup")
}
fingerprint, indexUUID, err := t.frozenRestoreTarget(ctx)
if err != nil {
return libraryrelease.RestoreMarkerBinding{}, err
}
changeRaw, err := json.Marshal(c)
if err != nil {
return libraryrelease.RestoreMarkerBinding{}, err
}
changeDigest := sha256.Sum256(changeRaw)
markerDigest := sha256.Sum256(body)
method, path, _, err := t.indexMutation(c, body, nil, nil, true)
if err != nil {
return libraryrelease.RestoreMarkerBinding{}, err
}
requestDigest := sha256.Sum256([]byte(method + "\x00" + path))
return libraryrelease.RestoreMarkerBinding{
Key: libraryrelease.RestoreMarkerKey{
TargetIdentity: t.config.RestoreTargetIdentity,
OperationID: t.config.RestoreOperationID,
WindowJobID: t.config.RestoreWindowJobID,
PlanDigest: t.activePlan.Digest,
Index: c.Index,
DocumentID: c.ID,
Phase: libraryrelease.RestoreMarkerPhaseCreate,
},
Scope: t.restoreScope,
SourceSnapshot: t.restoreSourceSnapshot,
BackupSnapshot: t.restoreBackupSnapshot,
TargetInstance: t.config.RestoreInstance,
TargetFingerprint: fingerprint,
IndexUUID: indexUUID,
RecoveryJobID: t.config.RestoreRecoveryJobID,
RecoveryOperationID: t.config.RestoreRecoveryOperationID,
ChangeDigest: hex.EncodeToString(changeDigest[:]),
MarkerDigest: hex.EncodeToString(markerDigest[:]),
RequestDigest: hex.EncodeToString(requestDigest[:]),
Owner: t.config.RestoreAttemptOwner,
Epoch: t.config.RestoreAttemptEpoch,
}, nil
}
func heldRestoreMarker(status libraryrelease.RestoreDispatchStatus, context string) error {
return fmt.Errorf("%w: %s(state=%s)", libraryrelease.ErrRestoreDispatchHeld, context, status.State)
}
// planEffect 按 index/document 找到当前冻结计划里的 effect(用于权威全集与文件证据对照)。
func (t *ProductionLibraryTarget) planEffect(index, document string) (libraryrelease.IndexWrite, bool) {
for _, c := range t.activePlan.Index {
if c.Index == index && c.ID == document {
return c, true
}
}
return libraryrelease.IndexWrite{}, false
}
// admitRestorePlan 是 Restore 在**任何** PG/对象/索引写之前的纯读准入。
//
// 删除型恢复一旦开始写 recovery 的行/对象,后面才发现某个 effect 的派发结果未知,
// 目标已经从"未动"变成"半恢复"。所以先按整份冻结计划逐 effect 判定,并先对照原
// window 的权威台账全集(跨 plan):
// - 台账不存在(旧世界):只有"没有任何恢复意图"才算首次恢复;有意图就 held;
// - prepared:唯一可续作区段,但出现 marker/claimed/finalize/restored 任一矛盾即 held;
// - dispatching:只有精确 marker+世代才可走确认分支,否则 held;
// - marker_confirmed:只允许确认后的 clean 路径。
func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
gate := t.config.RestoreDispatch
if gate != nil {
records, err := gate.UnresolvedRestoreDispatches(ctx, t.config.RestoreTargetIdentity,
t.config.RestoreOperationID, t.config.RestoreWindowJobID)
if err != nil {
return fmt.Errorf("恢复准入读不到权威派发台账:%w", err)
}
for _, record := range records {
if record.PlanDigest != t.activePlan.Digest {
return fmt.Errorf("%w: 原 window 存在另一份计划的派发记录(plan=%s index=%s doc=%s)",
libraryrelease.ErrRestoreDispatchHeld, record.PlanDigest, record.Index, record.DocumentID)
}
if _, ok := t.planEffect(record.Index, record.DocumentID); !ok {
return fmt.Errorf("%w: 原 window 存在计划外派发记录(index=%s doc=%s state=%s)",
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID, record.State)
}
}
}
for _, c := range t.activePlan.Index {
if len(c.After) != 0 {
continue
}
body, err := t.restoreIndexClaimBody(c)
if err != nil {
return err
}
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
if err != nil {
return err
}
claimed, err := t.loadIndexEffect(c, "restore-claimed")
if err != nil {
return err
}
finalize, err := t.loadIndexEffect(c, "restore-finalize-intent")
if err != nil {
return err
}
// 已完成 clean 的 effect 是终态:恢复内容与证据都在时,重放不是未决。
done, err := t.loadIndexEffect(c, "restored")
if err != nil {
return err
}
if gate != nil {
binding, bindErr := t.restoreMarkerBinding(ctx, c, body)
if bindErr != nil {
return bindErr
}
status, found, inspErr := gate.InspectRestoreMarker(ctx, binding)
if inspErr != nil {
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
}
if found {
// 绑定冲突是类型化分类:任何状态(含已完成 clean 的终态)都必须拒绝。
// 它不能与"仅禁止第二次 create"的 dispatching-held 混为一谈——后者在
// 精确 marker 仍在时是允许确认的。
if status.BindingConflict {
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
}
if done != nil {
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
return heldRestoreMarker(status, "已完成 clean 的 effect 台账却不是 marker_confirmed")
}
// 终态:内容与世代由下面的 done 分支核对。
} else {
switch status.State {
case libraryrelease.RestoreDispatchPrepared:
if status.Held {
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
}
if claimed != nil || finalize != nil {
return heldRestoreMarker(status, "prepared 台账却已有 claimed/finalize 证据")
}
observed, obsErr := t.observePlannedIndex(ctx, c)
if obsErr != nil {
return obsErr
}
if indexBytesEqual(observed, body) {
return heldRestoreMarker(status, "prepared 台账却已存在目标标记:协议证据矛盾")
}
continue
case libraryrelease.RestoreDispatchDispatching, libraryrelease.RestoreDispatchMarkerConfirmed:
observed, obsErr := t.observePlannedIndex(ctx, c)
if obsErr != nil {
return obsErr
}
if indexBytesEqual(observed, body) && observed.SeqNo != nil && observed.PrimaryTerm != nil {
continue // 只允许原确认分支继续,不重发 create
}
return heldRestoreMarker(status, "目标上缺少可归属的精确恢复标记")
default:
return heldRestoreMarker(status, "派发台账状态不可识别")
}
}
}
}
if done != nil {
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if done.Receipt == nil || !indexBytesEqual(observed, c.Before) ||
!sameIndexGeneration(observed, done.Receipt.SeqNo, done.Receipt.PrimaryTerm) {
return fmt.Errorf("%w: 已标记 restored 的 effect 与目标现状不符", libraryrelease.ErrRestoreDispatchHeld)
}
continue
}
if intent == nil && claimed == nil && finalize == nil {
continue
}
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if intent != nil && indexBytesEqual(observed, body) {
continue // 旧标记已在目标上:按既有确认/CAS 规则处理,不派发新 create
}
return fmt.Errorf("%w: 既存恢复意图/证据没有新协议派发台账,无法证明未派发", libraryrelease.ErrRestoreDispatchHeld)
}
return nil
}
// confirmDispatchedMarker 只在"marker 与冻结内容逐位相同且世代齐全"时记录回执。
// 它不发送任何请求;确认失败即 held,绝不因为"看起来已经写好了"就继续 clean。
func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, observed indexWriteGuard) error {
if t.config.RestoreDispatch == nil || len(c.After) != 0 {
return nil
}
binding, err := t.restoreMarkerBinding(ctx, c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return err
}
if !found {
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
}
if status.BindingConflict {
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
}
if status.Held && status.State == libraryrelease.RestoreDispatchPrepared {
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
}
switch status.State {
case libraryrelease.RestoreDispatchPrepared:
// 还没派发:由调用方按 prepared 竞争许可。
return nil
case libraryrelease.RestoreDispatchMarkerConfirmed:
return nil
case libraryrelease.RestoreDispatchDispatching:
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
return heldRestoreMarker(status, "dispatching 结果未知且目标上没有精确标记")
}
receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: "created", SeqNo: observed.SeqNo, PrimaryTerm: observed.PrimaryTerm}
raw, err := json.Marshal(receipt)
if err != nil {
return err
}
return t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw)
default:
return heldRestoreMarker(status, "派发台账状态不可识别")
}
}
// dispatchRestoreMarkerCreate 是恢复期唯一获得发送许可的入口:preflight 与故障钩子
// 都在 CAS 之前,Acquire 只在 dispatching 明确提交后返回许可。
func (t *ProductionLibraryTarget) dispatchRestoreMarkerCreate(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (indexReceipt, error) {
if t.config.RestoreDispatch == nil {
return indexReceipt{}, errors.New("恢复标记派发协议未接线:拒绝发送")
}
binding, err := t.restoreMarkerBinding(ctx, c, body)
if err != nil {
return indexReceipt{}, err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return indexReceipt{}, err
}
if !found || status.State != libraryrelease.RestoreDispatchPrepared || status.Held {
return indexReceipt{}, heldRestoreMarker(status, "没有可竞争的 prepared 许可")
}
// 控制流故障注入必须在 CAS 之前:拒绝时状态仍是 prepared,可安全重试,零转发。
if err := t.ds01TestRefusal(c); err != nil {
return indexReceipt{}, err
}
if err := t.refuseRestoreWrite(c); err != nil {
return indexReceipt{}, err
}
permit, acquireStatus, err := t.config.RestoreDispatch.AcquireRestoreMarker(ctx, binding)
if err != nil {
return indexReceipt{}, err
}
if permit == nil {
return indexReceipt{}, heldRestoreMarker(acquireStatus, "许可竞争未胜出")
}
if err := t.ds01TestHoldAfterCAS(ctx, c); err != nil {
return indexReceipt{}, err
}
receipt, err := t.sendRestoreMarkerOnce(ctx, c, body, binding, permit)
if err != nil {
return indexReceipt{}, err
}
raw, err := json.Marshal(receipt)
if err != nil {
return indexReceipt{}, err
}
// 回执来自这次唯一发送:必须落进台账才算确认。写失败时 marker 已在目标上,
// 下一进程走"精确 marker+世代"确认分支,不会重发 create。
if err := t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw); err != nil {
return indexReceipt{}, fmt.Errorf("恢复标记已发送但确认未落库:%w", err)
}
return receipt, nil
}
// sendRestoreMarkerOnce 是唯一的真实网络发送点:许可消费一次、请求不可被传输层重放。
func (t *ProductionLibraryTarget) sendRestoreMarkerOnce(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage,
binding libraryrelease.RestoreMarkerBinding, permit *libraryrelease.RestoreSendPermit) (indexReceipt, error) {
digest, err := binding.Digest()
if err != nil {
return indexReceipt{}, err
}
if err := permit.Consume(binding.Key, digest); err != nil {
return indexReceipt{}, err
}
method, path, payload, err := t.indexMutation(c, body, nil, nil, true)
if err != nil {
return indexReceipt{}, err
}
var receipt indexReceipt
if err := t.search.requestJSONOnce(ctx, method, path, payload, &receipt); err != nil {
return receipt, fmt.Errorf("restore marker outcome unknown: %w", err)
}
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
return receipt, err
}
return receipt, nil
}
// verifyRestoreEffectsClosed 是 Restore 收尾前的最后一道检查:以原 window 的权威台账
// 全集为准(跨 plan),每个记录都必须是 marker_confirmed,且对应的 restored 文件证据
// 存在。prepared/dispatching、计划外 effect、换 plan 别名、缺 restored 一律拒绝。
func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context) error {
if t.config.RestoreDispatch != nil {
records, err := t.config.RestoreDispatch.UnresolvedRestoreDispatches(ctx,
t.config.RestoreTargetIdentity, t.config.RestoreOperationID, t.config.RestoreWindowJobID)
if err != nil {
return fmt.Errorf("恢复收尾读不到权威派发台账:%w", err)
}
for _, record := range records {
if record.PlanDigest != t.activePlan.Digest {
return fmt.Errorf("%w: 收尾时原 window 仍有另一份计划的派发记录(plan=%s)",
libraryrelease.ErrRestoreDispatchHeld, record.PlanDigest)
}
if record.State != libraryrelease.RestoreDispatchMarkerConfirmed {
return fmt.Errorf("%w: 收尾前派发台账仍有未决 effect(index=%s doc=%s state=%s)",
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID, record.State)
}
c, ok := t.planEffect(record.Index, record.DocumentID)
if !ok {
return fmt.Errorf("%w: 收尾时存在计划外派发记录(index=%s doc=%s)",
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID)
}
done, err := t.loadIndexEffect(c, "restored")
if err != nil {
return err
}
if done == nil {
return fmt.Errorf("%w: 收尾前仍有未 clean 的 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
// 收尾同样是**使用点**:绑定冲突即使状态看起来是 marker_confirmed 也必须拒绝。
body, err := t.restoreIndexClaimBody(c)
if err != nil {
return err
}
binding, err := t.restoreMarkerBinding(ctx, c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return fmt.Errorf("恢复收尾读不到派发台账:%w", err)
}
if !found {
return fmt.Errorf("%w: 收尾时 effect %s 的派发台账消失了", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
if status.BindingConflict {
return heldRestoreMarker(status, "收尾时派发台账的不可变绑定与本次计划冲突")
}
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
return fmt.Errorf("%w: 收尾时 effect %s 的派发状态是 %s", libraryrelease.ErrRestoreDispatchHeld, c.ID, status.State)
}
}
}
for _, c := range t.activePlan.Index {
if len(c.After) != 0 {
continue
}
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
if err != nil {
return err
}
if intent == nil {
continue
}
done, err := t.loadIndexEffect(c, "restored")
if err != nil {
return err
}
if done == nil {
return fmt.Errorf("%w: 恢复收尾前仍有未决 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
}
return nil
}
// ds01TestRefusal 是正式 worker 全链验收的最小测试接缝:只有显式隔离运行
// (KG_LIBRARY_TEST_ISOLATED=1)且指定了 effect 时才在 CAS 之前拒绝一次;生产默认
// 全关,且每个进程只触发一次,重启后的 worker 不会重复拒绝。
// ds01TestHoldAfterCAS 是 N1 的最小测试接缝:CAS 已提交、许可已拿到,但在真实发送
// 之前挂起,让验收进程能观察到 durable dispatching 后终止执行者。生产默认全关。
func (t *ProductionLibraryTarget) ds01TestHoldAfterCAS(ctx context.Context, c libraryrelease.IndexWrite) error {
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" || t.restoreHoldUsed {
return nil
}
spec := os.Getenv("KG_DS01_TEST_HOLD_AFTER_CAS")
if spec == "" || spec != c.Index+"/"+c.ID {
return nil
}
t.restoreHoldUsed = true
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(90 * time.Second):
return errors.New("injected DS01 hold after CAS expired")
}
}
func (t *ProductionLibraryTarget) ds01TestRefusal(c libraryrelease.IndexWrite) error {
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" || t.restoreRefusalUsed {
return nil
}
spec := os.Getenv("KG_DS01_TEST_REFUSE_MARKER")
if spec == "" || spec != c.Index+"/"+c.ID {
return nil
}
t.restoreRefusalUsed = true
return errors.New("injected DS01 marker refusal before dispatch CAS")
}