245cefd6b0
F2.a:Acquire/Confirm 在取得 effect 行锁后用数据库时钟重核 job/attempt/ 恢复投递租约,等锁跨过到期零许可且状态保持 prepared。 F2.b:同一 CAS 事务锁 recovery queue 与 operation,复核 window/scope、 idempotency、owner/attempt/lease,并按原 window 锚点重算恢复请求摘要。 F3.identity:激活证据 v3 携带 PG 集群 system_identifier + 库 OID、完整构建 SHA 与协议批次;激活与使用点都现场重读并拒绝同名异域/副本/unknown/不兼容 SHA。 F4.binding:区分绑定冲突与仅禁止重发的 Held,准入/确认/续作/收尾在所有状态 拒绝冲突,合法 dispatching + 精确 marker 仍可确认。
461 lines
19 KiB
Go
461 lines
19 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
)
|
||
|
||
// DS01:删除型恢复(Before 非空、After 为空)的标记 create 是唯一会"无中生有"的
|
||
// 恢复写:ES 里文档缺失与"从未写过"完全同形,所以它不能靠外观、stats 或内存计数
|
||
// 放行。新协议把"是否已经有人取得发送许可"放进治理库的单调台账;本文件是 adapter
|
||
// 侧的全部接入点——准入、唯一派发、确认与收尾检查。
|
||
//
|
||
// 明确不保证:ES 端 fence;已 dispatching 的旧 owner 绝不迟到一次。协议只保证
|
||
// 遵守协议的派发者不会得到第二次许可,且 dispatching 永不回退。
|
||
|
||
// restoreMarkerBinding 用**原持久锚点**构造不可变绑定:
|
||
// - scope 来自真实装配;
|
||
// - source 是发布基线(WriteSet.TargetSnapshotID),backup 是本次写前镜像;
|
||
// - target instance/fingerprint/index UUID 实时观察并与本次恢复的冻结值比较。
|
||
func (t *ProductionLibraryTarget) restoreMarkerBinding(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (libraryrelease.RestoreMarkerBinding, error) {
|
||
if t.config.RestoreDispatch == nil {
|
||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch protocol is not wired")
|
||
}
|
||
if t.activePlan == nil {
|
||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch requires the frozen plan")
|
||
}
|
||
if t.config.RestoreTargetIdentity == "" || t.config.RestoreInstance == "" ||
|
||
t.config.RestoreOperationID == "" || t.config.RestoreWindowJobID == "" ||
|
||
t.config.RestoreAttemptOwner == "" || t.config.RestoreAttemptEpoch <= 0 {
|
||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original job/attempt identity")
|
||
}
|
||
if t.restoreScope == "" || t.restoreSourceSnapshot == "" || t.restoreBackupSnapshot == "" {
|
||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original scope/snapshot anchors")
|
||
}
|
||
if t.restoreSourceSnapshot == t.restoreBackupSnapshot {
|
||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch would use one snapshot as both source and backup")
|
||
}
|
||
fingerprint, indexUUID, err := t.frozenRestoreTarget(ctx)
|
||
if err != nil {
|
||
return libraryrelease.RestoreMarkerBinding{}, err
|
||
}
|
||
changeRaw, err := json.Marshal(c)
|
||
if err != nil {
|
||
return libraryrelease.RestoreMarkerBinding{}, err
|
||
}
|
||
changeDigest := sha256.Sum256(changeRaw)
|
||
markerDigest := sha256.Sum256(body)
|
||
method, path, _, err := t.indexMutation(c, body, nil, nil, true)
|
||
if err != nil {
|
||
return libraryrelease.RestoreMarkerBinding{}, err
|
||
}
|
||
requestDigest := sha256.Sum256([]byte(method + "\x00" + path))
|
||
return libraryrelease.RestoreMarkerBinding{
|
||
Key: libraryrelease.RestoreMarkerKey{
|
||
TargetIdentity: t.config.RestoreTargetIdentity,
|
||
OperationID: t.config.RestoreOperationID,
|
||
WindowJobID: t.config.RestoreWindowJobID,
|
||
PlanDigest: t.activePlan.Digest,
|
||
Index: c.Index,
|
||
DocumentID: c.ID,
|
||
Phase: libraryrelease.RestoreMarkerPhaseCreate,
|
||
},
|
||
Scope: t.restoreScope,
|
||
SourceSnapshot: t.restoreSourceSnapshot,
|
||
BackupSnapshot: t.restoreBackupSnapshot,
|
||
TargetInstance: t.config.RestoreInstance,
|
||
TargetFingerprint: fingerprint,
|
||
IndexUUID: indexUUID,
|
||
RecoveryJobID: t.config.RestoreRecoveryJobID,
|
||
RecoveryOperationID: t.config.RestoreRecoveryOperationID,
|
||
ChangeDigest: hex.EncodeToString(changeDigest[:]),
|
||
MarkerDigest: hex.EncodeToString(markerDigest[:]),
|
||
RequestDigest: hex.EncodeToString(requestDigest[:]),
|
||
Owner: t.config.RestoreAttemptOwner,
|
||
Epoch: t.config.RestoreAttemptEpoch,
|
||
}, nil
|
||
}
|
||
|
||
func heldRestoreMarker(status libraryrelease.RestoreDispatchStatus, context string) error {
|
||
return fmt.Errorf("%w: %s(state=%s)", libraryrelease.ErrRestoreDispatchHeld, context, status.State)
|
||
}
|
||
|
||
// planEffect 按 index/document 找到当前冻结计划里的 effect(用于权威全集与文件证据对照)。
|
||
func (t *ProductionLibraryTarget) planEffect(index, document string) (libraryrelease.IndexWrite, bool) {
|
||
for _, c := range t.activePlan.Index {
|
||
if c.Index == index && c.ID == document {
|
||
return c, true
|
||
}
|
||
}
|
||
return libraryrelease.IndexWrite{}, false
|
||
}
|
||
|
||
// admitRestorePlan 是 Restore 在**任何** PG/对象/索引写之前的纯读准入。
|
||
//
|
||
// 删除型恢复一旦开始写 recovery 的行/对象,后面才发现某个 effect 的派发结果未知,
|
||
// 目标已经从"未动"变成"半恢复"。所以先按整份冻结计划逐 effect 判定,并先对照原
|
||
// window 的权威台账全集(跨 plan):
|
||
// - 台账不存在(旧世界):只有"没有任何恢复意图"才算首次恢复;有意图就 held;
|
||
// - prepared:唯一可续作区段,但出现 marker/claimed/finalize/restored 任一矛盾即 held;
|
||
// - dispatching:只有精确 marker+世代才可走确认分支,否则 held;
|
||
// - marker_confirmed:只允许确认后的 clean 路径。
|
||
func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
|
||
gate := t.config.RestoreDispatch
|
||
if gate != nil {
|
||
records, err := gate.UnresolvedRestoreDispatches(ctx, t.config.RestoreTargetIdentity,
|
||
t.config.RestoreOperationID, t.config.RestoreWindowJobID)
|
||
if err != nil {
|
||
return fmt.Errorf("恢复准入读不到权威派发台账:%w", err)
|
||
}
|
||
for _, record := range records {
|
||
if record.PlanDigest != t.activePlan.Digest {
|
||
return fmt.Errorf("%w: 原 window 存在另一份计划的派发记录(plan=%s index=%s doc=%s)",
|
||
libraryrelease.ErrRestoreDispatchHeld, record.PlanDigest, record.Index, record.DocumentID)
|
||
}
|
||
if _, ok := t.planEffect(record.Index, record.DocumentID); !ok {
|
||
return fmt.Errorf("%w: 原 window 存在计划外派发记录(index=%s doc=%s state=%s)",
|
||
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID, record.State)
|
||
}
|
||
}
|
||
}
|
||
for _, c := range t.activePlan.Index {
|
||
if len(c.After) != 0 {
|
||
continue
|
||
}
|
||
body, err := t.restoreIndexClaimBody(c)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
claimed, err := t.loadIndexEffect(c, "restore-claimed")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
finalize, err := t.loadIndexEffect(c, "restore-finalize-intent")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// 已完成 clean 的 effect 是终态:恢复内容与证据都在时,重放不是未决。
|
||
done, err := t.loadIndexEffect(c, "restored")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if gate != nil {
|
||
binding, bindErr := t.restoreMarkerBinding(ctx, c, body)
|
||
if bindErr != nil {
|
||
return bindErr
|
||
}
|
||
status, found, inspErr := gate.InspectRestoreMarker(ctx, binding)
|
||
if inspErr != nil {
|
||
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
|
||
}
|
||
if found {
|
||
// 绑定冲突是类型化分类:任何状态(含已完成 clean 的终态)都必须拒绝。
|
||
// 它不能与"仅禁止第二次 create"的 dispatching-held 混为一谈——后者在
|
||
// 精确 marker 仍在时是允许确认的。
|
||
if status.BindingConflict {
|
||
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
|
||
}
|
||
if done != nil {
|
||
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||
return heldRestoreMarker(status, "已完成 clean 的 effect 台账却不是 marker_confirmed")
|
||
}
|
||
// 终态:内容与世代由下面的 done 分支核对。
|
||
} else {
|
||
switch status.State {
|
||
case libraryrelease.RestoreDispatchPrepared:
|
||
if status.Held {
|
||
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
|
||
}
|
||
if claimed != nil || finalize != nil {
|
||
return heldRestoreMarker(status, "prepared 台账却已有 claimed/finalize 证据")
|
||
}
|
||
observed, obsErr := t.observePlannedIndex(ctx, c)
|
||
if obsErr != nil {
|
||
return obsErr
|
||
}
|
||
if indexBytesEqual(observed, body) {
|
||
return heldRestoreMarker(status, "prepared 台账却已存在目标标记:协议证据矛盾")
|
||
}
|
||
continue
|
||
case libraryrelease.RestoreDispatchDispatching, libraryrelease.RestoreDispatchMarkerConfirmed:
|
||
observed, obsErr := t.observePlannedIndex(ctx, c)
|
||
if obsErr != nil {
|
||
return obsErr
|
||
}
|
||
if indexBytesEqual(observed, body) && observed.SeqNo != nil && observed.PrimaryTerm != nil {
|
||
continue // 只允许原确认分支继续,不重发 create
|
||
}
|
||
return heldRestoreMarker(status, "目标上缺少可归属的精确恢复标记")
|
||
default:
|
||
return heldRestoreMarker(status, "派发台账状态不可识别")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if done != nil {
|
||
observed, err := t.observePlannedIndex(ctx, c)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if done.Receipt == nil || !indexBytesEqual(observed, c.Before) ||
|
||
!sameIndexGeneration(observed, done.Receipt.SeqNo, done.Receipt.PrimaryTerm) {
|
||
return fmt.Errorf("%w: 已标记 restored 的 effect 与目标现状不符", libraryrelease.ErrRestoreDispatchHeld)
|
||
}
|
||
continue
|
||
}
|
||
if intent == nil && claimed == nil && finalize == nil {
|
||
continue
|
||
}
|
||
observed, err := t.observePlannedIndex(ctx, c)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if intent != nil && indexBytesEqual(observed, body) {
|
||
continue // 旧标记已在目标上:按既有确认/CAS 规则处理,不派发新 create
|
||
}
|
||
return fmt.Errorf("%w: 既存恢复意图/证据没有新协议派发台账,无法证明未派发", libraryrelease.ErrRestoreDispatchHeld)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// confirmDispatchedMarker 只在"marker 与冻结内容逐位相同且世代齐全"时记录回执。
|
||
// 它不发送任何请求;确认失败即 held,绝不因为"看起来已经写好了"就继续 clean。
|
||
func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, observed indexWriteGuard) error {
|
||
if t.config.RestoreDispatch == nil || len(c.After) != 0 {
|
||
return nil
|
||
}
|
||
binding, err := t.restoreMarkerBinding(ctx, c, body)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !found {
|
||
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
|
||
}
|
||
if status.BindingConflict {
|
||
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
|
||
}
|
||
if status.Held && status.State == libraryrelease.RestoreDispatchPrepared {
|
||
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
|
||
}
|
||
switch status.State {
|
||
case libraryrelease.RestoreDispatchPrepared:
|
||
// 还没派发:由调用方按 prepared 竞争许可。
|
||
return nil
|
||
case libraryrelease.RestoreDispatchMarkerConfirmed:
|
||
return nil
|
||
case libraryrelease.RestoreDispatchDispatching:
|
||
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
|
||
return heldRestoreMarker(status, "dispatching 结果未知且目标上没有精确标记")
|
||
}
|
||
receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: "created", SeqNo: observed.SeqNo, PrimaryTerm: observed.PrimaryTerm}
|
||
raw, err := json.Marshal(receipt)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw)
|
||
default:
|
||
return heldRestoreMarker(status, "派发台账状态不可识别")
|
||
}
|
||
}
|
||
|
||
// dispatchRestoreMarkerCreate 是恢复期唯一获得发送许可的入口:preflight 与故障钩子
|
||
// 都在 CAS 之前,Acquire 只在 dispatching 明确提交后返回许可。
|
||
func (t *ProductionLibraryTarget) dispatchRestoreMarkerCreate(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (indexReceipt, error) {
|
||
if t.config.RestoreDispatch == nil {
|
||
return indexReceipt{}, errors.New("恢复标记派发协议未接线:拒绝发送")
|
||
}
|
||
binding, err := t.restoreMarkerBinding(ctx, c, body)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
if !found || status.State != libraryrelease.RestoreDispatchPrepared || status.Held {
|
||
return indexReceipt{}, heldRestoreMarker(status, "没有可竞争的 prepared 许可")
|
||
}
|
||
// 控制流故障注入必须在 CAS 之前:拒绝时状态仍是 prepared,可安全重试,零转发。
|
||
if err := t.ds01TestRefusal(c); err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
if err := t.refuseRestoreWrite(c); err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
permit, acquireStatus, err := t.config.RestoreDispatch.AcquireRestoreMarker(ctx, binding)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
if permit == nil {
|
||
return indexReceipt{}, heldRestoreMarker(acquireStatus, "许可竞争未胜出")
|
||
}
|
||
if err := t.ds01TestHoldAfterCAS(ctx, c); err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
receipt, err := t.sendRestoreMarkerOnce(ctx, c, body, binding, permit)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
raw, err := json.Marshal(receipt)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
// 回执来自这次唯一发送:必须落进台账才算确认。写失败时 marker 已在目标上,
|
||
// 下一进程走"精确 marker+世代"确认分支,不会重发 create。
|
||
if err := t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw); err != nil {
|
||
return indexReceipt{}, fmt.Errorf("恢复标记已发送但确认未落库:%w", err)
|
||
}
|
||
return receipt, nil
|
||
}
|
||
|
||
// sendRestoreMarkerOnce 是唯一的真实网络发送点:许可消费一次、请求不可被传输层重放。
|
||
func (t *ProductionLibraryTarget) sendRestoreMarkerOnce(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage,
|
||
binding libraryrelease.RestoreMarkerBinding, permit *libraryrelease.RestoreSendPermit) (indexReceipt, error) {
|
||
digest, err := binding.Digest()
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
if err := permit.Consume(binding.Key, digest); err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
method, path, payload, err := t.indexMutation(c, body, nil, nil, true)
|
||
if err != nil {
|
||
return indexReceipt{}, err
|
||
}
|
||
var receipt indexReceipt
|
||
if err := t.search.requestJSONOnce(ctx, method, path, payload, &receipt); err != nil {
|
||
return receipt, fmt.Errorf("restore marker outcome unknown: %w", err)
|
||
}
|
||
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
|
||
return receipt, err
|
||
}
|
||
return receipt, nil
|
||
}
|
||
|
||
// verifyRestoreEffectsClosed 是 Restore 收尾前的最后一道检查:以原 window 的权威台账
|
||
// 全集为准(跨 plan),每个记录都必须是 marker_confirmed,且对应的 restored 文件证据
|
||
// 存在。prepared/dispatching、计划外 effect、换 plan 别名、缺 restored 一律拒绝。
|
||
func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context) error {
|
||
if t.config.RestoreDispatch != nil {
|
||
records, err := t.config.RestoreDispatch.UnresolvedRestoreDispatches(ctx,
|
||
t.config.RestoreTargetIdentity, t.config.RestoreOperationID, t.config.RestoreWindowJobID)
|
||
if err != nil {
|
||
return fmt.Errorf("恢复收尾读不到权威派发台账:%w", err)
|
||
}
|
||
for _, record := range records {
|
||
if record.PlanDigest != t.activePlan.Digest {
|
||
return fmt.Errorf("%w: 收尾时原 window 仍有另一份计划的派发记录(plan=%s)",
|
||
libraryrelease.ErrRestoreDispatchHeld, record.PlanDigest)
|
||
}
|
||
if record.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||
return fmt.Errorf("%w: 收尾前派发台账仍有未决 effect(index=%s doc=%s state=%s)",
|
||
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID, record.State)
|
||
}
|
||
c, ok := t.planEffect(record.Index, record.DocumentID)
|
||
if !ok {
|
||
return fmt.Errorf("%w: 收尾时存在计划外派发记录(index=%s doc=%s)",
|
||
libraryrelease.ErrRestoreDispatchHeld, record.Index, record.DocumentID)
|
||
}
|
||
done, err := t.loadIndexEffect(c, "restored")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if done == nil {
|
||
return fmt.Errorf("%w: 收尾前仍有未 clean 的 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||
}
|
||
// 收尾同样是**使用点**:绑定冲突即使状态看起来是 marker_confirmed 也必须拒绝。
|
||
body, err := t.restoreIndexClaimBody(c)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
binding, err := t.restoreMarkerBinding(ctx, c, body)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||
if err != nil {
|
||
return fmt.Errorf("恢复收尾读不到派发台账:%w", err)
|
||
}
|
||
if !found {
|
||
return fmt.Errorf("%w: 收尾时 effect %s 的派发台账消失了", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||
}
|
||
if status.BindingConflict {
|
||
return heldRestoreMarker(status, "收尾时派发台账的不可变绑定与本次计划冲突")
|
||
}
|
||
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||
return fmt.Errorf("%w: 收尾时 effect %s 的派发状态是 %s", libraryrelease.ErrRestoreDispatchHeld, c.ID, status.State)
|
||
}
|
||
}
|
||
}
|
||
for _, c := range t.activePlan.Index {
|
||
if len(c.After) != 0 {
|
||
continue
|
||
}
|
||
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if intent == nil {
|
||
continue
|
||
}
|
||
done, err := t.loadIndexEffect(c, "restored")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if done == nil {
|
||
return fmt.Errorf("%w: 恢复收尾前仍有未决 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ds01TestRefusal 是正式 worker 全链验收的最小测试接缝:只有显式隔离运行
|
||
// (KG_LIBRARY_TEST_ISOLATED=1)且指定了 effect 时才在 CAS 之前拒绝一次;生产默认
|
||
// 全关,且每个进程只触发一次,重启后的 worker 不会重复拒绝。
|
||
// ds01TestHoldAfterCAS 是 N1 的最小测试接缝:CAS 已提交、许可已拿到,但在真实发送
|
||
// 之前挂起,让验收进程能观察到 durable dispatching 后终止执行者。生产默认全关。
|
||
func (t *ProductionLibraryTarget) ds01TestHoldAfterCAS(ctx context.Context, c libraryrelease.IndexWrite) error {
|
||
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" || t.restoreHoldUsed {
|
||
return nil
|
||
}
|
||
spec := os.Getenv("KG_DS01_TEST_HOLD_AFTER_CAS")
|
||
if spec == "" || spec != c.Index+"/"+c.ID {
|
||
return nil
|
||
}
|
||
t.restoreHoldUsed = true
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
case <-time.After(90 * time.Second):
|
||
return errors.New("injected DS01 hold after CAS expired")
|
||
}
|
||
}
|
||
|
||
func (t *ProductionLibraryTarget) ds01TestRefusal(c libraryrelease.IndexWrite) error {
|
||
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" || t.restoreRefusalUsed {
|
||
return nil
|
||
}
|
||
spec := os.Getenv("KG_DS01_TEST_REFUSE_MARKER")
|
||
if spec == "" || spec != c.Index+"/"+c.ID {
|
||
return nil
|
||
}
|
||
t.restoreRefusalUsed = true
|
||
return errors.New("injected DS01 marker refusal before dispatch CAS")
|
||
}
|