fix(library): 收紧 DS01 派发租约/投递/治理身份与绑定冲突
F2.a:Acquire/Confirm 在取得 effect 行锁后用数据库时钟重核 job/attempt/ 恢复投递租约,等锁跨过到期零许可且状态保持 prepared。 F2.b:同一 CAS 事务锁 recovery queue 与 operation,复核 window/scope、 idempotency、owner/attempt/lease,并按原 window 锚点重算恢复请求摘要。 F3.identity:激活证据 v3 携带 PG 集群 system_identifier + 库 OID、完整构建 SHA 与协议批次;激活与使用点都现场重读并拒绝同名异域/副本/unknown/不兼容 SHA。 F4.binding:区分绑定冲突与仅禁止重发的 Held,准入/确认/续作/收尾在所有状态 拒绝冲突,合法 dispatching + 精确 marker 仍可确认。
This commit is contained in:
@@ -1,5 +1,29 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(DS01 收口:CAS 点实时租约、恢复投递归属、治理权威域与绑定冲突分类)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **F2.a CAS 点实时租约**:`Acquire` 在取得 effect 行锁(最后一个可能阻塞的锁)之后、
|
||||
状态 CAS 之前,按数据库 `clock_timestamp()` 重核 job/attempt/恢复投递租约;`Confirm`
|
||||
同样先锁 effect 行再核。等锁跨过到期一律零许可、状态保持 `prepared`。新增真 PG
|
||||
反例:等 effect 行锁期间租约过期。
|
||||
- **F2.b 恢复投递同事务绑定**:同一短事务内 `FOR UPDATE` 锁 queue 与 recovery operation,
|
||||
复核 queue id/type/state/payload 的 window+scope、idempotency key、当前 owner/attempt/
|
||||
lease,并按原 window 的不可变锚点重算 `LibraryRecoveryDigest`,与 operation 的
|
||||
`request_digest` 逐位比较;错 queue/window/digest/owner/过期/错备份/错 plan 全部零许可。
|
||||
投递 owner 合成规则收敛到 `governance.LibraryDeliveryOwner` 一份。
|
||||
- **F3.identity 治理权威域与构建身份**:激活证据升至 v3,必须携带 PG 集群
|
||||
`system_identifier` + 库 OID、完整 40 位构建 SHA 与协议兼容批次;激活与每次使用点都
|
||||
现场重读 `pg_control_system()`/`pg_database.oid`/`pg_is_in_recovery()` 并比对,拒绝
|
||||
同名异域、副本、unknown 构建、同 display 不同不兼容 SHA 与错目标指纹。同一兼容批次
|
||||
内的不同完整 SHA 记为 `compatible_upgrade`,不静默等同。
|
||||
- **F4.binding 绑定冲突类型化**:`RestoreDispatchStatus.BindingConflict` 与"仅禁止重发"
|
||||
的 `Held` 分开;准入、确认、既有意图续作与收尾在 prepared/dispatching/marker_confirmed/
|
||||
done 任一状态都拒绝绑定冲突;合法 dispatching + 精确 marker 仍可确认一次。
|
||||
- 真 PG/adapter 反例:effect 锁等待过期、恢复投递九类错配、错治理域/错批次/unknown、
|
||||
四状态绑定冲突与合法 marker 正例。
|
||||
|
||||
## 开发快照(DS01 激活证据 v2:只认真实维护 journal)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -135,9 +135,7 @@ func libraryQueueOwner(workerID string, job jobs.Job, libraryJobID string) strin
|
||||
if queueID == "" {
|
||||
queueID = libraryJobID
|
||||
}
|
||||
worker := workerID
|
||||
if worker == "" {
|
||||
worker = "worker"
|
||||
}
|
||||
return fmt.Sprintf("%s#%s#%d", worker, queueID, job.Attempt)
|
||||
// 合成规则只有一份(governance.LibraryDeliveryOwner):派发台账要按同一格式核
|
||||
// "这次许可属于哪次真实投递"。
|
||||
return governance.LibraryDeliveryOwner(workerID, queueID, job.Attempt)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package governance
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The recovery request has its own queue identity and stable operation. It
|
||||
@@ -18,3 +20,33 @@ func LibraryRecoveryDigest(scope, queue, window, operation, target, source, back
|
||||
"target_id": target, "source_snapshot": source, "backup_snapshot": backup, "writeset_digest": plan,
|
||||
})
|
||||
}
|
||||
|
||||
// LibraryDeliveryOwner 是队列投递 + 业务 attempt 的合成 owner:
|
||||
//
|
||||
// <worker>#<queue job>#<queue attempt>
|
||||
//
|
||||
// 它就是 library_attempts.owner,也决定了"这次派发属于哪一次真实投递"。worker 与
|
||||
// 派发台账必须用同一份格式,否则"当前 owner"会变成两处各自解释的字符串。
|
||||
func LibraryDeliveryOwner(workerID, queueID string, attempt int) string {
|
||||
if queueID == "" {
|
||||
queueID = "-"
|
||||
}
|
||||
if workerID == "" {
|
||||
workerID = "worker"
|
||||
}
|
||||
return fmt.Sprintf("%s#%s#%d", workerID, queueID, attempt)
|
||||
}
|
||||
|
||||
// ParseLibraryDeliveryOwner 把 owner 拆回 (worker, ok);queue/attempt 必须逐位相同,
|
||||
// 不能靠后缀匹配把别的投递混进来。
|
||||
func ParseLibraryDeliveryOwner(owner, queueID string, attempt int) (string, bool) {
|
||||
suffix := fmt.Sprintf("#%s#%d", queueID, attempt)
|
||||
if queueID == "" || !strings.HasSuffix(owner, suffix) {
|
||||
return "", false
|
||||
}
|
||||
worker := strings.TrimSuffix(owner, suffix)
|
||||
if worker == "" || strings.Contains(worker, "#") {
|
||||
return "", false
|
||||
}
|
||||
return worker, true
|
||||
}
|
||||
|
||||
@@ -126,10 +126,17 @@ func (b RestoreMarkerBinding) Digest() (string, error) {
|
||||
}
|
||||
|
||||
// RestoreDispatchStatus 是纯读分类结果。Held 为真时永远不能发送。
|
||||
//
|
||||
// BindingConflict 是与 Held 不同的**类型化**分类:Held 只说"这次不能再 create"
|
||||
// (prepared 冲突、dispatching 结果未知、marker 已确认都可能是合法的不可重发),
|
||||
// 而 BindingConflict 说明台账行的不可变绑定与本次绑定不是同一个 effect。绑定冲突
|
||||
// 在**任何状态**下都必须拒绝,包括已完成 clean 的 marker_confirmed;否则"只允许原
|
||||
// 确认分支继续"会变成放行一份不属于本计划的写入。
|
||||
type RestoreDispatchStatus struct {
|
||||
State string `json:"state"`
|
||||
Held bool `json:"held"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
State string `json:"state"`
|
||||
Held bool `json:"held"`
|
||||
BindingConflict bool `json:"binding_conflict,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// RestoreSendPermit 是"这一次调用、这一个 effect、这一次发送"的许可。
|
||||
|
||||
@@ -29,7 +29,24 @@ import (
|
||||
|
||||
const RestoreDispatchActivationMaxAge = 2 * time.Hour
|
||||
|
||||
const RestoreDispatchActivationProtocolVersion = 2
|
||||
// 证据文档 schema 版本与语义兼容批次是两件事:
|
||||
// - ProtocolVersion:这份 JSON 的字段形状(解析契约);
|
||||
// - ProtocolEpoch:派发协议语义批次;只有同一批次的构建才允许共享激活证据
|
||||
// (例如同一批次的兼容重建/打包),不同批次必须重新登记。
|
||||
//
|
||||
// 构建身份用完整 40 位 SHA,不用 display:开发号会随提交数变化,两个不同二进制
|
||||
// 完全可能都叫 0.1.322。
|
||||
const RestoreDispatchActivationProtocolVersion = 3
|
||||
|
||||
const RestoreDispatchProtocolEpoch = 3
|
||||
|
||||
// RestoreDispatchGovernanceAuthority 是治理库的**权威域**身份:系统标识(集群)
|
||||
// 加库 OID。只看 current_database() 无法区分"同名但另一个集群/另一份克隆"。
|
||||
// primary 由 Store 现场读 pg_is_in_recovery() 得出,不采信调用方声明。
|
||||
type RestoreDispatchGovernanceAuthority struct {
|
||||
SystemIdentifier string `json:"system_identifier"`
|
||||
DatabaseOID string `json:"database_oid"`
|
||||
}
|
||||
|
||||
// RestoreDispatchWriterDrain 是调用方对"旧 writer 集合"的声明:只给身份,不给结论。
|
||||
// InstanceID 必须等于 journal 里登记的不可变 writer 身份(容器/进程 ID)。
|
||||
@@ -41,27 +58,35 @@ type RestoreDispatchWriterDrain struct {
|
||||
|
||||
type RestoreDispatchActivationEvidence struct {
|
||||
ProtocolVersion int `json:"protocol_version"`
|
||||
ProtocolEpoch int `json:"protocol_epoch"`
|
||||
TargetIdentity string `json:"target_identity"`
|
||||
// TargetFingerprint 是维护窗口/部署目标指纹(journal 与治理库维护行记录的那一个);
|
||||
// RestoreTargetFingerprint 是激活时对恢复目标内容/索引/桶的实时观察值。两者是不同层,
|
||||
// 不能相互替代:前者绑定"动的是哪个实例",后者绑定"恢复要写的那个目标身份"。
|
||||
TargetFingerprint string `json:"target_fingerprint"`
|
||||
RestoreTargetFingerprint string `json:"restore_target_fingerprint"`
|
||||
TargetInstance string `json:"target_instance"`
|
||||
GovernanceDatabase string `json:"governance_database"`
|
||||
ProtocolBuild string `json:"protocol_build"`
|
||||
AttestedBy string `json:"attested_by"`
|
||||
AttestedAt time.Time `json:"attested_at"`
|
||||
MaintenanceJob string `json:"maintenance_job"`
|
||||
MaintenanceJournal string `json:"maintenance_journal"`
|
||||
MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"`
|
||||
Writers []RestoreDispatchWriterDrain `json:"writers"`
|
||||
TargetFingerprint string `json:"target_fingerprint"`
|
||||
RestoreTargetFingerprint string `json:"restore_target_fingerprint"`
|
||||
TargetInstance string `json:"target_instance"`
|
||||
GovernanceDatabase string `json:"governance_database"`
|
||||
GovernanceAuthority RestoreDispatchGovernanceAuthority `json:"governance_authority"`
|
||||
ProtocolBuild string `json:"protocol_build"`
|
||||
ProtocolBuildSHA string `json:"protocol_build_sha"`
|
||||
AttestedBy string `json:"attested_by"`
|
||||
AttestedAt time.Time `json:"attested_at"`
|
||||
MaintenanceJob string `json:"maintenance_job"`
|
||||
MaintenanceJournal string `json:"maintenance_journal"`
|
||||
MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"`
|
||||
Writers []RestoreDispatchWriterDrain `json:"writers"`
|
||||
}
|
||||
|
||||
func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.Duration) error {
|
||||
if e.ProtocolVersion != RestoreDispatchActivationProtocolVersion {
|
||||
return fmt.Errorf("restore dispatch activation protocol version %d is not supported", e.ProtocolVersion)
|
||||
}
|
||||
// 兼容批次不同 = 派发语义不兼容:即使 display 相同也不能共用激活证据。
|
||||
if e.ProtocolEpoch != RestoreDispatchProtocolEpoch {
|
||||
return fmt.Errorf("restore dispatch activation protocol epoch %d is not compatible with %d",
|
||||
e.ProtocolEpoch, RestoreDispatchProtocolEpoch)
|
||||
}
|
||||
for name, value := range map[string]string{
|
||||
"target identity": e.TargetIdentity, "target fingerprint": e.TargetFingerprint,
|
||||
"restore target fingerprint": e.RestoreTargetFingerprint,
|
||||
@@ -74,6 +99,14 @@ func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.D
|
||||
return fmt.Errorf("restore dispatch activation evidence lacks %s", name)
|
||||
}
|
||||
}
|
||||
// 构建身份必须是完整 SHA;display 可以随提交数变化,不能作为身份。
|
||||
if !isFullSHA(e.ProtocolBuildSHA) {
|
||||
return errors.New("restore dispatch activation build identity is not a full 40-character commit sha")
|
||||
}
|
||||
// 治理权威域必须是可读回的 PG 事实形状:集群系统标识 + 库 OID,两个都是十进制数字。
|
||||
if !isDecimal(e.GovernanceAuthority.SystemIdentifier) || !isDecimal(e.GovernanceAuthority.DatabaseOID) {
|
||||
return errors.New("restore dispatch activation governance authority is not a cluster/database identity")
|
||||
}
|
||||
if !isHex64(e.TargetFingerprint) || !isHex64(e.RestoreTargetFingerprint) {
|
||||
return errors.New("restore dispatch activation target fingerprints are not sha256 hex strings")
|
||||
}
|
||||
@@ -161,6 +194,26 @@ func isHex64(value string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isFullSHA(value string) bool {
|
||||
if len(value) != 40 || strings.ToLower(value) != value {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(value)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isDecimal(value string) bool {
|
||||
if value == "" || len(value) > 20 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// validActivationID 与维护 journal 的作业/服务 ID 同一字符集:证据里出现的身份
|
||||
// 必须能原样出现在 journal 中,不能靠空格/路径字符制造"看起来不同"的别名。
|
||||
func validActivationID(value string) bool {
|
||||
|
||||
@@ -103,11 +103,17 @@ WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetI
|
||||
// lockRestoreDispatchActivation 在**同一个 CAS 事务**里锁激活行。撤销是同一行的
|
||||
// UPDATE:撤销先完成 ⇒ 这里看到 revoked_at;CAS 先取锁 ⇒ 撤销必须等 CAS 提交,
|
||||
// 之后按"已派发"的窄边界处理。两者只能有一个线性化顺序。
|
||||
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity string) error {
|
||||
//
|
||||
// 这里同时是**使用点**的权威域/构建身份重核:激活行里的 verified 事实必须与当前
|
||||
// 连接的 PG 权威域(集群 system_identifier + 库 OID + primary)一致;运行构建必须
|
||||
// 有完整 SHA,且与激活批次兼容;恢复目标指纹必须等于本次观察值。同名库、副本、
|
||||
// unknown 构建或换目标都在这里失败关闭,而不是只在激活那一刻检查。
|
||||
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity, targetFingerprint string) error {
|
||||
var revoked *time.Time
|
||||
var evidenceRaw, rowFingerprint string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT revoked_at FROM library_index_restore_dispatch_activation
|
||||
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked)
|
||||
SELECT revoked_at, evidence::text, target_fingerprint FROM library_index_restore_dispatch_activation
|
||||
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked, &evidenceRaw, &rowFingerprint)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
|
||||
}
|
||||
@@ -117,6 +123,39 @@ WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Sca
|
||||
if revoked != nil {
|
||||
return fmt.Errorf("%w(目标 %s 的激活已于 %s 撤销)", ErrRestoreDispatchNotActivated, targetIdentity, revoked.UTC().Format(time.RFC3339))
|
||||
}
|
||||
if rowFingerprint != targetFingerprint {
|
||||
return fmt.Errorf("%w(目标 %s 的激活绑定指纹 %s,本次恢复目标指纹 %s)",
|
||||
ErrRestoreDispatchNotActivated, targetIdentity, rowFingerprint, targetFingerprint)
|
||||
}
|
||||
var record restoreDispatchActivationRecord
|
||||
if err := json.Unmarshal([]byte(evidenceRaw), &record); err != nil {
|
||||
return fmt.Errorf("parse restore dispatch activation record: %w", err)
|
||||
}
|
||||
verified := record.Verified
|
||||
authority, primary, _, err := observedGovernanceAuthority(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if verified.GovernanceAuthority.SystemIdentifier == "" || verified.GovernanceAuthority.DatabaseOID == "" {
|
||||
return fmt.Errorf("%w(目标 %s 的激活登记没有治理权威域身份:拒绝仅凭库名启用)",
|
||||
ErrRestoreDispatchNotActivated, targetIdentity)
|
||||
}
|
||||
if authority != verified.GovernanceAuthority {
|
||||
return fmt.Errorf("%w(目标 %s 的激活登记属于治理域 %s/%s,当前是 %s/%s)",
|
||||
ErrRestoreDispatchNotActivated, targetIdentity, verified.GovernanceAuthority.SystemIdentifier,
|
||||
verified.GovernanceAuthority.DatabaseOID, authority.SystemIdentifier, authority.DatabaseOID)
|
||||
}
|
||||
if !primary {
|
||||
return fmt.Errorf("%w(治理库当前处于恢复/副本角色,拒绝授予派发许可)", ErrRestoreDispatchNotActivated)
|
||||
}
|
||||
buildEvidence := libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolEpoch: verified.ProtocolEpoch,
|
||||
ProtocolBuild: verified.ProtocolBuild,
|
||||
ProtocolBuildSHA: verified.ProtocolBuildSHA,
|
||||
}
|
||||
if _, _, _, err := verifyRestoreDispatchBuild(buildEvidence); err != nil {
|
||||
return fmt.Errorf("%w(目标 %s:%v)", ErrRestoreDispatchNotActivated, targetIdentity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -143,7 +182,7 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
|
||||
}
|
||||
if stored != digest {
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
|
||||
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
|
||||
}
|
||||
switch state {
|
||||
@@ -223,7 +262,7 @@ func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil {
|
||||
@@ -270,7 +309,7 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
// 撤销与 CAS 在同一事务线性化:这里拿激活行锁,撤销的 UPDATE 必须排队。
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
// 当前代次必须属于这个原 window 作业,且租户/epoch/租约都仍有效。lease 用
|
||||
@@ -322,13 +361,24 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err)
|
||||
}
|
||||
if stored != digest {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
|
||||
Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
// effect 行锁是本次 CAS 前最后一个可能阻塞的锁。job/attempt/恢复投递的租约可能
|
||||
// 正好在等它的时候过期;前面那几次检查发生在拿锁之前,不能充当 CAS 点的判据。
|
||||
// 这些行锁都已被本事务持有,这里只重读、不改变锁序。
|
||||
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
|
||||
var held *restoreDispatchLeaseExpired
|
||||
if errors.As(err, &held) {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true,
|
||||
Reason: held.Error()}, fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, held.Error())
|
||||
}
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
|
||||
var raw [16]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
@@ -384,7 +434,7 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
|
||||
return err
|
||||
}
|
||||
var epoch int64
|
||||
@@ -404,6 +454,23 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper
|
||||
if attemptJob != b.Key.WindowJobID || owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
|
||||
}
|
||||
// 与 Acquire 同一个顺序:确认前的最后一把锁也要先拿到,再按数据库实时时钟重核
|
||||
// 租约。否则"确认"可以在等锁跨过到期之后仍落库,把一次过期 owner 的回执写成事实。
|
||||
var effectID int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT id FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&effectID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errors.New("restore marker confirmation did not match a dispatch record")
|
||||
}
|
||||
return fmt.Errorf("lock restore marker for confirmation: %w", err)
|
||||
}
|
||||
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
|
||||
return fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=clock_timestamp(), updated_at=clock_timestamp()
|
||||
@@ -513,23 +580,8 @@ FROM library_instance_maintenance WHERE instance=$1 FOR UPDATE`, b.TargetInstanc
|
||||
}
|
||||
|
||||
if b.RecoveryJobID != "" {
|
||||
var queueState string
|
||||
var lease string
|
||||
if err := tx.QueryRow(ctx, `SELECT state, COALESCE(lease_owner,'') FROM jobs WHERE id=$1`, b.RecoveryJobID).
|
||||
Scan(&queueState, &lease); err != nil {
|
||||
return fmt.Errorf("read recovery queue job: %w", err)
|
||||
}
|
||||
if queueState != "running" || lease == "" {
|
||||
return fmt.Errorf("recovery queue job %s is not a live delivery", b.RecoveryJobID)
|
||||
}
|
||||
var recoveryKind, recoveryScope, recoveryBackup string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT kind, scope, target_snapshot_id FROM library_operations WHERE id=$1`, b.RecoveryOperationID).
|
||||
Scan(&recoveryKind, &recoveryScope, &recoveryBackup); err != nil {
|
||||
return fmt.Errorf("read recovery operation: %w", err)
|
||||
}
|
||||
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope || recoveryBackup != b.BackupSnapshot {
|
||||
return fmt.Errorf("recovery operation %s is not bound to this window and backup", b.RecoveryOperationID)
|
||||
if err := lockAndVerifyRecoveryDelivery(ctx, tx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -2,10 +2,12 @@ package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -42,8 +44,51 @@ type restoreDispatchVerifiedView struct {
|
||||
Quiesce restoreDispatchQuiesceView `json:"quiesce"`
|
||||
Writers []restoreDispatchWriterView `json:"writers"`
|
||||
GovernanceDatabase string `json:"governance_database"`
|
||||
ProtocolBuild string `json:"protocol_build"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
// 治理权威域是现场读回的 PG 事实:集群 system_identifier + 库 OID + 是否 primary。
|
||||
// 激活与每次使用点都重读比较,库名只作可读标签。
|
||||
GovernanceAuthority restoreDispatchAuthorityView `json:"governance_authority"`
|
||||
GovernancePrimary bool `json:"governance_primary"`
|
||||
ProtocolBuild string `json:"protocol_build"`
|
||||
ProtocolBuildSHA string `json:"protocol_build_sha"`
|
||||
ProtocolEpoch int `json:"protocol_epoch"`
|
||||
CompatibleUpgrade bool `json:"compatible_upgrade"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
|
||||
// restoreDispatchAuthorityView 只放可比较的权威域字段;不含密钥/连接串。
|
||||
type restoreDispatchAuthorityView struct {
|
||||
SystemIdentifier string `json:"system_identifier"`
|
||||
DatabaseOID string `json:"database_oid"`
|
||||
}
|
||||
|
||||
// lockRestoreDispatchActivationVerification 是使用点重核所需的最小事实。
|
||||
type restoreDispatchActivationVerification struct {
|
||||
Authority restoreDispatchAuthorityView
|
||||
Primary bool
|
||||
ProtocolBuild string
|
||||
ProtocolBuildSHA string
|
||||
ProtocolEpoch int
|
||||
TargetFingerprint string
|
||||
}
|
||||
|
||||
// observedGovernanceAuthority 读回当前连接的治理库权威域。pg_control_system() 的
|
||||
// system_identifier 是集群级、跨备份/克隆仍不同的身份;库 OID 在同一集群内区分同名库。
|
||||
// 没有权限读它时返回错误(能力失败关闭),绝不退回"库名相同就算同一个库"。
|
||||
func observedGovernanceAuthority(ctx context.Context, q interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
}) (restoreDispatchAuthorityView, bool, string, error) {
|
||||
var authority restoreDispatchAuthorityView
|
||||
var database string
|
||||
var inRecovery bool
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT current_database(),
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
|
||||
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery)
|
||||
if err != nil {
|
||||
return authority, false, "", fmt.Errorf("observe governance authority: %w", err)
|
||||
}
|
||||
return authority, !inRecovery, database, nil
|
||||
}
|
||||
|
||||
type restoreDispatchGateView struct {
|
||||
@@ -128,21 +173,26 @@ func (s *Store) verifyRestoreDispatchEvidence(ctx context.Context, evidence libr
|
||||
if evidence.AttestedAt.Before(journal.Quiesce.PostStopAt) {
|
||||
return verified, errors.New("restore dispatch attestation predates the maintenance stop observation")
|
||||
}
|
||||
var database string
|
||||
if err := s.db.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
|
||||
return verified, fmt.Errorf("read governance database identity: %w", err)
|
||||
authority, primary, database, err := observedGovernanceAuthority(ctx, s.db)
|
||||
if err != nil {
|
||||
return verified, err
|
||||
}
|
||||
if database == "" || database != evidence.GovernanceDatabase {
|
||||
if authority != (restoreDispatchAuthorityView{SystemIdentifier: evidence.GovernanceAuthority.SystemIdentifier,
|
||||
DatabaseOID: evidence.GovernanceAuthority.DatabaseOID}) {
|
||||
return verified, fmt.Errorf("restore dispatch activation names governance authority %s/%s, live is %s/%s",
|
||||
evidence.GovernanceAuthority.SystemIdentifier, evidence.GovernanceAuthority.DatabaseOID,
|
||||
authority.SystemIdentifier, authority.DatabaseOID)
|
||||
}
|
||||
if !primary {
|
||||
return verified, errors.New("restore dispatch activation requires the governance database to be primary, not a replica")
|
||||
}
|
||||
if database != evidence.GovernanceDatabase {
|
||||
return verified, fmt.Errorf("restore dispatch activation evidence names governance database %q, live is %q",
|
||||
evidence.GovernanceDatabase, database)
|
||||
}
|
||||
build := version.DisplayVersion()
|
||||
if !version.HasInjection() || build == "" || build == "dev" {
|
||||
return verified, errors.New("restore dispatch activation requires a build with injected version identity")
|
||||
}
|
||||
if build != evidence.ProtocolBuild {
|
||||
return verified, fmt.Errorf("restore dispatch activation evidence was produced by build %q, running build is %q",
|
||||
evidence.ProtocolBuild, build)
|
||||
build, buildSHA, compatible, err := verifyRestoreDispatchBuild(evidence)
|
||||
if err != nil {
|
||||
return verified, err
|
||||
}
|
||||
// 维护窗口指纹与恢复目标指纹是两层:前者已与 journal/维护行比对,后者必须等于
|
||||
// 激活方刚刚对恢复目标做的那次只读观察。用一个值同时顶替两者会掩盖"配置指向
|
||||
@@ -230,7 +280,52 @@ FROM library_instance_maintenance WHERE instance=$1`, evidence.TargetInstance).S
|
||||
Quiesce: restoreDispatchQuiesceView{Depth: journal.Quiesce.Depth, PendingOps: journal.Quiesce.PendingOps,
|
||||
Unsettled: journal.Quiesce.Unsettled, Digest: journal.Quiesce.Digest, StableMS: journal.Quiesce.StableMS,
|
||||
Samples: journal.Quiesce.Samples, ProvenAt: journal.Quiesce.ProvenAt.UTC(), PostStopAt: journal.Quiesce.PostStopAt.UTC()},
|
||||
Writers: verifiedWriters, GovernanceDatabase: database, ProtocolBuild: build, ObservedAt: now.UTC(),
|
||||
Writers: verifiedWriters, GovernanceDatabase: database,
|
||||
GovernanceAuthority: authority, GovernancePrimary: primary,
|
||||
ProtocolBuild: build, ProtocolBuildSHA: buildSHA,
|
||||
ProtocolEpoch: evidence.ProtocolEpoch, CompatibleUpgrade: compatible,
|
||||
ObservedAt: now.UTC(),
|
||||
}
|
||||
return verified, nil
|
||||
}
|
||||
|
||||
// verifyRestoreDispatchBuild 是构建身份的兼容策略:
|
||||
// - 运行中的二进制必须有注入的**完整 40 位 SHA**,否则 unknown —— 拒绝;
|
||||
// - 证据的构建 SHA 也必须是完整 SHA;
|
||||
// - 同一 SHA ⇒ 必须与运行 display 相符(防止同一二进制自称别的版本);
|
||||
// - 不同 SHA ⇒ 只有 ProtocolEpoch 相同(已在 Validate 里核过)才算兼容重建,
|
||||
// 并在台账里显式记 compatible_upgrade=true,不静默等同。
|
||||
func verifyRestoreDispatchBuild(evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string, bool, error) {
|
||||
build := version.DisplayVersion()
|
||||
runningSHA := version.CommitSHA
|
||||
if !version.HasInjection() || build == "" || build == "dev" || !isFullHexSHA(runningSHA) {
|
||||
return "", "", false, errors.New("restore dispatch activation requires a build with an injected full commit sha")
|
||||
}
|
||||
if !isFullHexSHA(evidence.ProtocolBuildSHA) {
|
||||
return "", "", false, errors.New("restore dispatch activation evidence carries no full build sha")
|
||||
}
|
||||
if evidence.ProtocolEpoch != libraryrelease.RestoreDispatchProtocolEpoch {
|
||||
return "", "", false, fmt.Errorf("restore dispatch activation build epoch %d is incompatible with %d",
|
||||
evidence.ProtocolEpoch, libraryrelease.RestoreDispatchProtocolEpoch)
|
||||
}
|
||||
if evidence.ProtocolBuildSHA == runningSHA {
|
||||
if evidence.ProtocolBuild != build {
|
||||
return "", "", false, fmt.Errorf("restore dispatch activation claims build %q for the running commit, display is %q",
|
||||
evidence.ProtocolBuild, build)
|
||||
}
|
||||
return build, runningSHA, false, nil
|
||||
}
|
||||
// 兼容批次内的不同 SHA:允许,但必须记录成"升级",不能伪装成同一个二进制。
|
||||
if strings.TrimSpace(evidence.ProtocolBuild) == "" {
|
||||
return "", "", false, errors.New("restore dispatch activation compatible build has no display identity")
|
||||
}
|
||||
return build, runningSHA, true, nil
|
||||
}
|
||||
|
||||
func isFullHexSHA(value string) bool {
|
||||
if len(value) != 40 || strings.ToLower(value) != value {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(value)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// DS01-F2.a:CAS 点的实时租约判据。
|
||||
//
|
||||
// 授权事务里的前几次租约检查发生在**拿 effect 行锁之前**;等锁可以跨过租约到期。
|
||||
// 因此拿到最后一个可能阻塞的锁之后必须按数据库时钟重读一次。这里的行都已经在
|
||||
// 本事务里持有锁,重读不会新增等待,也不改变 job→operation→maintenance→attempt→effect
|
||||
// 的锁序。
|
||||
type restoreDispatchLeaseExpired struct {
|
||||
reason string
|
||||
}
|
||||
|
||||
func (e *restoreDispatchLeaseExpired) Error() string { return e.reason }
|
||||
|
||||
func restoreDispatchLeaseLive(ctx context.Context, tx pgx.Tx, until *time.Time) (bool, error) {
|
||||
var live bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, until).Scan(&live); err != nil {
|
||||
return false, fmt.Errorf("read lease clock: %w", err)
|
||||
}
|
||||
return live, nil
|
||||
}
|
||||
|
||||
func recheckRestoreDispatchLeases(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
||||
var owner string
|
||||
var epoch int64
|
||||
var leaseUntil *time.Time
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT lease_owner, attempt_epoch, lease_until FROM library_jobs WHERE id=$1`,
|
||||
b.Key.WindowJobID).Scan(&owner, &epoch, &leaseUntil)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("window job %s disappeared", b.Key.WindowJobID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck window job lease: %w", err)
|
||||
}
|
||||
if owner != b.Owner || epoch != b.Epoch {
|
||||
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
|
||||
"window job lease now belongs to %s#%d, not %s#%d", owner, epoch, b.Owner, b.Epoch)}
|
||||
}
|
||||
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !live {
|
||||
return &restoreDispatchLeaseExpired{reason: "window job lease expired while waiting for the dispatch CAS"}
|
||||
}
|
||||
|
||||
var attemptOwner string
|
||||
var attemptEpoch int64
|
||||
var attemptLease *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT owner, epoch, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
|
||||
&attemptOwner, &attemptEpoch, &attemptLease)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf("no active attempt for operation %s", b.Key.OperationID)}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck active attempt lease: %w", err)
|
||||
}
|
||||
if attemptOwner != b.Owner || attemptEpoch != b.Epoch {
|
||||
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
|
||||
"active attempt is now %s#%d, not %s#%d", attemptOwner, attemptEpoch, b.Owner, b.Epoch)}
|
||||
}
|
||||
live, err = restoreDispatchLeaseLive(ctx, tx, attemptLease)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !live {
|
||||
return &restoreDispatchLeaseExpired{reason: "attempt lease expired while waiting for the dispatch CAS"}
|
||||
}
|
||||
|
||||
if b.RecoveryJobID != "" {
|
||||
var deliveryOwner string
|
||||
var deliveryAttempt int
|
||||
var deliveryLease *time.Time
|
||||
err = tx.QueryRow(ctx, `SELECT lease_owner, attempt, lease_until FROM jobs WHERE id=$1`,
|
||||
b.RecoveryJobID).Scan(&deliveryOwner, &deliveryAttempt, &deliveryLease)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("recovery queue job %s disappeared", b.RecoveryJobID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck recovery queue lease: %w", err)
|
||||
}
|
||||
worker, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, deliveryAttempt)
|
||||
if !ok || worker != deliveryOwner {
|
||||
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
|
||||
"recovery delivery %s attempt %d is not owned by %s", b.RecoveryJobID, deliveryAttempt, b.Owner)}
|
||||
}
|
||||
live, err = restoreDispatchLeaseLive(ctx, tx, deliveryLease)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !live {
|
||||
return &restoreDispatchLeaseExpired{reason: "recovery queue lease expired while waiting for the dispatch CAS"}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DS01-F2.b:恢复投递身份必须在**同一个 CAS 事务**里锁定并复核:
|
||||
// queue → operation → 原 window/request_digest,再核当前 owner/attempt/lease。
|
||||
//
|
||||
// 只验"queue 是 running 且 operation 是 recover 类型"会让一条本属于别的窗口的
|
||||
// 恢复投递借用本次 effect 的许可;worker 入口的先验校验(ValidateLibraryRecoveryQueue)
|
||||
// 不能替代 CAS 点的实时授权。
|
||||
func lockAndVerifyRecoveryDelivery(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
||||
window := b.Key.WindowJobID
|
||||
if b.RecoveryJobID != governance.LibraryRecoveryQueueID(window) {
|
||||
return fmt.Errorf("recovery queue %s is not the registered queue for window %s", b.RecoveryJobID, window)
|
||||
}
|
||||
var queueType, queueState, payloadRaw, idempotency string
|
||||
var leaseOwner string
|
||||
var attempt int
|
||||
var leaseUntil *time.Time
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT type, state, payload::text, COALESCE(lease_owner,''), attempt, lease_until, idempotency_key
|
||||
FROM jobs WHERE id=$1 FOR UPDATE`, b.RecoveryJobID).Scan(
|
||||
&queueType, &queueState, &payloadRaw, &leaseOwner, &attempt, &leaseUntil, &idempotency)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("recovery queue job %s does not exist", b.RecoveryJobID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock recovery queue job: %w", err)
|
||||
}
|
||||
if queueType != "library.recover" || queueState != "running" {
|
||||
return fmt.Errorf("recovery queue job %s is not a running delivery (type=%s state=%s)",
|
||||
b.RecoveryJobID, queueType, queueState)
|
||||
}
|
||||
if idempotency != "library.recover:"+window {
|
||||
return fmt.Errorf("recovery queue %s is not idempotency-bound to window %s", b.RecoveryJobID, window)
|
||||
}
|
||||
var payload map[string]string
|
||||
if err := json.Unmarshal([]byte(payloadRaw), &payload); err != nil {
|
||||
return fmt.Errorf("recovery queue payload is not a delivery claim: %w", err)
|
||||
}
|
||||
if payload["scope"] != b.Scope || payload["window_job_id"] != window {
|
||||
return fmt.Errorf("recovery queue %s carries window %s/%s, not %s/%s",
|
||||
b.RecoveryJobID, payload["window_job_id"], payload["scope"], window, b.Scope)
|
||||
}
|
||||
// 可选的锚点只要出现就必须与不可变绑定相同;缺省表示"由 journal/写集合决定"。
|
||||
for key, want := range map[string]string{
|
||||
"source_snapshot_id": b.SourceSnapshot,
|
||||
"target_id": b.Key.TargetIdentity,
|
||||
"original_operation_id": b.Key.OperationID,
|
||||
"backup_snapshot_id": b.BackupSnapshot,
|
||||
"plan_digest": b.Key.PlanDigest,
|
||||
} {
|
||||
if got := payload[key]; got != "" && got != want {
|
||||
return fmt.Errorf("recovery queue %s declares %s=%s, binding is %s",
|
||||
b.RecoveryJobID, key, got, want)
|
||||
}
|
||||
}
|
||||
if _, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); !ok {
|
||||
return fmt.Errorf("recovery queue %s attempt %d is not the current delivery of %s",
|
||||
b.RecoveryJobID, attempt, b.Owner)
|
||||
}
|
||||
if ownerWorker, _ := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); ownerWorker != leaseOwner {
|
||||
return fmt.Errorf("recovery queue %s is leased by %q, not by the attempt owner %q",
|
||||
b.RecoveryJobID, leaseOwner, ownerWorker)
|
||||
}
|
||||
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !live {
|
||||
return fmt.Errorf("recovery queue %s lease is expired", b.RecoveryJobID)
|
||||
}
|
||||
|
||||
var recoveryKind, recoveryScope, recoveryTarget, requestDigest, operationTarget string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT kind, scope, target_snapshot_id, request_digest, COALESCE(target_id,'')
|
||||
FROM library_operations WHERE id=$1 FOR UPDATE`, b.RecoveryOperationID).Scan(
|
||||
&recoveryKind, &recoveryScope, &recoveryTarget, &requestDigest, &operationTarget)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("recovery operation %s does not exist", b.RecoveryOperationID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock recovery operation: %w", err)
|
||||
}
|
||||
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope {
|
||||
return fmt.Errorf("recovery operation %s is not a recover operation for scope %s", b.RecoveryOperationID, b.Scope)
|
||||
}
|
||||
if recoveryTarget != b.BackupSnapshot {
|
||||
return fmt.Errorf("recovery operation %s restores %s, not the window backup %s",
|
||||
b.RecoveryOperationID, recoveryTarget, b.BackupSnapshot)
|
||||
}
|
||||
if operationTarget != "" && operationTarget != b.Key.TargetIdentity {
|
||||
return fmt.Errorf("recovery operation %s targets %s, not %s",
|
||||
b.RecoveryOperationID, operationTarget, b.Key.TargetIdentity)
|
||||
}
|
||||
if requestDigest == "" {
|
||||
return fmt.Errorf("recovery operation %s has no request digest anchor", b.RecoveryOperationID)
|
||||
}
|
||||
// 恢复请求摘要只能由原 window 的不可变锚点重算:queue/operation/window/原 operation/
|
||||
// 目标/来源/备份/写集合摘要。换任何一项都会得到另一份摘要。
|
||||
expected, err := governance.LibraryRecoveryDigest(b.Scope, b.RecoveryJobID, window, b.Key.OperationID,
|
||||
b.Key.TargetIdentity, b.SourceSnapshot, b.BackupSnapshot, b.Key.PlanDigest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recompute recovery request digest: %w", err)
|
||||
}
|
||||
if requestDigest != expected {
|
||||
return fmt.Errorf("recovery operation %s request digest is bound to another window/plan", b.RecoveryOperationID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/version"
|
||||
)
|
||||
|
||||
// DS01-F2.b:恢复投递必须在 CAS 事务里与 queue→operation→原 window/摘要绑定。
|
||||
func TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction(t *testing.T) {
|
||||
type fixture struct {
|
||||
store *Store
|
||||
pool *pgxpool.Pool
|
||||
binding libraryrelease.RestoreMarkerBinding
|
||||
}
|
||||
seeded := func(t *testing.T, jobID string) fixture {
|
||||
t.Helper()
|
||||
store, pool := newIdentityStore(t)
|
||||
now := time.Now().UTC()
|
||||
windowOp := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
|
||||
queueID := governance.LibraryRecoveryQueueID(jobID)
|
||||
owner := governance.LibraryDeliveryOwner("worker-r", queueID, 1)
|
||||
if _, err := store.BeginLibraryAttempt(context.Background(), jobID, owner, 10*time.Minute, now); err != nil {
|
||||
t.Fatalf("begin attempt: %v", err)
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]string{"scope": "internal", "window_job_id": jobID})
|
||||
if _, err := pool.Exec(context.Background(), `
|
||||
INSERT INTO jobs (id, type, payload, state, lease_owner, lease_until, attempt, idempotency_key)
|
||||
VALUES ($1,'library.recover',$2::jsonb,'running','worker-r',$3,1,$4)`,
|
||||
queueID, string(payload), now.Add(10*time.Minute), "library.recover:"+jobID); err != nil {
|
||||
t.Fatalf("seed recovery queue: %v", err)
|
||||
}
|
||||
recoveryOp := governance.LibraryOperationID("internal", governance.LibraryJobRecover, queueID)
|
||||
digest, err := governance.LibraryRecoveryDigest("internal", queueID, jobID, windowOp, "target-ds01",
|
||||
"snap-target", "snap-backup", "plan-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := store.EnsureLibraryOperation(context.Background(), governance.LibraryOperation{
|
||||
ID: recoveryOp, Scope: "internal", Kind: governance.LibraryJobRecover,
|
||||
TargetSnapshotID: "snap-backup", RequestDigest: digest, OperationKey: queueID,
|
||||
Actor: "worker-r", CreatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed recovery operation: %v", err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = jobID
|
||||
binding.Key.OperationID = windowOp
|
||||
binding.RecoveryJobID = queueID
|
||||
binding.RecoveryOperationID = recoveryOp
|
||||
binding.Owner = owner
|
||||
binding.Epoch = 1
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
|
||||
t.Fatalf("activate: %v", err)
|
||||
}
|
||||
return fixture{store: store, pool: pool, binding: binding}
|
||||
}
|
||||
expectRejected := func(t *testing.T, fx fixture) {
|
||||
t.Helper()
|
||||
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err == nil {
|
||||
t.Fatal("recovery delivery mismatch was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("positive real recovery delivery prepares and acquires", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-ok")
|
||||
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err != nil {
|
||||
t.Fatalf("valid recovery delivery: %v", err)
|
||||
}
|
||||
permit, _, err := fx.store.AcquireRestoreMarker(context.Background(), fx.binding)
|
||||
if err != nil || permit == nil {
|
||||
t.Fatalf("valid recovery delivery acquire: permit=%v err=%v", permit, err)
|
||||
}
|
||||
})
|
||||
t.Run("queue for another window is rejected", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-wrongqueue")
|
||||
fx.binding.RecoveryJobID = governance.LibraryRecoveryQueueID("another-window")
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("payload window differs", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-wrongwindow")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE jobs SET payload=jsonb_set(payload,'{window_job_id}','"other-window"') WHERE id=$1`,
|
||||
fx.binding.RecoveryJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("request digest belongs to another plan", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-wrongdigest")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE library_operations SET request_digest='digest-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("queue leased by another worker", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-owner")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE jobs SET lease_owner='intruder' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("expired queue lease", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-expired")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE jobs SET lease_until=now()-interval '1 minute' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("operation restores another backup", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-backup")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE library_operations SET target_snapshot_id='snap-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("binding plan differs from the recovery request", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-plan")
|
||||
fx.binding.Key.PlanDigest = "plan-other"
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("queue is not running", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-state")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE jobs SET state='pending', lease_owner='', lease_until=NULL WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
t.Run("operation is not a recovery", func(t *testing.T) {
|
||||
fx := seeded(t, "libsync-ds01-recovery-kind")
|
||||
if _, err := fx.pool.Exec(context.Background(),
|
||||
`UPDATE library_operations SET kind='sync' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectRejected(t, fx)
|
||||
})
|
||||
}
|
||||
|
||||
// DS01-F3.identity:激活与使用点都必须核治理权威域(集群+库 OID)、primary 与完整
|
||||
// 构建 SHA;同 display 不同不兼容 SHA、unknown、同名错域、错目标一律零激活/零许可。
|
||||
func TestRestoreDispatchGovernanceAuthorityAndBuildIdentity(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
// 使用点会读运行构建身份:测试进程由 runner 注入完整 SHA,这里核一下前提。
|
||||
if !version.HasInjection() || len(version.CommitSHA) != 40 {
|
||||
t.Skip("requires an injected full commit sha (run via the version-injecting runner)")
|
||||
}
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
job := "authority-window"
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level,
|
||||
services, ingress_state, state, original_service_state, note, acquired_at, updated_at)
|
||||
VALUES ($1,$2,'internal',$3,'instance',$4::jsonb,'closed','needs_restore','','',now()-interval '20 minutes',now())`,
|
||||
ds01TestInstance, job, ds01TestFingerprint, `["`+ds01TestWriter+`"]`); err != nil {
|
||||
t.Fatalf("seed maintenance window: %v", err)
|
||||
}
|
||||
dir, _, journalDigest := ds01Journal(t, job, nil)
|
||||
toStore := store.WithRestoreDispatchEvidence(dir)
|
||||
target := "target-ds01-authority"
|
||||
base := func() libraryrelease.RestoreDispatchActivationEvidence {
|
||||
return ds01ActivationEvidence(t, store, target, ds01TestFingerprint, job, journalDigest)
|
||||
}
|
||||
activate := func(evidence libraryrelease.RestoreDispatchActivationEvidence) error {
|
||||
t.Helper()
|
||||
path, digest := writeDS01ActivationEvidence(t, evidence)
|
||||
return toStore.ActivateRestoreDispatch(ctx, path, digest, target, ds01TestFingerprint, now)
|
||||
}
|
||||
count := func() int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch_activation`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// 1) 同名库、不同集群:库名相同但 system_identifier 不同。
|
||||
evidence := base()
|
||||
evidence.GovernanceAuthority.SystemIdentifier = "1"
|
||||
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
|
||||
t.Fatalf("same-name other-cluster evidence = %v", err)
|
||||
}
|
||||
// 2) 同集群、另一个同名库(OID 不同)。
|
||||
evidence = base()
|
||||
evidence.GovernanceAuthority.DatabaseOID = "1"
|
||||
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
|
||||
t.Fatalf("same-name other-database evidence = %v", err)
|
||||
}
|
||||
// 3) unknown 构建身份。
|
||||
evidence = base()
|
||||
evidence.ProtocolBuildSHA = "unknown"
|
||||
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "commit sha") {
|
||||
t.Fatalf("unknown build sha evidence = %v", err)
|
||||
}
|
||||
// 4) 同 display、不同 SHA、不同协议批次:不兼容,必须拒绝。
|
||||
evidence = base()
|
||||
evidence.ProtocolBuild = version.DisplayVersion()
|
||||
evidence.ProtocolBuildSHA = strings.Repeat("b", 40)
|
||||
evidence.ProtocolEpoch = libraryrelease.RestoreDispatchProtocolEpoch - 1
|
||||
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "epoch") {
|
||||
t.Fatalf("same-display incompatible build = %v", err)
|
||||
}
|
||||
if n := count(); n != 0 {
|
||||
t.Fatalf("rejected evidence created %d activation rows", n)
|
||||
}
|
||||
|
||||
// 5) 合法兼容升级:不同完整 SHA、同一协议批次 ⇒ 允许,但必须显式记成升级。
|
||||
compat := base()
|
||||
compat.ProtocolBuild = "0.1.999-compatible"
|
||||
compat.ProtocolBuildSHA = strings.Repeat("a", 40)
|
||||
if err := activate(compat); err != nil {
|
||||
t.Fatalf("compatible upgrade evidence: %v", err)
|
||||
}
|
||||
var stored string
|
||||
if err := pool.QueryRow(ctx, `SELECT evidence::text FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, target).Scan(&stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compact := strings.ReplaceAll(stored, " ", "")
|
||||
if !strings.Contains(compact, `"compatible_upgrade":true`) ||
|
||||
!strings.Contains(compact, `"system_identifier":"`+compat.GovernanceAuthority.SystemIdentifier+`"`) {
|
||||
t.Fatalf("activation record does not state the verified authority/upgrade: %s", stored)
|
||||
}
|
||||
|
||||
// 使用点正例:同一兼容批次下真实窗口可以准备/取得许可。
|
||||
windowOp := seedOperationJob(t, store, pool, "internal", "libsync-ds01-authority", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-authority", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-authority"
|
||||
binding.Key.OperationID = windowOp
|
||||
binding.Key.TargetIdentity = target
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatalf("use point rejected a compatible build: %v", err)
|
||||
}
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, binding); err != nil || permit == nil {
|
||||
t.Fatalf("use point acquire under compatible build: permit=%v err=%v", permit, err)
|
||||
}
|
||||
|
||||
// 使用点反例:把登记行篡改成错域/错批次/unknown 构建,Prepare 必须零许可失败。
|
||||
tamperTargets := map[string]func(map[string]any){
|
||||
"target-ds01-tamper-domain": func(v map[string]any) {
|
||||
authority, _ := v["governance_authority"].(map[string]any)
|
||||
authority["system_identifier"] = "1"
|
||||
},
|
||||
"target-ds01-tamper-epoch": func(v map[string]any) {
|
||||
v["protocol_epoch"] = float64(libraryrelease.RestoreDispatchProtocolEpoch - 1)
|
||||
},
|
||||
"target-ds01-tamper-sha": func(v map[string]any) {
|
||||
v["protocol_build_sha"] = "unknown"
|
||||
},
|
||||
}
|
||||
for tamperTarget, mutate := range tamperTargets {
|
||||
var record map[string]any
|
||||
if err := json.Unmarshal([]byte(stored), &record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verified, _ := record["verified"].(map[string]any)
|
||||
mutate(verified)
|
||||
raw, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO library_index_restore_dispatch_activation
|
||||
(target_identity, protocol_version, target_fingerprint, target_instance, governance_database,
|
||||
protocol_build, activated_by, drain_evidence_digest, evidence, attested_at, activated_at)
|
||||
SELECT $1, protocol_version, target_fingerprint, target_instance, governance_database,
|
||||
protocol_build, activated_by, drain_evidence_digest, $2::jsonb, attested_at, activated_at
|
||||
FROM library_index_restore_dispatch_activation WHERE target_identity=$3`,
|
||||
tamperTarget, string(raw), target); err != nil {
|
||||
t.Fatalf("insert tampered activation %s: %v", tamperTarget, err)
|
||||
}
|
||||
tampered := binding
|
||||
tampered.Key.TargetIdentity = tamperTarget
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, tampered); permit != nil || err == nil {
|
||||
t.Fatalf("use point accepted tampered activation %s: permit=%v err=%v", tamperTarget, permit, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 使用点错目标:同一激活行但本次观察的目标指纹不同。
|
||||
wrongTarget := binding
|
||||
wrongTarget.Key.DocumentID = "doc-other-target"
|
||||
wrongTarget.TargetFingerprint = ds01TestFingerprintOther
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongTarget); permit != nil || err == nil {
|
||||
t.Fatalf("use point accepted another target fingerprint: permit=%v err=%v", permit, err)
|
||||
}
|
||||
|
||||
// 运行构建 unknown:即使已有合法登记,使用点也必须失败关闭。
|
||||
originalSHA := version.CommitSHA
|
||||
version.CommitSHA = "unknown"
|
||||
t.Cleanup(func() { version.CommitSHA = originalSHA })
|
||||
unknown := binding
|
||||
unknown.Key.DocumentID = "doc-unknown-build"
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, unknown)
|
||||
if permit != nil || err == nil {
|
||||
t.Fatalf("use point accepted an unknown running build: permit=%v err=%v", permit, err)
|
||||
}
|
||||
if errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Log("unknown build rejected as held (expected fail-closed)")
|
||||
}
|
||||
}
|
||||
@@ -190,6 +190,63 @@ func TestRestoreDispatchLeaseExpiryWhileWaitingForLockIsRejected(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 等锁跨过租约(effect 行):前面的检查发生在拿 effect 锁之前,全部通过;真正等
|
||||
// effect 行锁时租约到期。CAS 点必须按数据库实时时钟重核,零许可、状态保持 prepared。
|
||||
func TestRestoreDispatchLeaseExpiryWhileWaitingForEffectLockIsRejected(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-effect-lease", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-effect-lease", "worker-a#q1#0", 1500*time.Millisecond, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-effect-lease"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lock, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := lock.Exec(ctx, `SELECT id FROM library_index_restore_dispatch WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
|
||||
binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest,
|
||||
binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
permit *libraryrelease.RestoreSendPermit
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
done <- result{permit, err}
|
||||
}()
|
||||
time.Sleep(2200 * time.Millisecond)
|
||||
if err := lock.Rollback(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := <-done
|
||||
if got.permit != nil {
|
||||
t.Fatal("permit issued although the attempt lease expired while waiting for the effect row lock")
|
||||
}
|
||||
if got.err == nil || !strings.Contains(got.err.Error(), "expired while waiting for the dispatch CAS") {
|
||||
t.Fatalf("expired effect-lock wait error = %v", got.err)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
t.Fatalf("state = %s after a rejected CAS", state)
|
||||
}
|
||||
}
|
||||
|
||||
// 错目标/错来源/换 plan 别名/错 UUID 都必须在 CAS 前失败,且台账不动。
|
||||
func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
@@ -238,38 +295,43 @@ func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
|
||||
if count != 1 {
|
||||
t.Fatalf("dispatch rows = %d, want exactly the original plan row", count)
|
||||
}
|
||||
// 恢复投递身份缺失/错误:Prepare 就必须失败,不能等到发送前。
|
||||
// 恢复投递身份缺失:Prepare 就必须失败,不能等到发送前。
|
||||
recovery := binding
|
||||
recovery.RecoveryJobID = "queue-missing"
|
||||
recovery.RecoveryOperationID = "op-missing"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, recovery); err == nil {
|
||||
t.Fatal("unknown recovery delivery identity was accepted")
|
||||
}
|
||||
// 真实的 recovery queue/operation 绑定可以被验证通过;换成另一份 backup 的
|
||||
// recovery operation 则必须拒绝。
|
||||
// 形状合法但与本 window 无关的 recovery queue/operation:同样零许可。
|
||||
// 真正绑定成功的正例在 TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction。
|
||||
queueID := governance.LibraryRecoveryQueueID(binding.Key.WindowJobID)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs (id, type, payload, state, lease_owner, idempotency_key)
|
||||
VALUES ('queue-recovery-1','library.recover','{}'::jsonb,'running','worker-a','queue-recovery-1')`); err != nil {
|
||||
VALUES ($1,'library.recover',$2::jsonb,'running','worker-a',$3)`,
|
||||
"queue-recovery-unrelated", `{"scope":"internal","window_job_id":"another-window"}`,
|
||||
"library.recover:another-window"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: "op-recovery-1", Scope: "internal", Kind: governance.LibraryJobRecover,
|
||||
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-1",
|
||||
ID: "op-recovery-unrelated", Scope: "internal", Kind: governance.LibraryJobRecover,
|
||||
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-unrelated",
|
||||
Actor: "tester", CreatedAt: now.UTC(),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validRecovery := binding
|
||||
validRecovery.Key.DocumentID = "doc-recovery"
|
||||
validRecovery.RecoveryJobID = "queue-recovery-1"
|
||||
validRecovery.RecoveryOperationID = "op-recovery-1"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, validRecovery); err != nil {
|
||||
t.Fatalf("valid recovery delivery binding: %v", err)
|
||||
unrelated := binding
|
||||
unrelated.Key.DocumentID = "doc-recovery"
|
||||
unrelated.RecoveryJobID = "queue-recovery-unrelated"
|
||||
unrelated.RecoveryOperationID = "op-recovery-unrelated"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, unrelated); err == nil {
|
||||
t.Fatal("recovery delivery for another window was accepted")
|
||||
}
|
||||
wrongRecovery := validRecovery
|
||||
wrongRecovery.Key.DocumentID = "doc-recovery-2"
|
||||
wrongRecovery.RecoveryOperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, wrongRecovery); err == nil {
|
||||
t.Fatal("recovery operation for another intent was accepted")
|
||||
// 正确 queue 但 recovery operation 指向原业务 operation:不是恢复请求,必须拒绝。
|
||||
wrongOperation := binding
|
||||
wrongOperation.Key.DocumentID = "doc-recovery-2"
|
||||
wrongOperation.RecoveryJobID = queueID
|
||||
wrongOperation.RecoveryOperationID = binding.Key.OperationID
|
||||
if _, err := store.PrepareRestoreMarker(ctx, wrongOperation); err == nil {
|
||||
t.Fatal("non-recovery operation was accepted as a recovery delivery")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -433,22 +433,42 @@ func ds01JournalRowJob(t *testing.T, store *Store) string {
|
||||
return job
|
||||
}
|
||||
|
||||
// ds01ActivationEvidence 组装 v2 证据:只给身份/作业/摘要/writer 集合。
|
||||
// ds01ActivationEvidence 组装 v3 证据:身份/作业/摘要/writer 集合 + 治理权威域 + 构建 SHA。
|
||||
func ds01ActivationEvidence(t *testing.T, store *Store, targetIdentity, fingerprint, job, journalDigest string) libraryrelease.RestoreDispatchActivationEvidence {
|
||||
t.Helper()
|
||||
var database string
|
||||
if err := store.db.QueryRow(context.Background(), `SELECT current_database()`).Scan(&database); err != nil {
|
||||
t.Fatalf("read governance database: %v", err)
|
||||
authority, primary, database := ds01ObservedAuthority(t, store)
|
||||
if !primary {
|
||||
t.Fatal("governance database is not primary")
|
||||
}
|
||||
return libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(),
|
||||
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
|
||||
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
|
||||
TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database,
|
||||
GovernanceAuthority: authority, ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA,
|
||||
AttestedBy: "ds01-test", AttestedAt: time.Now().UTC(),
|
||||
MaintenanceJob: job, MaintenanceJournal: job + ".json", MaintenanceJournalSHA256: journalDigest,
|
||||
Writers: []libraryrelease.RestoreDispatchWriterDrain{{Name: ds01TestWriter, Kind: "worker", InstanceID: ds01TestWriter}},
|
||||
}
|
||||
}
|
||||
|
||||
// ds01ObservedAuthority 读回测试进程实际连接的治理库权威域;证据必须携带它,
|
||||
// 否则"同名异集群"永远无法被拒绝。
|
||||
func ds01ObservedAuthority(t *testing.T, store *Store) (libraryrelease.RestoreDispatchGovernanceAuthority, bool, string) {
|
||||
t.Helper()
|
||||
var authority libraryrelease.RestoreDispatchGovernanceAuthority
|
||||
var inRecovery bool
|
||||
var database string
|
||||
if err := store.db.QueryRow(context.Background(), `
|
||||
SELECT current_database(),
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
|
||||
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery); err != nil {
|
||||
t.Fatalf("observe governance authority: %v", err)
|
||||
}
|
||||
return authority, !inRecovery, database
|
||||
}
|
||||
|
||||
func writeDS01ActivationEvidence(t *testing.T, evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(evidence)
|
||||
|
||||
@@ -434,9 +434,16 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance)
|
||||
if expected := os.Getenv("KG_R38_VERSION"); expected != "" && build != expected {
|
||||
return "", nil, fmt.Errorf("test binary build %q != runner identity %q", build, expected)
|
||||
}
|
||||
var database string
|
||||
if err := gov.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
|
||||
return "", nil, fmt.Errorf("read governance database: %w", err)
|
||||
var database, systemIdentifier, databaseOID string
|
||||
if err := gov.QueryRow(ctx, `
|
||||
SELECT current_database(),
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
(SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan(
|
||||
&database, &systemIdentifier, &databaseOID); err != nil {
|
||||
return "", nil, fmt.Errorf("read governance authority: %w", err)
|
||||
}
|
||||
if expected := os.Getenv("KG_R38_COMMIT"); expected != "" && version.CommitSHA != expected {
|
||||
return "", nil, fmt.Errorf("test binary commit %q != runner identity %q", version.CommitSHA, expected)
|
||||
}
|
||||
journalSum := sha256.Sum256(rawJournal)
|
||||
// 激活后同一窗口还会继续写 journal(recovered 等);把激活那一刻的字节原样留档,
|
||||
@@ -447,8 +454,12 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
evidence := libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolVersion: 2, TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database, ProtocolBuild: build,
|
||||
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
|
||||
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
|
||||
TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database,
|
||||
GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID},
|
||||
ProtocolBuild: build, ProtocolBuildSHA: version.CommitSHA,
|
||||
AttestedBy: "r38-runner", AttestedAt: now,
|
||||
MaintenanceJob: job.ID, MaintenanceJournal: job.ID + ".json",
|
||||
MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]), Writers: writers,
|
||||
@@ -480,7 +491,10 @@ FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, cfg.Li
|
||||
facts := map[string]any{
|
||||
"target": target, "fingerprint": fingerprint, "window_fingerprint": windowFingerprint,
|
||||
"governance_database": database,
|
||||
"maintenance_job": job.ID, "protocol_build": build,
|
||||
"governance_authority": map[string]string{"system_identifier": systemIdentifier, "database_oid": databaseOID,
|
||||
"primary": "true"},
|
||||
"protocol_build_sha": version.CommitSHA,
|
||||
"maintenance_job": job.ID, "protocol_build": build,
|
||||
"journal_sha256": drainDigest, "journal_snapshot": filepath.Base(journalSnapshot),
|
||||
"activation_evidence_sha256": hex.EncodeToString(evidenceSum[:]),
|
||||
"journal_state": journal["state"], "gate": gate, "quiesce": quiesce,
|
||||
|
||||
@@ -16,6 +16,7 @@ type ds01StubGate struct {
|
||||
inspectStatus libraryrelease.RestoreDispatchStatus
|
||||
inspectFound bool
|
||||
records []libraryrelease.RestoreDispatchRecord
|
||||
confirmed int
|
||||
}
|
||||
|
||||
func (g *ds01StubGate) InspectRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
|
||||
@@ -28,6 +29,7 @@ func (g *ds01StubGate) AcquireRestoreMarker(context.Context, libraryrelease.Rest
|
||||
return nil, g.inspectStatus, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
func (g *ds01StubGate) ConfirmRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding, []byte) error {
|
||||
g.confirmed++
|
||||
return nil
|
||||
}
|
||||
func (g *ds01StubGate) UnresolvedRestoreDispatches(context.Context, string, string, string) ([]libraryrelease.RestoreDispatchRecord, error) {
|
||||
@@ -127,6 +129,102 @@ func TestDS01BindingConflictHeldBeforeAnyWrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// DS01-F4.binding:绑定冲突是**类型化**分类——prepared/dispatching/marker_confirmed/
|
||||
// done 任一状态、准入/确认/收尾任一入口都必须拒绝;合法 dispatching + 精确 marker
|
||||
// (不冲突)仍允许确认,不能用"拒绝全部 Held"掩盖。
|
||||
func TestDS01BindingConflictRejectedInEveryState(t *testing.T) {
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
authority.attach(target)
|
||||
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
ledger := outcome.Backup.WriteSet
|
||||
if err := target.AdoptWriteSet(*ledger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds01FreezeBindingInputs(t, target, outcome.Backup)
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
body, err := target.restoreIndexClaimBody(change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 在目标上写入精确 marker:这样 dispatching 状态本来是可确认的,冲突检查必须
|
||||
// 独立地拒绝它,而不是靠"marker 不在"。
|
||||
applied, err := target.loadIndexEffect(change, "applied")
|
||||
if err != nil || applied == nil || applied.Receipt == nil {
|
||||
t.Fatalf("applied receipt missing: %v", err)
|
||||
}
|
||||
method, path, payload, err := target.indexMutation(change, body, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var marker indexReceipt
|
||||
if err := target.search.requestJSON(fx.ctx, method, path, payload, &marker); err != nil {
|
||||
t.Fatalf("exact marker write: %v", err)
|
||||
}
|
||||
observed, err := target.observePlannedIndex(fx.ctx, change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conflict := func(state string) *ds01StubGate {
|
||||
return &ds01StubGate{inspectFound: true, inspectStatus: libraryrelease.RestoreDispatchStatus{
|
||||
State: state, Held: true, BindingConflict: true, Reason: "binding differs"},
|
||||
records: []libraryrelease.RestoreDispatchRecord{{PlanDigest: ledger.Digest, Index: change.Index,
|
||||
DocumentID: change.ID, State: libraryrelease.RestoreDispatchMarkerConfirmed}}}
|
||||
}
|
||||
for _, state := range []string{
|
||||
libraryrelease.RestoreDispatchPrepared,
|
||||
libraryrelease.RestoreDispatchDispatching,
|
||||
libraryrelease.RestoreDispatchMarkerConfirmed,
|
||||
} {
|
||||
gate := conflict(state)
|
||||
target.config.RestoreDispatch = gate
|
||||
if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
|
||||
t.Fatalf("admission accepted a binding conflict in %s: %v", state, err)
|
||||
}
|
||||
if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err == nil || !strings.Contains(err.Error(), "冲突") {
|
||||
t.Fatalf("confirmation accepted a binding conflict in %s: %v", state, err)
|
||||
}
|
||||
if gate.confirmed != 0 {
|
||||
t.Fatalf("binding conflict in %s still recorded a confirmation", state)
|
||||
}
|
||||
}
|
||||
|
||||
// done 状态:restored 文件已在,准入仍必须因绑定冲突拒绝(旧实现只看状态)。
|
||||
receipt := indexReceipt{Index: change.Index, ID: change.ID, Result: "created",
|
||||
SeqNo: marker.SeqNo, PrimaryTerm: marker.PrimaryTerm}
|
||||
if err := target.saveIndexEffect(change, "restored", &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target.config.RestoreDispatch = conflict(libraryrelease.RestoreDispatchMarkerConfirmed)
|
||||
if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
|
||||
t.Fatalf("admission accepted a binding conflict on a completed effect: %v", err)
|
||||
}
|
||||
// 收尾:记录状态看起来是 marker_confirmed、restored 也存在,但绑定冲突必须拒绝。
|
||||
gate := conflict(libraryrelease.RestoreDispatchMarkerConfirmed)
|
||||
target.config.RestoreDispatch = gate
|
||||
if err := target.verifyRestoreEffectsClosed(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
|
||||
t.Fatalf("closeout accepted a binding conflict: %v", err)
|
||||
}
|
||||
if gate.confirmed != 0 {
|
||||
t.Fatal("closeout recorded a confirmation for a conflicting binding")
|
||||
}
|
||||
|
||||
// 正例:合法 dispatching + 精确 marker(不冲突)仍可确认,且只确认一次。
|
||||
okGate := &ds01StubGate{inspectFound: true,
|
||||
inspectStatus: libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
|
||||
Reason: "dispatch outcome unknown: no further create is permitted"}}
|
||||
target.config.RestoreDispatch = okGate
|
||||
if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err != nil {
|
||||
t.Fatalf("legitimate dispatching marker confirmation: %v", err)
|
||||
}
|
||||
if okGate.confirmed != 1 {
|
||||
t.Fatalf("legitimate marker confirmation count = %d", okGate.confirmed)
|
||||
}
|
||||
if wire.dispatch != 0 {
|
||||
t.Fatalf("confirmation path forwarded %d marker writes", wire.dispatch)
|
||||
}
|
||||
}
|
||||
|
||||
// DS01-F4:prepared 台账却已在目标上出现精确 marker —— 证据矛盾,必须 held;
|
||||
// 旧实现会把它当成"已存在的标记"认领并 clean。
|
||||
func TestDS01PreparedWithMarkerIsContradictionHeld(t *testing.T) {
|
||||
|
||||
@@ -549,8 +549,12 @@ func TestDS01RestoreHelper(t *testing.T) {
|
||||
// 测试替身只负责造这两个真实来源,不替产品做判断。
|
||||
func activateDS01(ctx context.Context, store *postgresstore.Store, pool *pgxpool.Pool,
|
||||
targetIdentity, fingerprint string) error {
|
||||
var database, job string
|
||||
if err := pool.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
|
||||
var database, systemIdentifier, databaseOID, job string
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT current_database(),
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
(SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan(
|
||||
&database, &systemIdentifier, &databaseOID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT owner_job_id FROM library_instance_maintenance WHERE instance=$1`,
|
||||
@@ -599,8 +603,12 @@ WHERE instance=$1`, ds01TargetInstance, `["worker-ds01"]`, fingerprint); err !=
|
||||
return err
|
||||
}
|
||||
evidence := libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(),
|
||||
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
|
||||
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
|
||||
TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
|
||||
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database,
|
||||
GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID},
|
||||
ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA,
|
||||
AttestedBy: "ds01-test", AttestedAt: now,
|
||||
MaintenanceJob: job, MaintenanceJournal: job + ".json",
|
||||
MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]),
|
||||
|
||||
@@ -137,6 +137,9 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if !found {
|
||||
return fmt.Errorf("%w: 既存意图没有派发台账(旧协议未知历史)", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
if status.BindingConflict {
|
||||
return heldRestoreMarker(status, "既存意图的派发台账绑定与本次计划冲突")
|
||||
}
|
||||
if status.State == libraryrelease.RestoreDispatchPrepared && claimed != nil {
|
||||
return heldRestoreMarker(status, "prepared 台账却已有 claimed 证据:协议证据矛盾")
|
||||
}
|
||||
|
||||
@@ -161,6 +161,12 @@ func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
|
||||
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
|
||||
}
|
||||
if found {
|
||||
// 绑定冲突是类型化分类:任何状态(含已完成 clean 的终态)都必须拒绝。
|
||||
// 它不能与"仅禁止第二次 create"的 dispatching-held 混为一谈——后者在
|
||||
// 精确 marker 仍在时是允许确认的。
|
||||
if status.BindingConflict {
|
||||
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
|
||||
}
|
||||
if done != nil {
|
||||
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
return heldRestoreMarker(status, "已完成 clean 的 effect 台账却不是 marker_confirmed")
|
||||
@@ -241,6 +247,9 @@ func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c
|
||||
if !found {
|
||||
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
if status.BindingConflict {
|
||||
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
|
||||
}
|
||||
if status.Held && status.State == libraryrelease.RestoreDispatchPrepared {
|
||||
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
|
||||
}
|
||||
@@ -370,6 +379,28 @@ func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context
|
||||
if done == nil {
|
||||
return fmt.Errorf("%w: 收尾前仍有未 clean 的 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||||
}
|
||||
// 收尾同样是**使用点**:绑定冲突即使状态看起来是 marker_confirmed 也必须拒绝。
|
||||
body, err := t.restoreIndexClaimBody(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
binding, err := t.restoreMarkerBinding(ctx, c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return fmt.Errorf("恢复收尾读不到派发台账:%w", err)
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%w: 收尾时 effect %s 的派发台账消失了", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||||
}
|
||||
if status.BindingConflict {
|
||||
return heldRestoreMarker(status, "收尾时派发台账的不可变绑定与本次计划冲突")
|
||||
}
|
||||
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
return fmt.Errorf("%w: 收尾时 effect %s 的派发状态是 %s", libraryrelease.ErrRestoreDispatchHeld, c.ID, status.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range t.activePlan.Index {
|
||||
|
||||
Reference in New Issue
Block a user