diff --git a/CHANGELOG.md b/CHANGELOG.md index f42c72a..8185229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## 开发快照(DS01 收口:CAS 点实时租约、恢复投递归属、治理权威域与绑定冲突分类) + +### Fixed + +- **F2.a CAS 点实时租约**:`Acquire` 在取得 effect 行锁(最后一个可能阻塞的锁)之后、 + 状态 CAS 之前,按数据库 `clock_timestamp()` 重核 job/attempt/恢复投递租约;`Confirm` + 同样先锁 effect 行再核。等锁跨过到期一律零许可、状态保持 `prepared`。新增真 PG + 反例:等 effect 行锁期间租约过期。 +- **F2.b 恢复投递同事务绑定**:同一短事务内 `FOR UPDATE` 锁 queue 与 recovery operation, + 复核 queue id/type/state/payload 的 window+scope、idempotency key、当前 owner/attempt/ + lease,并按原 window 的不可变锚点重算 `LibraryRecoveryDigest`,与 operation 的 + `request_digest` 逐位比较;错 queue/window/digest/owner/过期/错备份/错 plan 全部零许可。 + 投递 owner 合成规则收敛到 `governance.LibraryDeliveryOwner` 一份。 +- **F3.identity 治理权威域与构建身份**:激活证据升至 v3,必须携带 PG 集群 + `system_identifier` + 库 OID、完整 40 位构建 SHA 与协议兼容批次;激活与每次使用点都 + 现场重读 `pg_control_system()`/`pg_database.oid`/`pg_is_in_recovery()` 并比对,拒绝 + 同名异域、副本、unknown 构建、同 display 不同不兼容 SHA 与错目标指纹。同一兼容批次 + 内的不同完整 SHA 记为 `compatible_upgrade`,不静默等同。 +- **F4.binding 绑定冲突类型化**:`RestoreDispatchStatus.BindingConflict` 与"仅禁止重发" + 的 `Held` 分开;准入、确认、既有意图续作与收尾在 prepared/dispatching/marker_confirmed/ + done 任一状态都拒绝绑定冲突;合法 dispatching + 精确 marker 仍可确认一次。 +- 真 PG/adapter 反例:effect 锁等待过期、恢复投递九类错配、错治理域/错批次/unknown、 + 四状态绑定冲突与合法 marker 正例。 + ## 开发快照(DS01 激活证据 v2:只认真实维护 journal) ### Fixed diff --git a/cmd/worker/library_writer.go b/cmd/worker/library_writer.go index 393d3ba..5d2ca81 100644 --- a/cmd/worker/library_writer.go +++ b/cmd/worker/library_writer.go @@ -135,9 +135,7 @@ func libraryQueueOwner(workerID string, job jobs.Job, libraryJobID string) strin if queueID == "" { queueID = libraryJobID } - worker := workerID - if worker == "" { - worker = "worker" - } - return fmt.Sprintf("%s#%s#%d", worker, queueID, job.Attempt) + // 合成规则只有一份(governance.LibraryDeliveryOwner):派发台账要按同一格式核 + // "这次许可属于哪次真实投递"。 + return governance.LibraryDeliveryOwner(workerID, queueID, job.Attempt) } diff --git a/internal/governance/library_recovery_identity.go b/internal/governance/library_recovery_identity.go index 5f13a07..53c12a1 100644 --- a/internal/governance/library_recovery_identity.go +++ b/internal/governance/library_recovery_identity.go @@ -3,6 +3,8 @@ package governance import ( "crypto/sha256" "encoding/hex" + "fmt" + "strings" ) // The recovery request has its own queue identity and stable operation. It @@ -18,3 +20,33 @@ func LibraryRecoveryDigest(scope, queue, window, operation, target, source, back "target_id": target, "source_snapshot": source, "backup_snapshot": backup, "writeset_digest": plan, }) } + +// LibraryDeliveryOwner 是队列投递 + 业务 attempt 的合成 owner: +// +// ## +// +// 它就是 library_attempts.owner,也决定了"这次派发属于哪一次真实投递"。worker 与 +// 派发台账必须用同一份格式,否则"当前 owner"会变成两处各自解释的字符串。 +func LibraryDeliveryOwner(workerID, queueID string, attempt int) string { + if queueID == "" { + queueID = "-" + } + if workerID == "" { + workerID = "worker" + } + return fmt.Sprintf("%s#%s#%d", workerID, queueID, attempt) +} + +// ParseLibraryDeliveryOwner 把 owner 拆回 (worker, ok);queue/attempt 必须逐位相同, +// 不能靠后缀匹配把别的投递混进来。 +func ParseLibraryDeliveryOwner(owner, queueID string, attempt int) (string, bool) { + suffix := fmt.Sprintf("#%s#%d", queueID, attempt) + if queueID == "" || !strings.HasSuffix(owner, suffix) { + return "", false + } + worker := strings.TrimSuffix(owner, suffix) + if worker == "" || strings.Contains(worker, "#") { + return "", false + } + return worker, true +} diff --git a/internal/libraryrelease/index_restore_dispatch.go b/internal/libraryrelease/index_restore_dispatch.go index bf29149..f825cfc 100644 --- a/internal/libraryrelease/index_restore_dispatch.go +++ b/internal/libraryrelease/index_restore_dispatch.go @@ -126,10 +126,17 @@ func (b RestoreMarkerBinding) Digest() (string, error) { } // RestoreDispatchStatus 是纯读分类结果。Held 为真时永远不能发送。 +// +// BindingConflict 是与 Held 不同的**类型化**分类:Held 只说"这次不能再 create" +// (prepared 冲突、dispatching 结果未知、marker 已确认都可能是合法的不可重发), +// 而 BindingConflict 说明台账行的不可变绑定与本次绑定不是同一个 effect。绑定冲突 +// 在**任何状态**下都必须拒绝,包括已完成 clean 的 marker_confirmed;否则"只允许原 +// 确认分支继续"会变成放行一份不属于本计划的写入。 type RestoreDispatchStatus struct { - State string `json:"state"` - Held bool `json:"held"` - Reason string `json:"reason,omitempty"` + State string `json:"state"` + Held bool `json:"held"` + BindingConflict bool `json:"binding_conflict,omitempty"` + Reason string `json:"reason,omitempty"` } // RestoreSendPermit 是"这一次调用、这一个 effect、这一次发送"的许可。 diff --git a/internal/libraryrelease/restore_dispatch_activation.go b/internal/libraryrelease/restore_dispatch_activation.go index 36b01fd..917b2c7 100644 --- a/internal/libraryrelease/restore_dispatch_activation.go +++ b/internal/libraryrelease/restore_dispatch_activation.go @@ -29,7 +29,24 @@ import ( const RestoreDispatchActivationMaxAge = 2 * time.Hour -const RestoreDispatchActivationProtocolVersion = 2 +// 证据文档 schema 版本与语义兼容批次是两件事: +// - ProtocolVersion:这份 JSON 的字段形状(解析契约); +// - ProtocolEpoch:派发协议语义批次;只有同一批次的构建才允许共享激活证据 +// (例如同一批次的兼容重建/打包),不同批次必须重新登记。 +// +// 构建身份用完整 40 位 SHA,不用 display:开发号会随提交数变化,两个不同二进制 +// 完全可能都叫 0.1.322。 +const RestoreDispatchActivationProtocolVersion = 3 + +const RestoreDispatchProtocolEpoch = 3 + +// RestoreDispatchGovernanceAuthority 是治理库的**权威域**身份:系统标识(集群) +// 加库 OID。只看 current_database() 无法区分"同名但另一个集群/另一份克隆"。 +// primary 由 Store 现场读 pg_is_in_recovery() 得出,不采信调用方声明。 +type RestoreDispatchGovernanceAuthority struct { + SystemIdentifier string `json:"system_identifier"` + DatabaseOID string `json:"database_oid"` +} // RestoreDispatchWriterDrain 是调用方对"旧 writer 集合"的声明:只给身份,不给结论。 // InstanceID 必须等于 journal 里登记的不可变 writer 身份(容器/进程 ID)。 @@ -41,27 +58,35 @@ type RestoreDispatchWriterDrain struct { type RestoreDispatchActivationEvidence struct { ProtocolVersion int `json:"protocol_version"` + ProtocolEpoch int `json:"protocol_epoch"` TargetIdentity string `json:"target_identity"` // TargetFingerprint 是维护窗口/部署目标指纹(journal 与治理库维护行记录的那一个); // RestoreTargetFingerprint 是激活时对恢复目标内容/索引/桶的实时观察值。两者是不同层, // 不能相互替代:前者绑定"动的是哪个实例",后者绑定"恢复要写的那个目标身份"。 - TargetFingerprint string `json:"target_fingerprint"` - RestoreTargetFingerprint string `json:"restore_target_fingerprint"` - TargetInstance string `json:"target_instance"` - GovernanceDatabase string `json:"governance_database"` - ProtocolBuild string `json:"protocol_build"` - AttestedBy string `json:"attested_by"` - AttestedAt time.Time `json:"attested_at"` - MaintenanceJob string `json:"maintenance_job"` - MaintenanceJournal string `json:"maintenance_journal"` - MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"` - Writers []RestoreDispatchWriterDrain `json:"writers"` + TargetFingerprint string `json:"target_fingerprint"` + RestoreTargetFingerprint string `json:"restore_target_fingerprint"` + TargetInstance string `json:"target_instance"` + GovernanceDatabase string `json:"governance_database"` + GovernanceAuthority RestoreDispatchGovernanceAuthority `json:"governance_authority"` + ProtocolBuild string `json:"protocol_build"` + ProtocolBuildSHA string `json:"protocol_build_sha"` + AttestedBy string `json:"attested_by"` + AttestedAt time.Time `json:"attested_at"` + MaintenanceJob string `json:"maintenance_job"` + MaintenanceJournal string `json:"maintenance_journal"` + MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"` + Writers []RestoreDispatchWriterDrain `json:"writers"` } func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.Duration) error { if e.ProtocolVersion != RestoreDispatchActivationProtocolVersion { return fmt.Errorf("restore dispatch activation protocol version %d is not supported", e.ProtocolVersion) } + // 兼容批次不同 = 派发语义不兼容:即使 display 相同也不能共用激活证据。 + if e.ProtocolEpoch != RestoreDispatchProtocolEpoch { + return fmt.Errorf("restore dispatch activation protocol epoch %d is not compatible with %d", + e.ProtocolEpoch, RestoreDispatchProtocolEpoch) + } for name, value := range map[string]string{ "target identity": e.TargetIdentity, "target fingerprint": e.TargetFingerprint, "restore target fingerprint": e.RestoreTargetFingerprint, @@ -74,6 +99,14 @@ func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.D return fmt.Errorf("restore dispatch activation evidence lacks %s", name) } } + // 构建身份必须是完整 SHA;display 可以随提交数变化,不能作为身份。 + if !isFullSHA(e.ProtocolBuildSHA) { + return errors.New("restore dispatch activation build identity is not a full 40-character commit sha") + } + // 治理权威域必须是可读回的 PG 事实形状:集群系统标识 + 库 OID,两个都是十进制数字。 + if !isDecimal(e.GovernanceAuthority.SystemIdentifier) || !isDecimal(e.GovernanceAuthority.DatabaseOID) { + return errors.New("restore dispatch activation governance authority is not a cluster/database identity") + } if !isHex64(e.TargetFingerprint) || !isHex64(e.RestoreTargetFingerprint) { return errors.New("restore dispatch activation target fingerprints are not sha256 hex strings") } @@ -161,6 +194,26 @@ func isHex64(value string) bool { return err == nil } +func isFullSHA(value string) bool { + if len(value) != 40 || strings.ToLower(value) != value { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func isDecimal(value string) bool { + if value == "" || len(value) > 20 { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + // validActivationID 与维护 journal 的作业/服务 ID 同一字符集:证据里出现的身份 // 必须能原样出现在 journal 中,不能靠空格/路径字符制造"看起来不同"的别名。 func validActivationID(value string) bool { diff --git a/internal/platform/postgres/library_restore_dispatch.go b/internal/platform/postgres/library_restore_dispatch.go index 1f9f01d..0132d27 100644 --- a/internal/platform/postgres/library_restore_dispatch.go +++ b/internal/platform/postgres/library_restore_dispatch.go @@ -103,11 +103,17 @@ WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetI // lockRestoreDispatchActivation 在**同一个 CAS 事务**里锁激活行。撤销是同一行的 // UPDATE:撤销先完成 ⇒ 这里看到 revoked_at;CAS 先取锁 ⇒ 撤销必须等 CAS 提交, // 之后按"已派发"的窄边界处理。两者只能有一个线性化顺序。 -func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity string) error { +// +// 这里同时是**使用点**的权威域/构建身份重核:激活行里的 verified 事实必须与当前 +// 连接的 PG 权威域(集群 system_identifier + 库 OID + primary)一致;运行构建必须 +// 有完整 SHA,且与激活批次兼容;恢复目标指纹必须等于本次观察值。同名库、副本、 +// unknown 构建或换目标都在这里失败关闭,而不是只在激活那一刻检查。 +func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity, targetFingerprint string) error { var revoked *time.Time + var evidenceRaw, rowFingerprint string err := tx.QueryRow(ctx, ` -SELECT revoked_at FROM library_index_restore_dispatch_activation -WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked) +SELECT revoked_at, evidence::text, target_fingerprint FROM library_index_restore_dispatch_activation +WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked, &evidenceRaw, &rowFingerprint) if errors.Is(err, pgx.ErrNoRows) { return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity) } @@ -117,6 +123,39 @@ WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Sca if revoked != nil { return fmt.Errorf("%w(目标 %s 的激活已于 %s 撤销)", ErrRestoreDispatchNotActivated, targetIdentity, revoked.UTC().Format(time.RFC3339)) } + if rowFingerprint != targetFingerprint { + return fmt.Errorf("%w(目标 %s 的激活绑定指纹 %s,本次恢复目标指纹 %s)", + ErrRestoreDispatchNotActivated, targetIdentity, rowFingerprint, targetFingerprint) + } + var record restoreDispatchActivationRecord + if err := json.Unmarshal([]byte(evidenceRaw), &record); err != nil { + return fmt.Errorf("parse restore dispatch activation record: %w", err) + } + verified := record.Verified + authority, primary, _, err := observedGovernanceAuthority(ctx, tx) + if err != nil { + return err + } + if verified.GovernanceAuthority.SystemIdentifier == "" || verified.GovernanceAuthority.DatabaseOID == "" { + return fmt.Errorf("%w(目标 %s 的激活登记没有治理权威域身份:拒绝仅凭库名启用)", + ErrRestoreDispatchNotActivated, targetIdentity) + } + if authority != verified.GovernanceAuthority { + return fmt.Errorf("%w(目标 %s 的激活登记属于治理域 %s/%s,当前是 %s/%s)", + ErrRestoreDispatchNotActivated, targetIdentity, verified.GovernanceAuthority.SystemIdentifier, + verified.GovernanceAuthority.DatabaseOID, authority.SystemIdentifier, authority.DatabaseOID) + } + if !primary { + return fmt.Errorf("%w(治理库当前处于恢复/副本角色,拒绝授予派发许可)", ErrRestoreDispatchNotActivated) + } + buildEvidence := libraryrelease.RestoreDispatchActivationEvidence{ + ProtocolEpoch: verified.ProtocolEpoch, + ProtocolBuild: verified.ProtocolBuild, + ProtocolBuildSHA: verified.ProtocolBuildSHA, + } + if _, _, _, err := verifyRestoreDispatchBuild(buildEvidence); err != nil { + return fmt.Errorf("%w(目标 %s:%v)", ErrRestoreDispatchNotActivated, targetIdentity, err) + } return nil } @@ -143,7 +182,7 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err) } if stored != digest { - return libraryrelease.RestoreDispatchStatus{State: state, Held: true, + return libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true, Reason: "restore dispatch binding differs from the immutable plan"}, true, nil } switch state { @@ -223,7 +262,7 @@ func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.Resto if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil { return libraryrelease.RestoreDispatchStatus{}, err } - if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil { + if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil { return libraryrelease.RestoreDispatchStatus{}, err } if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil { @@ -270,7 +309,7 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto return nil, libraryrelease.RestoreDispatchStatus{}, err } // 撤销与 CAS 在同一事务线性化:这里拿激活行锁,撤销的 UPDATE 必须排队。 - if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil { + if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil { return nil, libraryrelease.RestoreDispatchStatus{}, err } // 当前代次必须属于这个原 window 作业,且租户/epoch/租约都仍有效。lease 用 @@ -322,13 +361,24 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err) } if stored != digest { - return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, + return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true, Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld } if state != libraryrelease.RestoreDispatchPrepared { return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld } + // effect 行锁是本次 CAS 前最后一个可能阻塞的锁。job/attempt/恢复投递的租约可能 + // 正好在等它的时候过期;前面那几次检查发生在拿锁之前,不能充当 CAS 点的判据。 + // 这些行锁都已被本事务持有,这里只重读、不改变锁序。 + if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil { + var held *restoreDispatchLeaseExpired + if errors.As(err, &held) { + return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true, + Reason: held.Error()}, fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, held.Error()) + } + return nil, libraryrelease.RestoreDispatchStatus{}, err + } var raw [16]byte if _, err := rand.Read(raw[:]); err != nil { @@ -384,7 +434,7 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil { return err } - if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil { + if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil { return err } var epoch int64 @@ -404,6 +454,23 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper if attemptJob != b.Key.WindowJobID || owner != b.Owner || epoch != b.Epoch || !leaseLive { return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale) } + // 与 Acquire 同一个顺序:确认前的最后一把锁也要先拿到,再按数据库实时时钟重核 + // 租约。否则"确认"可以在等锁跨过到期之后仍落库,把一次过期 owner 的回执写成事实。 + var effectID int64 + if err := tx.QueryRow(ctx, ` +SELECT id FROM library_index_restore_dispatch +WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 + AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`, + b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest, + b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&effectID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return errors.New("restore marker confirmation did not match a dispatch record") + } + return fmt.Errorf("lock restore marker for confirmation: %w", err) + } + if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil { + return fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, err) + } tag, err := tx.Exec(ctx, ` UPDATE library_index_restore_dispatch SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=clock_timestamp(), updated_at=clock_timestamp() @@ -513,23 +580,8 @@ FROM library_instance_maintenance WHERE instance=$1 FOR UPDATE`, b.TargetInstanc } if b.RecoveryJobID != "" { - var queueState string - var lease string - if err := tx.QueryRow(ctx, `SELECT state, COALESCE(lease_owner,'') FROM jobs WHERE id=$1`, b.RecoveryJobID). - Scan(&queueState, &lease); err != nil { - return fmt.Errorf("read recovery queue job: %w", err) - } - if queueState != "running" || lease == "" { - return fmt.Errorf("recovery queue job %s is not a live delivery", b.RecoveryJobID) - } - var recoveryKind, recoveryScope, recoveryBackup string - if err := tx.QueryRow(ctx, ` -SELECT kind, scope, target_snapshot_id FROM library_operations WHERE id=$1`, b.RecoveryOperationID). - Scan(&recoveryKind, &recoveryScope, &recoveryBackup); err != nil { - return fmt.Errorf("read recovery operation: %w", err) - } - if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope || recoveryBackup != b.BackupSnapshot { - return fmt.Errorf("recovery operation %s is not bound to this window and backup", b.RecoveryOperationID) + if err := lockAndVerifyRecoveryDelivery(ctx, tx, b); err != nil { + return err } } return nil diff --git a/internal/platform/postgres/library_restore_dispatch_activation.go b/internal/platform/postgres/library_restore_dispatch_activation.go index f0af1cc..1f5b564 100644 --- a/internal/platform/postgres/library_restore_dispatch_activation.go +++ b/internal/platform/postgres/library_restore_dispatch_activation.go @@ -2,10 +2,12 @@ package postgres import ( "context" + "encoding/hex" "encoding/json" "errors" "fmt" "sort" + "strings" "time" "github.com/jackc/pgx/v5" @@ -42,8 +44,51 @@ type restoreDispatchVerifiedView struct { Quiesce restoreDispatchQuiesceView `json:"quiesce"` Writers []restoreDispatchWriterView `json:"writers"` GovernanceDatabase string `json:"governance_database"` - ProtocolBuild string `json:"protocol_build"` - ObservedAt time.Time `json:"observed_at"` + // 治理权威域是现场读回的 PG 事实:集群 system_identifier + 库 OID + 是否 primary。 + // 激活与每次使用点都重读比较,库名只作可读标签。 + GovernanceAuthority restoreDispatchAuthorityView `json:"governance_authority"` + GovernancePrimary bool `json:"governance_primary"` + ProtocolBuild string `json:"protocol_build"` + ProtocolBuildSHA string `json:"protocol_build_sha"` + ProtocolEpoch int `json:"protocol_epoch"` + CompatibleUpgrade bool `json:"compatible_upgrade"` + ObservedAt time.Time `json:"observed_at"` +} + +// restoreDispatchAuthorityView 只放可比较的权威域字段;不含密钥/连接串。 +type restoreDispatchAuthorityView struct { + SystemIdentifier string `json:"system_identifier"` + DatabaseOID string `json:"database_oid"` +} + +// lockRestoreDispatchActivationVerification 是使用点重核所需的最小事实。 +type restoreDispatchActivationVerification struct { + Authority restoreDispatchAuthorityView + Primary bool + ProtocolBuild string + ProtocolBuildSHA string + ProtocolEpoch int + TargetFingerprint string +} + +// observedGovernanceAuthority 读回当前连接的治理库权威域。pg_control_system() 的 +// system_identifier 是集群级、跨备份/克隆仍不同的身份;库 OID 在同一集群内区分同名库。 +// 没有权限读它时返回错误(能力失败关闭),绝不退回"库名相同就算同一个库"。 +func observedGovernanceAuthority(ctx context.Context, q interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +}) (restoreDispatchAuthorityView, bool, string, error) { + var authority restoreDispatchAuthorityView + var database string + var inRecovery bool + err := q.QueryRow(ctx, ` +SELECT current_database(), + (SELECT system_identifier::text FROM pg_control_system()), + (SELECT oid::text FROM pg_database WHERE datname = current_database()), + pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery) + if err != nil { + return authority, false, "", fmt.Errorf("observe governance authority: %w", err) + } + return authority, !inRecovery, database, nil } type restoreDispatchGateView struct { @@ -128,21 +173,26 @@ func (s *Store) verifyRestoreDispatchEvidence(ctx context.Context, evidence libr if evidence.AttestedAt.Before(journal.Quiesce.PostStopAt) { return verified, errors.New("restore dispatch attestation predates the maintenance stop observation") } - var database string - if err := s.db.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil { - return verified, fmt.Errorf("read governance database identity: %w", err) + authority, primary, database, err := observedGovernanceAuthority(ctx, s.db) + if err != nil { + return verified, err } - if database == "" || database != evidence.GovernanceDatabase { + if authority != (restoreDispatchAuthorityView{SystemIdentifier: evidence.GovernanceAuthority.SystemIdentifier, + DatabaseOID: evidence.GovernanceAuthority.DatabaseOID}) { + return verified, fmt.Errorf("restore dispatch activation names governance authority %s/%s, live is %s/%s", + evidence.GovernanceAuthority.SystemIdentifier, evidence.GovernanceAuthority.DatabaseOID, + authority.SystemIdentifier, authority.DatabaseOID) + } + if !primary { + return verified, errors.New("restore dispatch activation requires the governance database to be primary, not a replica") + } + if database != evidence.GovernanceDatabase { return verified, fmt.Errorf("restore dispatch activation evidence names governance database %q, live is %q", evidence.GovernanceDatabase, database) } - build := version.DisplayVersion() - if !version.HasInjection() || build == "" || build == "dev" { - return verified, errors.New("restore dispatch activation requires a build with injected version identity") - } - if build != evidence.ProtocolBuild { - return verified, fmt.Errorf("restore dispatch activation evidence was produced by build %q, running build is %q", - evidence.ProtocolBuild, build) + build, buildSHA, compatible, err := verifyRestoreDispatchBuild(evidence) + if err != nil { + return verified, err } // 维护窗口指纹与恢复目标指纹是两层:前者已与 journal/维护行比对,后者必须等于 // 激活方刚刚对恢复目标做的那次只读观察。用一个值同时顶替两者会掩盖"配置指向 @@ -230,7 +280,52 @@ FROM library_instance_maintenance WHERE instance=$1`, evidence.TargetInstance).S Quiesce: restoreDispatchQuiesceView{Depth: journal.Quiesce.Depth, PendingOps: journal.Quiesce.PendingOps, Unsettled: journal.Quiesce.Unsettled, Digest: journal.Quiesce.Digest, StableMS: journal.Quiesce.StableMS, Samples: journal.Quiesce.Samples, ProvenAt: journal.Quiesce.ProvenAt.UTC(), PostStopAt: journal.Quiesce.PostStopAt.UTC()}, - Writers: verifiedWriters, GovernanceDatabase: database, ProtocolBuild: build, ObservedAt: now.UTC(), + Writers: verifiedWriters, GovernanceDatabase: database, + GovernanceAuthority: authority, GovernancePrimary: primary, + ProtocolBuild: build, ProtocolBuildSHA: buildSHA, + ProtocolEpoch: evidence.ProtocolEpoch, CompatibleUpgrade: compatible, + ObservedAt: now.UTC(), } return verified, nil } + +// verifyRestoreDispatchBuild 是构建身份的兼容策略: +// - 运行中的二进制必须有注入的**完整 40 位 SHA**,否则 unknown —— 拒绝; +// - 证据的构建 SHA 也必须是完整 SHA; +// - 同一 SHA ⇒ 必须与运行 display 相符(防止同一二进制自称别的版本); +// - 不同 SHA ⇒ 只有 ProtocolEpoch 相同(已在 Validate 里核过)才算兼容重建, +// 并在台账里显式记 compatible_upgrade=true,不静默等同。 +func verifyRestoreDispatchBuild(evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string, bool, error) { + build := version.DisplayVersion() + runningSHA := version.CommitSHA + if !version.HasInjection() || build == "" || build == "dev" || !isFullHexSHA(runningSHA) { + return "", "", false, errors.New("restore dispatch activation requires a build with an injected full commit sha") + } + if !isFullHexSHA(evidence.ProtocolBuildSHA) { + return "", "", false, errors.New("restore dispatch activation evidence carries no full build sha") + } + if evidence.ProtocolEpoch != libraryrelease.RestoreDispatchProtocolEpoch { + return "", "", false, fmt.Errorf("restore dispatch activation build epoch %d is incompatible with %d", + evidence.ProtocolEpoch, libraryrelease.RestoreDispatchProtocolEpoch) + } + if evidence.ProtocolBuildSHA == runningSHA { + if evidence.ProtocolBuild != build { + return "", "", false, fmt.Errorf("restore dispatch activation claims build %q for the running commit, display is %q", + evidence.ProtocolBuild, build) + } + return build, runningSHA, false, nil + } + // 兼容批次内的不同 SHA:允许,但必须记录成"升级",不能伪装成同一个二进制。 + if strings.TrimSpace(evidence.ProtocolBuild) == "" { + return "", "", false, errors.New("restore dispatch activation compatible build has no display identity") + } + return build, runningSHA, true, nil +} + +func isFullHexSHA(value string) bool { + if len(value) != 40 || strings.ToLower(value) != value { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} diff --git a/internal/platform/postgres/library_restore_dispatch_binding.go b/internal/platform/postgres/library_restore_dispatch_binding.go new file mode 100644 index 0000000..be84bdc --- /dev/null +++ b/internal/platform/postgres/library_restore_dispatch_binding.go @@ -0,0 +1,220 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" +) + +// DS01-F2.a:CAS 点的实时租约判据。 +// +// 授权事务里的前几次租约检查发生在**拿 effect 行锁之前**;等锁可以跨过租约到期。 +// 因此拿到最后一个可能阻塞的锁之后必须按数据库时钟重读一次。这里的行都已经在 +// 本事务里持有锁,重读不会新增等待,也不改变 job→operation→maintenance→attempt→effect +// 的锁序。 +type restoreDispatchLeaseExpired struct { + reason string +} + +func (e *restoreDispatchLeaseExpired) Error() string { return e.reason } + +func restoreDispatchLeaseLive(ctx context.Context, tx pgx.Tx, until *time.Time) (bool, error) { + var live bool + if err := tx.QueryRow(ctx, + `SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, until).Scan(&live); err != nil { + return false, fmt.Errorf("read lease clock: %w", err) + } + return live, nil +} + +func recheckRestoreDispatchLeases(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error { + var owner string + var epoch int64 + var leaseUntil *time.Time + err := tx.QueryRow(ctx, ` +SELECT lease_owner, attempt_epoch, lease_until FROM library_jobs WHERE id=$1`, + b.Key.WindowJobID).Scan(&owner, &epoch, &leaseUntil) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("window job %s disappeared", b.Key.WindowJobID) + } + if err != nil { + return fmt.Errorf("recheck window job lease: %w", err) + } + if owner != b.Owner || epoch != b.Epoch { + return &restoreDispatchLeaseExpired{reason: fmt.Sprintf( + "window job lease now belongs to %s#%d, not %s#%d", owner, epoch, b.Owner, b.Epoch)} + } + live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil) + if err != nil { + return err + } + if !live { + return &restoreDispatchLeaseExpired{reason: "window job lease expired while waiting for the dispatch CAS"} + } + + var attemptOwner string + var attemptEpoch int64 + var attemptLease *time.Time + err = tx.QueryRow(ctx, ` +SELECT owner, epoch, lease_until FROM library_attempts +WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan( + &attemptOwner, &attemptEpoch, &attemptLease) + if errors.Is(err, pgx.ErrNoRows) { + return &restoreDispatchLeaseExpired{reason: fmt.Sprintf("no active attempt for operation %s", b.Key.OperationID)} + } + if err != nil { + return fmt.Errorf("recheck active attempt lease: %w", err) + } + if attemptOwner != b.Owner || attemptEpoch != b.Epoch { + return &restoreDispatchLeaseExpired{reason: fmt.Sprintf( + "active attempt is now %s#%d, not %s#%d", attemptOwner, attemptEpoch, b.Owner, b.Epoch)} + } + live, err = restoreDispatchLeaseLive(ctx, tx, attemptLease) + if err != nil { + return err + } + if !live { + return &restoreDispatchLeaseExpired{reason: "attempt lease expired while waiting for the dispatch CAS"} + } + + if b.RecoveryJobID != "" { + var deliveryOwner string + var deliveryAttempt int + var deliveryLease *time.Time + err = tx.QueryRow(ctx, `SELECT lease_owner, attempt, lease_until FROM jobs WHERE id=$1`, + b.RecoveryJobID).Scan(&deliveryOwner, &deliveryAttempt, &deliveryLease) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("recovery queue job %s disappeared", b.RecoveryJobID) + } + if err != nil { + return fmt.Errorf("recheck recovery queue lease: %w", err) + } + worker, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, deliveryAttempt) + if !ok || worker != deliveryOwner { + return &restoreDispatchLeaseExpired{reason: fmt.Sprintf( + "recovery delivery %s attempt %d is not owned by %s", b.RecoveryJobID, deliveryAttempt, b.Owner)} + } + live, err = restoreDispatchLeaseLive(ctx, tx, deliveryLease) + if err != nil { + return err + } + if !live { + return &restoreDispatchLeaseExpired{reason: "recovery queue lease expired while waiting for the dispatch CAS"} + } + } + return nil +} + +// DS01-F2.b:恢复投递身份必须在**同一个 CAS 事务**里锁定并复核: +// queue → operation → 原 window/request_digest,再核当前 owner/attempt/lease。 +// +// 只验"queue 是 running 且 operation 是 recover 类型"会让一条本属于别的窗口的 +// 恢复投递借用本次 effect 的许可;worker 入口的先验校验(ValidateLibraryRecoveryQueue) +// 不能替代 CAS 点的实时授权。 +func lockAndVerifyRecoveryDelivery(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error { + window := b.Key.WindowJobID + if b.RecoveryJobID != governance.LibraryRecoveryQueueID(window) { + return fmt.Errorf("recovery queue %s is not the registered queue for window %s", b.RecoveryJobID, window) + } + var queueType, queueState, payloadRaw, idempotency string + var leaseOwner string + var attempt int + var leaseUntil *time.Time + err := tx.QueryRow(ctx, ` +SELECT type, state, payload::text, COALESCE(lease_owner,''), attempt, lease_until, idempotency_key +FROM jobs WHERE id=$1 FOR UPDATE`, b.RecoveryJobID).Scan( + &queueType, &queueState, &payloadRaw, &leaseOwner, &attempt, &leaseUntil, &idempotency) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("recovery queue job %s does not exist", b.RecoveryJobID) + } + if err != nil { + return fmt.Errorf("lock recovery queue job: %w", err) + } + if queueType != "library.recover" || queueState != "running" { + return fmt.Errorf("recovery queue job %s is not a running delivery (type=%s state=%s)", + b.RecoveryJobID, queueType, queueState) + } + if idempotency != "library.recover:"+window { + return fmt.Errorf("recovery queue %s is not idempotency-bound to window %s", b.RecoveryJobID, window) + } + var payload map[string]string + if err := json.Unmarshal([]byte(payloadRaw), &payload); err != nil { + return fmt.Errorf("recovery queue payload is not a delivery claim: %w", err) + } + if payload["scope"] != b.Scope || payload["window_job_id"] != window { + return fmt.Errorf("recovery queue %s carries window %s/%s, not %s/%s", + b.RecoveryJobID, payload["window_job_id"], payload["scope"], window, b.Scope) + } + // 可选的锚点只要出现就必须与不可变绑定相同;缺省表示"由 journal/写集合决定"。 + for key, want := range map[string]string{ + "source_snapshot_id": b.SourceSnapshot, + "target_id": b.Key.TargetIdentity, + "original_operation_id": b.Key.OperationID, + "backup_snapshot_id": b.BackupSnapshot, + "plan_digest": b.Key.PlanDigest, + } { + if got := payload[key]; got != "" && got != want { + return fmt.Errorf("recovery queue %s declares %s=%s, binding is %s", + b.RecoveryJobID, key, got, want) + } + } + if _, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); !ok { + return fmt.Errorf("recovery queue %s attempt %d is not the current delivery of %s", + b.RecoveryJobID, attempt, b.Owner) + } + if ownerWorker, _ := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); ownerWorker != leaseOwner { + return fmt.Errorf("recovery queue %s is leased by %q, not by the attempt owner %q", + b.RecoveryJobID, leaseOwner, ownerWorker) + } + live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil) + if err != nil { + return err + } + if !live { + return fmt.Errorf("recovery queue %s lease is expired", b.RecoveryJobID) + } + + var recoveryKind, recoveryScope, recoveryTarget, requestDigest, operationTarget string + err = tx.QueryRow(ctx, ` +SELECT kind, scope, target_snapshot_id, request_digest, COALESCE(target_id,'') +FROM library_operations WHERE id=$1 FOR UPDATE`, b.RecoveryOperationID).Scan( + &recoveryKind, &recoveryScope, &recoveryTarget, &requestDigest, &operationTarget) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("recovery operation %s does not exist", b.RecoveryOperationID) + } + if err != nil { + return fmt.Errorf("lock recovery operation: %w", err) + } + if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope { + return fmt.Errorf("recovery operation %s is not a recover operation for scope %s", b.RecoveryOperationID, b.Scope) + } + if recoveryTarget != b.BackupSnapshot { + return fmt.Errorf("recovery operation %s restores %s, not the window backup %s", + b.RecoveryOperationID, recoveryTarget, b.BackupSnapshot) + } + if operationTarget != "" && operationTarget != b.Key.TargetIdentity { + return fmt.Errorf("recovery operation %s targets %s, not %s", + b.RecoveryOperationID, operationTarget, b.Key.TargetIdentity) + } + if requestDigest == "" { + return fmt.Errorf("recovery operation %s has no request digest anchor", b.RecoveryOperationID) + } + // 恢复请求摘要只能由原 window 的不可变锚点重算:queue/operation/window/原 operation/ + // 目标/来源/备份/写集合摘要。换任何一项都会得到另一份摘要。 + expected, err := governance.LibraryRecoveryDigest(b.Scope, b.RecoveryJobID, window, b.Key.OperationID, + b.Key.TargetIdentity, b.SourceSnapshot, b.BackupSnapshot, b.Key.PlanDigest) + if err != nil { + return fmt.Errorf("recompute recovery request digest: %w", err) + } + if requestDigest != expected { + return fmt.Errorf("recovery operation %s request digest is bound to another window/plan", b.RecoveryOperationID) + } + return nil +} diff --git a/internal/platform/postgres/library_restore_dispatch_final_binding_test.go b/internal/platform/postgres/library_restore_dispatch_final_binding_test.go new file mode 100644 index 0000000..493ff6c --- /dev/null +++ b/internal/platform/postgres/library_restore_dispatch_final_binding_test.go @@ -0,0 +1,316 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/version" +) + +// DS01-F2.b:恢复投递必须在 CAS 事务里与 queue→operation→原 window/摘要绑定。 +func TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction(t *testing.T) { + type fixture struct { + store *Store + pool *pgxpool.Pool + binding libraryrelease.RestoreMarkerBinding + } + seeded := func(t *testing.T, jobID string) fixture { + t.Helper() + store, pool := newIdentityStore(t) + now := time.Now().UTC() + windowOp := seedOperationJob(t, store, pool, "internal", jobID, "sync", now) + queueID := governance.LibraryRecoveryQueueID(jobID) + owner := governance.LibraryDeliveryOwner("worker-r", queueID, 1) + if _, err := store.BeginLibraryAttempt(context.Background(), jobID, owner, 10*time.Minute, now); err != nil { + t.Fatalf("begin attempt: %v", err) + } + payload, _ := json.Marshal(map[string]string{"scope": "internal", "window_job_id": jobID}) + if _, err := pool.Exec(context.Background(), ` +INSERT INTO jobs (id, type, payload, state, lease_owner, lease_until, attempt, idempotency_key) +VALUES ($1,'library.recover',$2::jsonb,'running','worker-r',$3,1,$4)`, + queueID, string(payload), now.Add(10*time.Minute), "library.recover:"+jobID); err != nil { + t.Fatalf("seed recovery queue: %v", err) + } + recoveryOp := governance.LibraryOperationID("internal", governance.LibraryJobRecover, queueID) + digest, err := governance.LibraryRecoveryDigest("internal", queueID, jobID, windowOp, "target-ds01", + "snap-target", "snap-backup", "plan-1") + if err != nil { + t.Fatal(err) + } + if _, _, err := store.EnsureLibraryOperation(context.Background(), governance.LibraryOperation{ + ID: recoveryOp, Scope: "internal", Kind: governance.LibraryJobRecover, + TargetSnapshotID: "snap-backup", RequestDigest: digest, OperationKey: queueID, + Actor: "worker-r", CreatedAt: now, + }); err != nil { + t.Fatalf("seed recovery operation: %v", err) + } + binding := ds01Binding() + binding.Key.WindowJobID = jobID + binding.Key.OperationID = windowOp + binding.RecoveryJobID = queueID + binding.RecoveryOperationID = recoveryOp + binding.Owner = owner + binding.Epoch = 1 + if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil { + t.Fatalf("activate: %v", err) + } + return fixture{store: store, pool: pool, binding: binding} + } + expectRejected := func(t *testing.T, fx fixture) { + t.Helper() + if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err == nil { + t.Fatal("recovery delivery mismatch was accepted") + } + } + + t.Run("positive real recovery delivery prepares and acquires", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-ok") + if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err != nil { + t.Fatalf("valid recovery delivery: %v", err) + } + permit, _, err := fx.store.AcquireRestoreMarker(context.Background(), fx.binding) + if err != nil || permit == nil { + t.Fatalf("valid recovery delivery acquire: permit=%v err=%v", permit, err) + } + }) + t.Run("queue for another window is rejected", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-wrongqueue") + fx.binding.RecoveryJobID = governance.LibraryRecoveryQueueID("another-window") + expectRejected(t, fx) + }) + t.Run("payload window differs", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-wrongwindow") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE jobs SET payload=jsonb_set(payload,'{window_job_id}','"other-window"') WHERE id=$1`, + fx.binding.RecoveryJobID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("request digest belongs to another plan", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-wrongdigest") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE library_operations SET request_digest='digest-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("queue leased by another worker", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-owner") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE jobs SET lease_owner='intruder' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("expired queue lease", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-expired") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE jobs SET lease_until=now()-interval '1 minute' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("operation restores another backup", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-backup") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE library_operations SET target_snapshot_id='snap-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("binding plan differs from the recovery request", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-plan") + fx.binding.Key.PlanDigest = "plan-other" + expectRejected(t, fx) + }) + t.Run("queue is not running", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-state") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE jobs SET state='pending', lease_owner='', lease_until=NULL WHERE id=$1`, fx.binding.RecoveryJobID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) + t.Run("operation is not a recovery", func(t *testing.T) { + fx := seeded(t, "libsync-ds01-recovery-kind") + if _, err := fx.pool.Exec(context.Background(), + `UPDATE library_operations SET kind='sync' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil { + t.Fatal(err) + } + expectRejected(t, fx) + }) +} + +// DS01-F3.identity:激活与使用点都必须核治理权威域(集群+库 OID)、primary 与完整 +// 构建 SHA;同 display 不同不兼容 SHA、unknown、同名错域、错目标一律零激活/零许可。 +func TestRestoreDispatchGovernanceAuthorityAndBuildIdentity(t *testing.T) { + store, pool := newIdentityStore(t) + // 使用点会读运行构建身份:测试进程由 runner 注入完整 SHA,这里核一下前提。 + if !version.HasInjection() || len(version.CommitSHA) != 40 { + t.Skip("requires an injected full commit sha (run via the version-injecting runner)") + } + ctx := context.Background() + now := time.Now().UTC() + job := "authority-window" + if _, err := pool.Exec(ctx, ` +INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, + services, ingress_state, state, original_service_state, note, acquired_at, updated_at) +VALUES ($1,$2,'internal',$3,'instance',$4::jsonb,'closed','needs_restore','','',now()-interval '20 minutes',now())`, + ds01TestInstance, job, ds01TestFingerprint, `["`+ds01TestWriter+`"]`); err != nil { + t.Fatalf("seed maintenance window: %v", err) + } + dir, _, journalDigest := ds01Journal(t, job, nil) + toStore := store.WithRestoreDispatchEvidence(dir) + target := "target-ds01-authority" + base := func() libraryrelease.RestoreDispatchActivationEvidence { + return ds01ActivationEvidence(t, store, target, ds01TestFingerprint, job, journalDigest) + } + activate := func(evidence libraryrelease.RestoreDispatchActivationEvidence) error { + t.Helper() + path, digest := writeDS01ActivationEvidence(t, evidence) + return toStore.ActivateRestoreDispatch(ctx, path, digest, target, ds01TestFingerprint, now) + } + count := func() int { + t.Helper() + var n int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch_activation`).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + + // 1) 同名库、不同集群:库名相同但 system_identifier 不同。 + evidence := base() + evidence.GovernanceAuthority.SystemIdentifier = "1" + if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") { + t.Fatalf("same-name other-cluster evidence = %v", err) + } + // 2) 同集群、另一个同名库(OID 不同)。 + evidence = base() + evidence.GovernanceAuthority.DatabaseOID = "1" + if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") { + t.Fatalf("same-name other-database evidence = %v", err) + } + // 3) unknown 构建身份。 + evidence = base() + evidence.ProtocolBuildSHA = "unknown" + if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "commit sha") { + t.Fatalf("unknown build sha evidence = %v", err) + } + // 4) 同 display、不同 SHA、不同协议批次:不兼容,必须拒绝。 + evidence = base() + evidence.ProtocolBuild = version.DisplayVersion() + evidence.ProtocolBuildSHA = strings.Repeat("b", 40) + evidence.ProtocolEpoch = libraryrelease.RestoreDispatchProtocolEpoch - 1 + if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "epoch") { + t.Fatalf("same-display incompatible build = %v", err) + } + if n := count(); n != 0 { + t.Fatalf("rejected evidence created %d activation rows", n) + } + + // 5) 合法兼容升级:不同完整 SHA、同一协议批次 ⇒ 允许,但必须显式记成升级。 + compat := base() + compat.ProtocolBuild = "0.1.999-compatible" + compat.ProtocolBuildSHA = strings.Repeat("a", 40) + if err := activate(compat); err != nil { + t.Fatalf("compatible upgrade evidence: %v", err) + } + var stored string + if err := pool.QueryRow(ctx, `SELECT evidence::text FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, target).Scan(&stored); err != nil { + t.Fatal(err) + } + compact := strings.ReplaceAll(stored, " ", "") + if !strings.Contains(compact, `"compatible_upgrade":true`) || + !strings.Contains(compact, `"system_identifier":"`+compat.GovernanceAuthority.SystemIdentifier+`"`) { + t.Fatalf("activation record does not state the verified authority/upgrade: %s", stored) + } + + // 使用点正例:同一兼容批次下真实窗口可以准备/取得许可。 + windowOp := seedOperationJob(t, store, pool, "internal", "libsync-ds01-authority", "sync", now) + if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-authority", "worker-a#q1#0", 10*time.Minute, now); err != nil { + t.Fatal(err) + } + binding := ds01Binding() + binding.Key.WindowJobID = "libsync-ds01-authority" + binding.Key.OperationID = windowOp + binding.Key.TargetIdentity = target + if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil { + t.Fatalf("use point rejected a compatible build: %v", err) + } + if permit, _, err := store.AcquireRestoreMarker(ctx, binding); err != nil || permit == nil { + t.Fatalf("use point acquire under compatible build: permit=%v err=%v", permit, err) + } + + // 使用点反例:把登记行篡改成错域/错批次/unknown 构建,Prepare 必须零许可失败。 + tamperTargets := map[string]func(map[string]any){ + "target-ds01-tamper-domain": func(v map[string]any) { + authority, _ := v["governance_authority"].(map[string]any) + authority["system_identifier"] = "1" + }, + "target-ds01-tamper-epoch": func(v map[string]any) { + v["protocol_epoch"] = float64(libraryrelease.RestoreDispatchProtocolEpoch - 1) + }, + "target-ds01-tamper-sha": func(v map[string]any) { + v["protocol_build_sha"] = "unknown" + }, + } + for tamperTarget, mutate := range tamperTargets { + var record map[string]any + if err := json.Unmarshal([]byte(stored), &record); err != nil { + t.Fatal(err) + } + verified, _ := record["verified"].(map[string]any) + mutate(verified) + raw, err := json.Marshal(record) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO library_index_restore_dispatch_activation + (target_identity, protocol_version, target_fingerprint, target_instance, governance_database, + protocol_build, activated_by, drain_evidence_digest, evidence, attested_at, activated_at) +SELECT $1, protocol_version, target_fingerprint, target_instance, governance_database, + protocol_build, activated_by, drain_evidence_digest, $2::jsonb, attested_at, activated_at +FROM library_index_restore_dispatch_activation WHERE target_identity=$3`, + tamperTarget, string(raw), target); err != nil { + t.Fatalf("insert tampered activation %s: %v", tamperTarget, err) + } + tampered := binding + tampered.Key.TargetIdentity = tamperTarget + if permit, _, err := store.AcquireRestoreMarker(ctx, tampered); permit != nil || err == nil { + t.Fatalf("use point accepted tampered activation %s: permit=%v err=%v", tamperTarget, permit, err) + } + } + + // 使用点错目标:同一激活行但本次观察的目标指纹不同。 + wrongTarget := binding + wrongTarget.Key.DocumentID = "doc-other-target" + wrongTarget.TargetFingerprint = ds01TestFingerprintOther + if permit, _, err := store.AcquireRestoreMarker(ctx, wrongTarget); permit != nil || err == nil { + t.Fatalf("use point accepted another target fingerprint: permit=%v err=%v", permit, err) + } + + // 运行构建 unknown:即使已有合法登记,使用点也必须失败关闭。 + originalSHA := version.CommitSHA + version.CommitSHA = "unknown" + t.Cleanup(func() { version.CommitSHA = originalSHA }) + unknown := binding + unknown.Key.DocumentID = "doc-unknown-build" + permit, _, err := store.AcquireRestoreMarker(ctx, unknown) + if permit != nil || err == nil { + t.Fatalf("use point accepted an unknown running build: permit=%v err=%v", permit, err) + } + if errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) { + t.Log("unknown build rejected as held (expected fail-closed)") + } +} diff --git a/internal/platform/postgres/library_restore_dispatch_linearization_test.go b/internal/platform/postgres/library_restore_dispatch_linearization_test.go index 930d1c9..aa7b1fd 100644 --- a/internal/platform/postgres/library_restore_dispatch_linearization_test.go +++ b/internal/platform/postgres/library_restore_dispatch_linearization_test.go @@ -190,6 +190,63 @@ func TestRestoreDispatchLeaseExpiryWhileWaitingForLockIsRejected(t *testing.T) { } } +// 等锁跨过租约(effect 行):前面的检查发生在拿 effect 锁之前,全部通过;真正等 +// effect 行锁时租约到期。CAS 点必须按数据库实时时钟重核,零许可、状态保持 prepared。 +func TestRestoreDispatchLeaseExpiryWhileWaitingForEffectLockIsRejected(t *testing.T) { + store, pool := newIdentityStore(t) + ctx := context.Background() + now := time.Now().UTC() + seedOperationJob(t, store, pool, "internal", "libsync-ds01-effect-lease", "sync", now) + if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-effect-lease", "worker-a#q1#0", 1500*time.Millisecond, now); err != nil { + t.Fatal(err) + } + if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil { + t.Fatal(err) + } + binding := ds01Binding() + binding.Key.WindowJobID = "libsync-ds01-effect-lease" + binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID) + if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil { + t.Fatal(err) + } + lock, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := lock.Exec(ctx, `SELECT id FROM library_index_restore_dispatch WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`, + binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest, + binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil { + t.Fatal(err) + } + type result struct { + permit *libraryrelease.RestoreSendPermit + err error + } + done := make(chan result, 1) + go func() { + permit, _, err := store.AcquireRestoreMarker(ctx, binding) + done <- result{permit, err} + }() + time.Sleep(2200 * time.Millisecond) + if err := lock.Rollback(ctx); err != nil { + t.Fatal(err) + } + got := <-done + if got.permit != nil { + t.Fatal("permit issued although the attempt lease expired while waiting for the effect row lock") + } + if got.err == nil || !strings.Contains(got.err.Error(), "expired while waiting for the dispatch CAS") { + t.Fatalf("expired effect-lock wait error = %v", got.err) + } + var state string + if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil { + t.Fatal(err) + } + if state != libraryrelease.RestoreDispatchPrepared { + t.Fatalf("state = %s after a rejected CAS", state) + } +} + // 错目标/错来源/换 plan 别名/错 UUID 都必须在 CAS 前失败,且台账不动。 func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) { store, pool := newIdentityStore(t) @@ -238,38 +295,43 @@ func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) { if count != 1 { t.Fatalf("dispatch rows = %d, want exactly the original plan row", count) } - // 恢复投递身份缺失/错误:Prepare 就必须失败,不能等到发送前。 + // 恢复投递身份缺失:Prepare 就必须失败,不能等到发送前。 recovery := binding recovery.RecoveryJobID = "queue-missing" recovery.RecoveryOperationID = "op-missing" if _, err := store.PrepareRestoreMarker(ctx, recovery); err == nil { t.Fatal("unknown recovery delivery identity was accepted") } - // 真实的 recovery queue/operation 绑定可以被验证通过;换成另一份 backup 的 - // recovery operation 则必须拒绝。 + // 形状合法但与本 window 无关的 recovery queue/operation:同样零许可。 + // 真正绑定成功的正例在 TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction。 + queueID := governance.LibraryRecoveryQueueID(binding.Key.WindowJobID) if _, err := pool.Exec(ctx, `INSERT INTO jobs (id, type, payload, state, lease_owner, idempotency_key) -VALUES ('queue-recovery-1','library.recover','{}'::jsonb,'running','worker-a','queue-recovery-1')`); err != nil { +VALUES ($1,'library.recover',$2::jsonb,'running','worker-a',$3)`, + "queue-recovery-unrelated", `{"scope":"internal","window_job_id":"another-window"}`, + "library.recover:another-window"); err != nil { t.Fatal(err) } if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ - ID: "op-recovery-1", Scope: "internal", Kind: governance.LibraryJobRecover, - TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-1", + ID: "op-recovery-unrelated", Scope: "internal", Kind: governance.LibraryJobRecover, + TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-unrelated", Actor: "tester", CreatedAt: now.UTC(), }); err != nil { t.Fatal(err) } - validRecovery := binding - validRecovery.Key.DocumentID = "doc-recovery" - validRecovery.RecoveryJobID = "queue-recovery-1" - validRecovery.RecoveryOperationID = "op-recovery-1" - if _, err := store.PrepareRestoreMarker(ctx, validRecovery); err != nil { - t.Fatalf("valid recovery delivery binding: %v", err) + unrelated := binding + unrelated.Key.DocumentID = "doc-recovery" + unrelated.RecoveryJobID = "queue-recovery-unrelated" + unrelated.RecoveryOperationID = "op-recovery-unrelated" + if _, err := store.PrepareRestoreMarker(ctx, unrelated); err == nil { + t.Fatal("recovery delivery for another window was accepted") } - wrongRecovery := validRecovery - wrongRecovery.Key.DocumentID = "doc-recovery-2" - wrongRecovery.RecoveryOperationID = seededOperationID(t, pool, binding.Key.WindowJobID) - if _, err := store.PrepareRestoreMarker(ctx, wrongRecovery); err == nil { - t.Fatal("recovery operation for another intent was accepted") + // 正确 queue 但 recovery operation 指向原业务 operation:不是恢复请求,必须拒绝。 + wrongOperation := binding + wrongOperation.Key.DocumentID = "doc-recovery-2" + wrongOperation.RecoveryJobID = queueID + wrongOperation.RecoveryOperationID = binding.Key.OperationID + if _, err := store.PrepareRestoreMarker(ctx, wrongOperation); err == nil { + t.Fatal("non-recovery operation was accepted as a recovery delivery") } } diff --git a/internal/platform/postgres/library_restore_dispatch_test.go b/internal/platform/postgres/library_restore_dispatch_test.go index c488d8b..1137f37 100644 --- a/internal/platform/postgres/library_restore_dispatch_test.go +++ b/internal/platform/postgres/library_restore_dispatch_test.go @@ -433,22 +433,42 @@ func ds01JournalRowJob(t *testing.T, store *Store) string { return job } -// ds01ActivationEvidence 组装 v2 证据:只给身份/作业/摘要/writer 集合。 +// ds01ActivationEvidence 组装 v3 证据:身份/作业/摘要/writer 集合 + 治理权威域 + 构建 SHA。 func ds01ActivationEvidence(t *testing.T, store *Store, targetIdentity, fingerprint, job, journalDigest string) libraryrelease.RestoreDispatchActivationEvidence { t.Helper() - var database string - if err := store.db.QueryRow(context.Background(), `SELECT current_database()`).Scan(&database); err != nil { - t.Fatalf("read governance database: %v", err) + authority, primary, database := ds01ObservedAuthority(t, store) + if !primary { + t.Fatal("governance database is not primary") } return libraryrelease.RestoreDispatchActivationEvidence{ - ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint, - RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(), + ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion, + ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch, + TargetIdentity: targetIdentity, TargetFingerprint: fingerprint, + RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database, + GovernanceAuthority: authority, ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA, AttestedBy: "ds01-test", AttestedAt: time.Now().UTC(), MaintenanceJob: job, MaintenanceJournal: job + ".json", MaintenanceJournalSHA256: journalDigest, Writers: []libraryrelease.RestoreDispatchWriterDrain{{Name: ds01TestWriter, Kind: "worker", InstanceID: ds01TestWriter}}, } } +// ds01ObservedAuthority 读回测试进程实际连接的治理库权威域;证据必须携带它, +// 否则"同名异集群"永远无法被拒绝。 +func ds01ObservedAuthority(t *testing.T, store *Store) (libraryrelease.RestoreDispatchGovernanceAuthority, bool, string) { + t.Helper() + var authority libraryrelease.RestoreDispatchGovernanceAuthority + var inRecovery bool + var database string + if err := store.db.QueryRow(context.Background(), ` +SELECT current_database(), + (SELECT system_identifier::text FROM pg_control_system()), + (SELECT oid::text FROM pg_database WHERE datname = current_database()), + pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery); err != nil { + t.Fatalf("observe governance authority: %v", err) + } + return authority, !inRecovery, database +} + func writeDS01ActivationEvidence(t *testing.T, evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string) { t.Helper() raw, err := json.Marshal(evidence) diff --git a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go index 91ea8b3..8414513 100644 --- a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go +++ b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go @@ -434,9 +434,16 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance) if expected := os.Getenv("KG_R38_VERSION"); expected != "" && build != expected { return "", nil, fmt.Errorf("test binary build %q != runner identity %q", build, expected) } - var database string - if err := gov.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil { - return "", nil, fmt.Errorf("read governance database: %w", err) + var database, systemIdentifier, databaseOID string + if err := gov.QueryRow(ctx, ` +SELECT current_database(), + (SELECT system_identifier::text FROM pg_control_system()), + (SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan( + &database, &systemIdentifier, &databaseOID); err != nil { + return "", nil, fmt.Errorf("read governance authority: %w", err) + } + if expected := os.Getenv("KG_R38_COMMIT"); expected != "" && version.CommitSHA != expected { + return "", nil, fmt.Errorf("test binary commit %q != runner identity %q", version.CommitSHA, expected) } journalSum := sha256.Sum256(rawJournal) // 激活后同一窗口还会继续写 journal(recovered 等);把激活那一刻的字节原样留档, @@ -447,8 +454,12 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance) } now := time.Now().UTC() evidence := libraryrelease.RestoreDispatchActivationEvidence{ - ProtocolVersion: 2, TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint, - RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database, ProtocolBuild: build, + ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion, + ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch, + TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint, + RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database, + GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID}, + ProtocolBuild: build, ProtocolBuildSHA: version.CommitSHA, AttestedBy: "r38-runner", AttestedAt: now, MaintenanceJob: job.ID, MaintenanceJournal: job.ID + ".json", MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]), Writers: writers, @@ -480,7 +491,10 @@ FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, cfg.Li facts := map[string]any{ "target": target, "fingerprint": fingerprint, "window_fingerprint": windowFingerprint, "governance_database": database, - "maintenance_job": job.ID, "protocol_build": build, + "governance_authority": map[string]string{"system_identifier": systemIdentifier, "database_oid": databaseOID, + "primary": "true"}, + "protocol_build_sha": version.CommitSHA, + "maintenance_job": job.ID, "protocol_build": build, "journal_sha256": drainDigest, "journal_snapshot": filepath.Base(journalSnapshot), "activation_evidence_sha256": hex.EncodeToString(evidenceSum[:]), "journal_state": journal["state"], "gate": gate, "quiesce": quiesce, diff --git a/internal/releaseadapter/v071/library_index_ds01_fixes_test.go b/internal/releaseadapter/v071/library_index_ds01_fixes_test.go index 6e77ebb..41fdb2a 100644 --- a/internal/releaseadapter/v071/library_index_ds01_fixes_test.go +++ b/internal/releaseadapter/v071/library_index_ds01_fixes_test.go @@ -16,6 +16,7 @@ type ds01StubGate struct { inspectStatus libraryrelease.RestoreDispatchStatus inspectFound bool records []libraryrelease.RestoreDispatchRecord + confirmed int } func (g *ds01StubGate) InspectRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) { @@ -28,6 +29,7 @@ func (g *ds01StubGate) AcquireRestoreMarker(context.Context, libraryrelease.Rest return nil, g.inspectStatus, libraryrelease.ErrRestoreDispatchHeld } func (g *ds01StubGate) ConfirmRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding, []byte) error { + g.confirmed++ return nil } func (g *ds01StubGate) UnresolvedRestoreDispatches(context.Context, string, string, string) ([]libraryrelease.RestoreDispatchRecord, error) { @@ -127,6 +129,102 @@ func TestDS01BindingConflictHeldBeforeAnyWrite(t *testing.T) { } } +// DS01-F4.binding:绑定冲突是**类型化**分类——prepared/dispatching/marker_confirmed/ +// done 任一状态、准入/确认/收尾任一入口都必须拒绝;合法 dispatching + 精确 marker +// (不冲突)仍允许确认,不能用"拒绝全部 Held"掩盖。 +func TestDS01BindingConflictRejectedInEveryState(t *testing.T) { + fx, snapshot, target, backups, wire := ds01Fixture(t) + authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot) + authority.attach(target) + outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire) + ledger := outcome.Backup.WriteSet + if err := target.AdoptWriteSet(*ledger); err != nil { + t.Fatal(err) + } + ds01FreezeBindingInputs(t, target, outcome.Backup) + change := indexChangeOf(t, ledger, "chunk-target") + body, err := target.restoreIndexClaimBody(change) + if err != nil { + t.Fatal(err) + } + // 在目标上写入精确 marker:这样 dispatching 状态本来是可确认的,冲突检查必须 + // 独立地拒绝它,而不是靠"marker 不在"。 + applied, err := target.loadIndexEffect(change, "applied") + if err != nil || applied == nil || applied.Receipt == nil { + t.Fatalf("applied receipt missing: %v", err) + } + method, path, payload, err := target.indexMutation(change, body, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, true) + if err != nil { + t.Fatal(err) + } + var marker indexReceipt + if err := target.search.requestJSON(fx.ctx, method, path, payload, &marker); err != nil { + t.Fatalf("exact marker write: %v", err) + } + observed, err := target.observePlannedIndex(fx.ctx, change) + if err != nil { + t.Fatal(err) + } + conflict := func(state string) *ds01StubGate { + return &ds01StubGate{inspectFound: true, inspectStatus: libraryrelease.RestoreDispatchStatus{ + State: state, Held: true, BindingConflict: true, Reason: "binding differs"}, + records: []libraryrelease.RestoreDispatchRecord{{PlanDigest: ledger.Digest, Index: change.Index, + DocumentID: change.ID, State: libraryrelease.RestoreDispatchMarkerConfirmed}}} + } + for _, state := range []string{ + libraryrelease.RestoreDispatchPrepared, + libraryrelease.RestoreDispatchDispatching, + libraryrelease.RestoreDispatchMarkerConfirmed, + } { + gate := conflict(state) + target.config.RestoreDispatch = gate + if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") { + t.Fatalf("admission accepted a binding conflict in %s: %v", state, err) + } + if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err == nil || !strings.Contains(err.Error(), "冲突") { + t.Fatalf("confirmation accepted a binding conflict in %s: %v", state, err) + } + if gate.confirmed != 0 { + t.Fatalf("binding conflict in %s still recorded a confirmation", state) + } + } + + // done 状态:restored 文件已在,准入仍必须因绑定冲突拒绝(旧实现只看状态)。 + receipt := indexReceipt{Index: change.Index, ID: change.ID, Result: "created", + SeqNo: marker.SeqNo, PrimaryTerm: marker.PrimaryTerm} + if err := target.saveIndexEffect(change, "restored", &receipt); err != nil { + t.Fatal(err) + } + target.config.RestoreDispatch = conflict(libraryrelease.RestoreDispatchMarkerConfirmed) + if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") { + t.Fatalf("admission accepted a binding conflict on a completed effect: %v", err) + } + // 收尾:记录状态看起来是 marker_confirmed、restored 也存在,但绑定冲突必须拒绝。 + gate := conflict(libraryrelease.RestoreDispatchMarkerConfirmed) + target.config.RestoreDispatch = gate + if err := target.verifyRestoreEffectsClosed(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") { + t.Fatalf("closeout accepted a binding conflict: %v", err) + } + if gate.confirmed != 0 { + t.Fatal("closeout recorded a confirmation for a conflicting binding") + } + + // 正例:合法 dispatching + 精确 marker(不冲突)仍可确认,且只确认一次。 + okGate := &ds01StubGate{inspectFound: true, + inspectStatus: libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true, + Reason: "dispatch outcome unknown: no further create is permitted"}} + target.config.RestoreDispatch = okGate + if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err != nil { + t.Fatalf("legitimate dispatching marker confirmation: %v", err) + } + if okGate.confirmed != 1 { + t.Fatalf("legitimate marker confirmation count = %d", okGate.confirmed) + } + if wire.dispatch != 0 { + t.Fatalf("confirmation path forwarded %d marker writes", wire.dispatch) + } +} + // DS01-F4:prepared 台账却已在目标上出现精确 marker —— 证据矛盾,必须 held; // 旧实现会把它当成"已存在的标记"认领并 clean。 func TestDS01PreparedWithMarkerIsContradictionHeld(t *testing.T) { diff --git a/internal/releaseadapter/v071/library_index_ds01_test.go b/internal/releaseadapter/v071/library_index_ds01_test.go index bb86c7d..ee1a2d3 100644 --- a/internal/releaseadapter/v071/library_index_ds01_test.go +++ b/internal/releaseadapter/v071/library_index_ds01_test.go @@ -549,8 +549,12 @@ func TestDS01RestoreHelper(t *testing.T) { // 测试替身只负责造这两个真实来源,不替产品做判断。 func activateDS01(ctx context.Context, store *postgresstore.Store, pool *pgxpool.Pool, targetIdentity, fingerprint string) error { - var database, job string - if err := pool.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil { + var database, systemIdentifier, databaseOID, job string + if err := pool.QueryRow(ctx, ` +SELECT current_database(), + (SELECT system_identifier::text FROM pg_control_system()), + (SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan( + &database, &systemIdentifier, &databaseOID); err != nil { return err } if err := pool.QueryRow(ctx, `SELECT owner_job_id FROM library_instance_maintenance WHERE instance=$1`, @@ -599,8 +603,12 @@ WHERE instance=$1`, ds01TargetInstance, `["worker-ds01"]`, fingerprint); err != return err } evidence := libraryrelease.RestoreDispatchActivationEvidence{ - ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint, - RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(), + ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion, + ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch, + TargetIdentity: targetIdentity, TargetFingerprint: fingerprint, + RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database, + GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID}, + ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA, AttestedBy: "ds01-test", AttestedAt: now, MaintenanceJob: job, MaintenanceJournal: job + ".json", MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]), diff --git a/internal/releaseadapter/v071/library_index_restore_claim.go b/internal/releaseadapter/v071/library_index_restore_claim.go index 071e4ce..d275cdd 100644 --- a/internal/releaseadapter/v071/library_index_restore_claim.go +++ b/internal/releaseadapter/v071/library_index_restore_claim.go @@ -137,6 +137,9 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if !found { return fmt.Errorf("%w: 既存意图没有派发台账(旧协议未知历史)", libraryrelease.ErrRestoreDispatchHeld) } + if status.BindingConflict { + return heldRestoreMarker(status, "既存意图的派发台账绑定与本次计划冲突") + } if status.State == libraryrelease.RestoreDispatchPrepared && claimed != nil { return heldRestoreMarker(status, "prepared 台账却已有 claimed 证据:协议证据矛盾") } diff --git a/internal/releaseadapter/v071/library_index_restore_dispatch.go b/internal/releaseadapter/v071/library_index_restore_dispatch.go index 8cf3ba0..00213c5 100644 --- a/internal/releaseadapter/v071/library_index_restore_dispatch.go +++ b/internal/releaseadapter/v071/library_index_restore_dispatch.go @@ -161,6 +161,12 @@ func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error { return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr) } if found { + // 绑定冲突是类型化分类:任何状态(含已完成 clean 的终态)都必须拒绝。 + // 它不能与"仅禁止第二次 create"的 dispatching-held 混为一谈——后者在 + // 精确 marker 仍在时是允许确认的。 + if status.BindingConflict { + return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突") + } if done != nil { if status.State != libraryrelease.RestoreDispatchMarkerConfirmed { return heldRestoreMarker(status, "已完成 clean 的 effect 台账却不是 marker_confirmed") @@ -241,6 +247,9 @@ func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c if !found { return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld) } + if status.BindingConflict { + return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突") + } if status.Held && status.State == libraryrelease.RestoreDispatchPrepared { return heldRestoreMarker(status, "派发绑定与不可变计划冲突") } @@ -370,6 +379,28 @@ func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context if done == nil { return fmt.Errorf("%w: 收尾前仍有未 clean 的 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID) } + // 收尾同样是**使用点**:绑定冲突即使状态看起来是 marker_confirmed 也必须拒绝。 + body, err := t.restoreIndexClaimBody(c) + if err != nil { + return err + } + binding, err := t.restoreMarkerBinding(ctx, c, body) + if err != nil { + return err + } + status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding) + if err != nil { + return fmt.Errorf("恢复收尾读不到派发台账:%w", err) + } + if !found { + return fmt.Errorf("%w: 收尾时 effect %s 的派发台账消失了", libraryrelease.ErrRestoreDispatchHeld, c.ID) + } + if status.BindingConflict { + return heldRestoreMarker(status, "收尾时派发台账的不可变绑定与本次计划冲突") + } + if status.State != libraryrelease.RestoreDispatchMarkerConfirmed { + return fmt.Errorf("%w: 收尾时 effect %s 的派发状态是 %s", libraryrelease.ErrRestoreDispatchHeld, c.ID, status.State) + } } } for _, c := range t.activePlan.Index {