fix(library): 收紧 DS01 派发租约/投递/治理身份与绑定冲突

F2.a:Acquire/Confirm 在取得 effect 行锁后用数据库时钟重核 job/attempt/
恢复投递租约,等锁跨过到期零许可且状态保持 prepared。

F2.b:同一 CAS 事务锁 recovery queue 与 operation,复核 window/scope、
idempotency、owner/attempt/lease,并按原 window 锚点重算恢复请求摘要。

F3.identity:激活证据 v3 携带 PG 集群 system_identifier + 库 OID、完整构建
SHA 与协议批次;激活与使用点都现场重读并拒绝同名异域/副本/unknown/不兼容 SHA。

F4.binding:区分绑定冲突与仅禁止重发的 Held,准入/确认/续作/收尾在所有状态
拒绝冲突,合法 dispatching + 精确 marker 仍可确认。
This commit is contained in:
姚章浩
2026-11-02 20:36:36 +08:00
parent 1e2bb385c7
commit 245cefd6b0
16 changed files with 1125 additions and 92 deletions
+24
View File
@@ -1,5 +1,29 @@
# Changelog
## 开发快照(DS01 收口:CAS 点实时租约、恢复投递归属、治理权威域与绑定冲突分类)
### Fixed
- **F2.a CAS 点实时租约**:`Acquire` 在取得 effect 行锁(最后一个可能阻塞的锁)之后、
状态 CAS 之前,按数据库 `clock_timestamp()` 重核 job/attempt/恢复投递租约;`Confirm`
同样先锁 effect 行再核。等锁跨过到期一律零许可、状态保持 `prepared`。新增真 PG
反例:等 effect 行锁期间租约过期。
- **F2.b 恢复投递同事务绑定**:同一短事务内 `FOR UPDATE` 锁 queue 与 recovery operation,
复核 queue id/type/state/payload 的 window+scope、idempotency key、当前 owner/attempt/
lease,并按原 window 的不可变锚点重算 `LibraryRecoveryDigest`,与 operation 的
`request_digest` 逐位比较;错 queue/window/digest/owner/过期/错备份/错 plan 全部零许可。
投递 owner 合成规则收敛到 `governance.LibraryDeliveryOwner` 一份。
- **F3.identity 治理权威域与构建身份**:激活证据升至 v3,必须携带 PG 集群
`system_identifier` + 库 OID、完整 40 位构建 SHA 与协议兼容批次;激活与每次使用点都
现场重读 `pg_control_system()`/`pg_database.oid`/`pg_is_in_recovery()` 并比对,拒绝
同名异域、副本、unknown 构建、同 display 不同不兼容 SHA 与错目标指纹。同一兼容批次
内的不同完整 SHA 记为 `compatible_upgrade`,不静默等同。
- **F4.binding 绑定冲突类型化**:`RestoreDispatchStatus.BindingConflict` 与"仅禁止重发"
的 `Held` 分开;准入、确认、既有意图续作与收尾在 prepared/dispatching/marker_confirmed/
done 任一状态都拒绝绑定冲突;合法 dispatching + 精确 marker 仍可确认一次。
- 真 PG/adapter 反例:effect 锁等待过期、恢复投递九类错配、错治理域/错批次/unknown、
四状态绑定冲突与合法 marker 正例。
## 开发快照(DS01 激活证据 v2:只认真实维护 journal)
### Fixed
+3 -5
View File
@@ -135,9 +135,7 @@ func libraryQueueOwner(workerID string, job jobs.Job, libraryJobID string) strin
if queueID == "" {
queueID = libraryJobID
}
worker := workerID
if worker == "" {
worker = "worker"
}
return fmt.Sprintf("%s#%s#%d", worker, queueID, job.Attempt)
// 合成规则只有一份(governance.LibraryDeliveryOwner):派发台账要按同一格式核
// "这次许可属于哪次真实投递"。
return governance.LibraryDeliveryOwner(workerID, queueID, job.Attempt)
}
@@ -3,6 +3,8 @@ package governance
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
)
// The recovery request has its own queue identity and stable operation. It
@@ -18,3 +20,33 @@ func LibraryRecoveryDigest(scope, queue, window, operation, target, source, back
"target_id": target, "source_snapshot": source, "backup_snapshot": backup, "writeset_digest": plan,
})
}
// LibraryDeliveryOwner 是队列投递 + 业务 attempt 的合成 owner:
//
// <worker>#<queue job>#<queue attempt>
//
// 它就是 library_attempts.owner,也决定了"这次派发属于哪一次真实投递"。worker 与
// 派发台账必须用同一份格式,否则"当前 owner"会变成两处各自解释的字符串。
func LibraryDeliveryOwner(workerID, queueID string, attempt int) string {
if queueID == "" {
queueID = "-"
}
if workerID == "" {
workerID = "worker"
}
return fmt.Sprintf("%s#%s#%d", workerID, queueID, attempt)
}
// ParseLibraryDeliveryOwner 把 owner 拆回 (worker, ok);queue/attempt 必须逐位相同,
// 不能靠后缀匹配把别的投递混进来。
func ParseLibraryDeliveryOwner(owner, queueID string, attempt int) (string, bool) {
suffix := fmt.Sprintf("#%s#%d", queueID, attempt)
if queueID == "" || !strings.HasSuffix(owner, suffix) {
return "", false
}
worker := strings.TrimSuffix(owner, suffix)
if worker == "" || strings.Contains(worker, "#") {
return "", false
}
return worker, true
}
@@ -126,10 +126,17 @@ func (b RestoreMarkerBinding) Digest() (string, error) {
}
// RestoreDispatchStatus 是纯读分类结果。Held 为真时永远不能发送。
//
// BindingConflict 是与 Held 不同的**类型化**分类:Held 只说"这次不能再 create"
// (prepared 冲突、dispatching 结果未知、marker 已确认都可能是合法的不可重发),
// 而 BindingConflict 说明台账行的不可变绑定与本次绑定不是同一个 effect。绑定冲突
// 在**任何状态**下都必须拒绝,包括已完成 clean 的 marker_confirmed;否则"只允许原
// 确认分支继续"会变成放行一份不属于本计划的写入。
type RestoreDispatchStatus struct {
State string `json:"state"`
Held bool `json:"held"`
Reason string `json:"reason,omitempty"`
State string `json:"state"`
Held bool `json:"held"`
BindingConflict bool `json:"binding_conflict,omitempty"`
Reason string `json:"reason,omitempty"`
}
// RestoreSendPermit 是"这一次调用、这一个 effect、这一次发送"的许可。
@@ -29,7 +29,24 @@ import (
const RestoreDispatchActivationMaxAge = 2 * time.Hour
const RestoreDispatchActivationProtocolVersion = 2
// 证据文档 schema 版本与语义兼容批次是两件事:
// - ProtocolVersion:这份 JSON 的字段形状(解析契约);
// - ProtocolEpoch:派发协议语义批次;只有同一批次的构建才允许共享激活证据
// (例如同一批次的兼容重建/打包),不同批次必须重新登记。
//
// 构建身份用完整 40 位 SHA,不用 display:开发号会随提交数变化,两个不同二进制
// 完全可能都叫 0.1.322。
const RestoreDispatchActivationProtocolVersion = 3
const RestoreDispatchProtocolEpoch = 3
// RestoreDispatchGovernanceAuthority 是治理库的**权威域**身份:系统标识(集群)
// 加库 OID。只看 current_database() 无法区分"同名但另一个集群/另一份克隆"。
// primary 由 Store 现场读 pg_is_in_recovery() 得出,不采信调用方声明。
type RestoreDispatchGovernanceAuthority struct {
SystemIdentifier string `json:"system_identifier"`
DatabaseOID string `json:"database_oid"`
}
// RestoreDispatchWriterDrain 是调用方对"旧 writer 集合"的声明:只给身份,不给结论。
// InstanceID 必须等于 journal 里登记的不可变 writer 身份(容器/进程 ID)。
@@ -41,27 +58,35 @@ type RestoreDispatchWriterDrain struct {
type RestoreDispatchActivationEvidence struct {
ProtocolVersion int `json:"protocol_version"`
ProtocolEpoch int `json:"protocol_epoch"`
TargetIdentity string `json:"target_identity"`
// TargetFingerprint 是维护窗口/部署目标指纹(journal 与治理库维护行记录的那一个);
// RestoreTargetFingerprint 是激活时对恢复目标内容/索引/桶的实时观察值。两者是不同层,
// 不能相互替代:前者绑定"动的是哪个实例",后者绑定"恢复要写的那个目标身份"。
TargetFingerprint string `json:"target_fingerprint"`
RestoreTargetFingerprint string `json:"restore_target_fingerprint"`
TargetInstance string `json:"target_instance"`
GovernanceDatabase string `json:"governance_database"`
ProtocolBuild string `json:"protocol_build"`
AttestedBy string `json:"attested_by"`
AttestedAt time.Time `json:"attested_at"`
MaintenanceJob string `json:"maintenance_job"`
MaintenanceJournal string `json:"maintenance_journal"`
MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"`
Writers []RestoreDispatchWriterDrain `json:"writers"`
TargetFingerprint string `json:"target_fingerprint"`
RestoreTargetFingerprint string `json:"restore_target_fingerprint"`
TargetInstance string `json:"target_instance"`
GovernanceDatabase string `json:"governance_database"`
GovernanceAuthority RestoreDispatchGovernanceAuthority `json:"governance_authority"`
ProtocolBuild string `json:"protocol_build"`
ProtocolBuildSHA string `json:"protocol_build_sha"`
AttestedBy string `json:"attested_by"`
AttestedAt time.Time `json:"attested_at"`
MaintenanceJob string `json:"maintenance_job"`
MaintenanceJournal string `json:"maintenance_journal"`
MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"`
Writers []RestoreDispatchWriterDrain `json:"writers"`
}
func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.Duration) error {
if e.ProtocolVersion != RestoreDispatchActivationProtocolVersion {
return fmt.Errorf("restore dispatch activation protocol version %d is not supported", e.ProtocolVersion)
}
// 兼容批次不同 = 派发语义不兼容:即使 display 相同也不能共用激活证据。
if e.ProtocolEpoch != RestoreDispatchProtocolEpoch {
return fmt.Errorf("restore dispatch activation protocol epoch %d is not compatible with %d",
e.ProtocolEpoch, RestoreDispatchProtocolEpoch)
}
for name, value := range map[string]string{
"target identity": e.TargetIdentity, "target fingerprint": e.TargetFingerprint,
"restore target fingerprint": e.RestoreTargetFingerprint,
@@ -74,6 +99,14 @@ func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.D
return fmt.Errorf("restore dispatch activation evidence lacks %s", name)
}
}
// 构建身份必须是完整 SHA;display 可以随提交数变化,不能作为身份。
if !isFullSHA(e.ProtocolBuildSHA) {
return errors.New("restore dispatch activation build identity is not a full 40-character commit sha")
}
// 治理权威域必须是可读回的 PG 事实形状:集群系统标识 + 库 OID,两个都是十进制数字。
if !isDecimal(e.GovernanceAuthority.SystemIdentifier) || !isDecimal(e.GovernanceAuthority.DatabaseOID) {
return errors.New("restore dispatch activation governance authority is not a cluster/database identity")
}
if !isHex64(e.TargetFingerprint) || !isHex64(e.RestoreTargetFingerprint) {
return errors.New("restore dispatch activation target fingerprints are not sha256 hex strings")
}
@@ -161,6 +194,26 @@ func isHex64(value string) bool {
return err == nil
}
func isFullSHA(value string) bool {
if len(value) != 40 || strings.ToLower(value) != value {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}
func isDecimal(value string) bool {
if value == "" || len(value) > 20 {
return false
}
for _, r := range value {
if r < '0' || r > '9' {
return false
}
}
return true
}
// validActivationID 与维护 journal 的作业/服务 ID 同一字符集:证据里出现的身份
// 必须能原样出现在 journal 中,不能靠空格/路径字符制造"看起来不同"的别名。
func validActivationID(value string) bool {
@@ -103,11 +103,17 @@ WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetI
// lockRestoreDispatchActivation 在**同一个 CAS 事务**里锁激活行。撤销是同一行的
// UPDATE:撤销先完成 ⇒ 这里看到 revoked_at;CAS 先取锁 ⇒ 撤销必须等 CAS 提交,
// 之后按"已派发"的窄边界处理。两者只能有一个线性化顺序。
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity string) error {
//
// 这里同时是**使用点**的权威域/构建身份重核:激活行里的 verified 事实必须与当前
// 连接的 PG 权威域(集群 system_identifier + 库 OID + primary)一致;运行构建必须
// 有完整 SHA,且与激活批次兼容;恢复目标指纹必须等于本次观察值。同名库、副本、
// unknown 构建或换目标都在这里失败关闭,而不是只在激活那一刻检查。
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity, targetFingerprint string) error {
var revoked *time.Time
var evidenceRaw, rowFingerprint string
err := tx.QueryRow(ctx, `
SELECT revoked_at FROM library_index_restore_dispatch_activation
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked)
SELECT revoked_at, evidence::text, target_fingerprint FROM library_index_restore_dispatch_activation
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked, &evidenceRaw, &rowFingerprint)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
}
@@ -117,6 +123,39 @@ WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Sca
if revoked != nil {
return fmt.Errorf("%w(目标 %s 的激活已于 %s 撤销)", ErrRestoreDispatchNotActivated, targetIdentity, revoked.UTC().Format(time.RFC3339))
}
if rowFingerprint != targetFingerprint {
return fmt.Errorf("%w(目标 %s 的激活绑定指纹 %s,本次恢复目标指纹 %s)",
ErrRestoreDispatchNotActivated, targetIdentity, rowFingerprint, targetFingerprint)
}
var record restoreDispatchActivationRecord
if err := json.Unmarshal([]byte(evidenceRaw), &record); err != nil {
return fmt.Errorf("parse restore dispatch activation record: %w", err)
}
verified := record.Verified
authority, primary, _, err := observedGovernanceAuthority(ctx, tx)
if err != nil {
return err
}
if verified.GovernanceAuthority.SystemIdentifier == "" || verified.GovernanceAuthority.DatabaseOID == "" {
return fmt.Errorf("%w(目标 %s 的激活登记没有治理权威域身份:拒绝仅凭库名启用)",
ErrRestoreDispatchNotActivated, targetIdentity)
}
if authority != verified.GovernanceAuthority {
return fmt.Errorf("%w(目标 %s 的激活登记属于治理域 %s/%s,当前是 %s/%s)",
ErrRestoreDispatchNotActivated, targetIdentity, verified.GovernanceAuthority.SystemIdentifier,
verified.GovernanceAuthority.DatabaseOID, authority.SystemIdentifier, authority.DatabaseOID)
}
if !primary {
return fmt.Errorf("%w(治理库当前处于恢复/副本角色,拒绝授予派发许可)", ErrRestoreDispatchNotActivated)
}
buildEvidence := libraryrelease.RestoreDispatchActivationEvidence{
ProtocolEpoch: verified.ProtocolEpoch,
ProtocolBuild: verified.ProtocolBuild,
ProtocolBuildSHA: verified.ProtocolBuildSHA,
}
if _, _, _, err := verifyRestoreDispatchBuild(buildEvidence); err != nil {
return fmt.Errorf("%w(目标 %s:%v)", ErrRestoreDispatchNotActivated, targetIdentity, err)
}
return nil
}
@@ -143,7 +182,7 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
}
if stored != digest {
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
return libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
}
switch state {
@@ -223,7 +262,7 @@ func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.Resto
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
return libraryrelease.RestoreDispatchStatus{}, err
}
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return libraryrelease.RestoreDispatchStatus{}, err
}
if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil {
@@ -270,7 +309,7 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
// 撤销与 CAS 在同一事务线性化:这里拿激活行锁,撤销的 UPDATE 必须排队。
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
// 当前代次必须属于这个原 window 作业,且租户/epoch/租约都仍有效。lease 用
@@ -322,13 +361,24 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err)
}
if stored != digest {
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld
}
if state != libraryrelease.RestoreDispatchPrepared {
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld
}
// effect 行锁是本次 CAS 前最后一个可能阻塞的锁。job/attempt/恢复投递的租约可能
// 正好在等它的时候过期;前面那几次检查发生在拿锁之前,不能充当 CAS 点的判据。
// 这些行锁都已被本事务持有,这里只重读、不改变锁序。
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
var held *restoreDispatchLeaseExpired
if errors.As(err, &held) {
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true,
Reason: held.Error()}, fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, held.Error())
}
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
@@ -384,7 +434,7 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
return err
}
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return err
}
var epoch int64
@@ -404,6 +454,23 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper
if attemptJob != b.Key.WindowJobID || owner != b.Owner || epoch != b.Epoch || !leaseLive {
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
}
// 与 Acquire 同一个顺序:确认前的最后一把锁也要先拿到,再按数据库实时时钟重核
// 租约。否则"确认"可以在等锁跨过到期之后仍落库,把一次过期 owner 的回执写成事实。
var effectID int64
if err := tx.QueryRow(ctx, `
SELECT id FROM library_index_restore_dispatch
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&effectID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("restore marker confirmation did not match a dispatch record")
}
return fmt.Errorf("lock restore marker for confirmation: %w", err)
}
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
return fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, err)
}
tag, err := tx.Exec(ctx, `
UPDATE library_index_restore_dispatch
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=clock_timestamp(), updated_at=clock_timestamp()
@@ -513,23 +580,8 @@ FROM library_instance_maintenance WHERE instance=$1 FOR UPDATE`, b.TargetInstanc
}
if b.RecoveryJobID != "" {
var queueState string
var lease string
if err := tx.QueryRow(ctx, `SELECT state, COALESCE(lease_owner,'') FROM jobs WHERE id=$1`, b.RecoveryJobID).
Scan(&queueState, &lease); err != nil {
return fmt.Errorf("read recovery queue job: %w", err)
}
if queueState != "running" || lease == "" {
return fmt.Errorf("recovery queue job %s is not a live delivery", b.RecoveryJobID)
}
var recoveryKind, recoveryScope, recoveryBackup string
if err := tx.QueryRow(ctx, `
SELECT kind, scope, target_snapshot_id FROM library_operations WHERE id=$1`, b.RecoveryOperationID).
Scan(&recoveryKind, &recoveryScope, &recoveryBackup); err != nil {
return fmt.Errorf("read recovery operation: %w", err)
}
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope || recoveryBackup != b.BackupSnapshot {
return fmt.Errorf("recovery operation %s is not bound to this window and backup", b.RecoveryOperationID)
if err := lockAndVerifyRecoveryDelivery(ctx, tx, b); err != nil {
return err
}
}
return nil
@@ -2,10 +2,12 @@ package postgres
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -42,8 +44,51 @@ type restoreDispatchVerifiedView struct {
Quiesce restoreDispatchQuiesceView `json:"quiesce"`
Writers []restoreDispatchWriterView `json:"writers"`
GovernanceDatabase string `json:"governance_database"`
ProtocolBuild string `json:"protocol_build"`
ObservedAt time.Time `json:"observed_at"`
// 治理权威域是现场读回的 PG 事实:集群 system_identifier + 库 OID + 是否 primary。
// 激活与每次使用点都重读比较,库名只作可读标签。
GovernanceAuthority restoreDispatchAuthorityView `json:"governance_authority"`
GovernancePrimary bool `json:"governance_primary"`
ProtocolBuild string `json:"protocol_build"`
ProtocolBuildSHA string `json:"protocol_build_sha"`
ProtocolEpoch int `json:"protocol_epoch"`
CompatibleUpgrade bool `json:"compatible_upgrade"`
ObservedAt time.Time `json:"observed_at"`
}
// restoreDispatchAuthorityView 只放可比较的权威域字段;不含密钥/连接串。
type restoreDispatchAuthorityView struct {
SystemIdentifier string `json:"system_identifier"`
DatabaseOID string `json:"database_oid"`
}
// lockRestoreDispatchActivationVerification 是使用点重核所需的最小事实。
type restoreDispatchActivationVerification struct {
Authority restoreDispatchAuthorityView
Primary bool
ProtocolBuild string
ProtocolBuildSHA string
ProtocolEpoch int
TargetFingerprint string
}
// observedGovernanceAuthority 读回当前连接的治理库权威域。pg_control_system() 的
// system_identifier 是集群级、跨备份/克隆仍不同的身份;库 OID 在同一集群内区分同名库。
// 没有权限读它时返回错误(能力失败关闭),绝不退回"库名相同就算同一个库"。
func observedGovernanceAuthority(ctx context.Context, q interface {
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
}) (restoreDispatchAuthorityView, bool, string, error) {
var authority restoreDispatchAuthorityView
var database string
var inRecovery bool
err := q.QueryRow(ctx, `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery)
if err != nil {
return authority, false, "", fmt.Errorf("observe governance authority: %w", err)
}
return authority, !inRecovery, database, nil
}
type restoreDispatchGateView struct {
@@ -128,21 +173,26 @@ func (s *Store) verifyRestoreDispatchEvidence(ctx context.Context, evidence libr
if evidence.AttestedAt.Before(journal.Quiesce.PostStopAt) {
return verified, errors.New("restore dispatch attestation predates the maintenance stop observation")
}
var database string
if err := s.db.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
return verified, fmt.Errorf("read governance database identity: %w", err)
authority, primary, database, err := observedGovernanceAuthority(ctx, s.db)
if err != nil {
return verified, err
}
if database == "" || database != evidence.GovernanceDatabase {
if authority != (restoreDispatchAuthorityView{SystemIdentifier: evidence.GovernanceAuthority.SystemIdentifier,
DatabaseOID: evidence.GovernanceAuthority.DatabaseOID}) {
return verified, fmt.Errorf("restore dispatch activation names governance authority %s/%s, live is %s/%s",
evidence.GovernanceAuthority.SystemIdentifier, evidence.GovernanceAuthority.DatabaseOID,
authority.SystemIdentifier, authority.DatabaseOID)
}
if !primary {
return verified, errors.New("restore dispatch activation requires the governance database to be primary, not a replica")
}
if database != evidence.GovernanceDatabase {
return verified, fmt.Errorf("restore dispatch activation evidence names governance database %q, live is %q",
evidence.GovernanceDatabase, database)
}
build := version.DisplayVersion()
if !version.HasInjection() || build == "" || build == "dev" {
return verified, errors.New("restore dispatch activation requires a build with injected version identity")
}
if build != evidence.ProtocolBuild {
return verified, fmt.Errorf("restore dispatch activation evidence was produced by build %q, running build is %q",
evidence.ProtocolBuild, build)
build, buildSHA, compatible, err := verifyRestoreDispatchBuild(evidence)
if err != nil {
return verified, err
}
// 维护窗口指纹与恢复目标指纹是两层:前者已与 journal/维护行比对,后者必须等于
// 激活方刚刚对恢复目标做的那次只读观察。用一个值同时顶替两者会掩盖"配置指向
@@ -230,7 +280,52 @@ FROM library_instance_maintenance WHERE instance=$1`, evidence.TargetInstance).S
Quiesce: restoreDispatchQuiesceView{Depth: journal.Quiesce.Depth, PendingOps: journal.Quiesce.PendingOps,
Unsettled: journal.Quiesce.Unsettled, Digest: journal.Quiesce.Digest, StableMS: journal.Quiesce.StableMS,
Samples: journal.Quiesce.Samples, ProvenAt: journal.Quiesce.ProvenAt.UTC(), PostStopAt: journal.Quiesce.PostStopAt.UTC()},
Writers: verifiedWriters, GovernanceDatabase: database, ProtocolBuild: build, ObservedAt: now.UTC(),
Writers: verifiedWriters, GovernanceDatabase: database,
GovernanceAuthority: authority, GovernancePrimary: primary,
ProtocolBuild: build, ProtocolBuildSHA: buildSHA,
ProtocolEpoch: evidence.ProtocolEpoch, CompatibleUpgrade: compatible,
ObservedAt: now.UTC(),
}
return verified, nil
}
// verifyRestoreDispatchBuild 是构建身份的兼容策略:
// - 运行中的二进制必须有注入的**完整 40 位 SHA**,否则 unknown —— 拒绝;
// - 证据的构建 SHA 也必须是完整 SHA;
// - 同一 SHA ⇒ 必须与运行 display 相符(防止同一二进制自称别的版本);
// - 不同 SHA ⇒ 只有 ProtocolEpoch 相同(已在 Validate 里核过)才算兼容重建,
// 并在台账里显式记 compatible_upgrade=true,不静默等同。
func verifyRestoreDispatchBuild(evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string, bool, error) {
build := version.DisplayVersion()
runningSHA := version.CommitSHA
if !version.HasInjection() || build == "" || build == "dev" || !isFullHexSHA(runningSHA) {
return "", "", false, errors.New("restore dispatch activation requires a build with an injected full commit sha")
}
if !isFullHexSHA(evidence.ProtocolBuildSHA) {
return "", "", false, errors.New("restore dispatch activation evidence carries no full build sha")
}
if evidence.ProtocolEpoch != libraryrelease.RestoreDispatchProtocolEpoch {
return "", "", false, fmt.Errorf("restore dispatch activation build epoch %d is incompatible with %d",
evidence.ProtocolEpoch, libraryrelease.RestoreDispatchProtocolEpoch)
}
if evidence.ProtocolBuildSHA == runningSHA {
if evidence.ProtocolBuild != build {
return "", "", false, fmt.Errorf("restore dispatch activation claims build %q for the running commit, display is %q",
evidence.ProtocolBuild, build)
}
return build, runningSHA, false, nil
}
// 兼容批次内的不同 SHA:允许,但必须记录成"升级",不能伪装成同一个二进制。
if strings.TrimSpace(evidence.ProtocolBuild) == "" {
return "", "", false, errors.New("restore dispatch activation compatible build has no display identity")
}
return build, runningSHA, true, nil
}
func isFullHexSHA(value string) bool {
if len(value) != 40 || strings.ToLower(value) != value {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}
@@ -0,0 +1,220 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
)
// DS01-F2.a:CAS 点的实时租约判据。
//
// 授权事务里的前几次租约检查发生在**拿 effect 行锁之前**;等锁可以跨过租约到期。
// 因此拿到最后一个可能阻塞的锁之后必须按数据库时钟重读一次。这里的行都已经在
// 本事务里持有锁,重读不会新增等待,也不改变 job→operation→maintenance→attempt→effect
// 的锁序。
type restoreDispatchLeaseExpired struct {
reason string
}
func (e *restoreDispatchLeaseExpired) Error() string { return e.reason }
func restoreDispatchLeaseLive(ctx context.Context, tx pgx.Tx, until *time.Time) (bool, error) {
var live bool
if err := tx.QueryRow(ctx,
`SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, until).Scan(&live); err != nil {
return false, fmt.Errorf("read lease clock: %w", err)
}
return live, nil
}
func recheckRestoreDispatchLeases(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
var owner string
var epoch int64
var leaseUntil *time.Time
err := tx.QueryRow(ctx, `
SELECT lease_owner, attempt_epoch, lease_until FROM library_jobs WHERE id=$1`,
b.Key.WindowJobID).Scan(&owner, &epoch, &leaseUntil)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("window job %s disappeared", b.Key.WindowJobID)
}
if err != nil {
return fmt.Errorf("recheck window job lease: %w", err)
}
if owner != b.Owner || epoch != b.Epoch {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"window job lease now belongs to %s#%d, not %s#%d", owner, epoch, b.Owner, b.Epoch)}
}
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "window job lease expired while waiting for the dispatch CAS"}
}
var attemptOwner string
var attemptEpoch int64
var attemptLease *time.Time
err = tx.QueryRow(ctx, `
SELECT owner, epoch, lease_until FROM library_attempts
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
&attemptOwner, &attemptEpoch, &attemptLease)
if errors.Is(err, pgx.ErrNoRows) {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf("no active attempt for operation %s", b.Key.OperationID)}
}
if err != nil {
return fmt.Errorf("recheck active attempt lease: %w", err)
}
if attemptOwner != b.Owner || attemptEpoch != b.Epoch {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"active attempt is now %s#%d, not %s#%d", attemptOwner, attemptEpoch, b.Owner, b.Epoch)}
}
live, err = restoreDispatchLeaseLive(ctx, tx, attemptLease)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "attempt lease expired while waiting for the dispatch CAS"}
}
if b.RecoveryJobID != "" {
var deliveryOwner string
var deliveryAttempt int
var deliveryLease *time.Time
err = tx.QueryRow(ctx, `SELECT lease_owner, attempt, lease_until FROM jobs WHERE id=$1`,
b.RecoveryJobID).Scan(&deliveryOwner, &deliveryAttempt, &deliveryLease)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery queue job %s disappeared", b.RecoveryJobID)
}
if err != nil {
return fmt.Errorf("recheck recovery queue lease: %w", err)
}
worker, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, deliveryAttempt)
if !ok || worker != deliveryOwner {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"recovery delivery %s attempt %d is not owned by %s", b.RecoveryJobID, deliveryAttempt, b.Owner)}
}
live, err = restoreDispatchLeaseLive(ctx, tx, deliveryLease)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "recovery queue lease expired while waiting for the dispatch CAS"}
}
}
return nil
}
// DS01-F2.b:恢复投递身份必须在**同一个 CAS 事务**里锁定并复核:
// queue → operation → 原 window/request_digest,再核当前 owner/attempt/lease。
//
// 只验"queue 是 running 且 operation 是 recover 类型"会让一条本属于别的窗口的
// 恢复投递借用本次 effect 的许可;worker 入口的先验校验(ValidateLibraryRecoveryQueue)
// 不能替代 CAS 点的实时授权。
func lockAndVerifyRecoveryDelivery(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
window := b.Key.WindowJobID
if b.RecoveryJobID != governance.LibraryRecoveryQueueID(window) {
return fmt.Errorf("recovery queue %s is not the registered queue for window %s", b.RecoveryJobID, window)
}
var queueType, queueState, payloadRaw, idempotency string
var leaseOwner string
var attempt int
var leaseUntil *time.Time
err := tx.QueryRow(ctx, `
SELECT type, state, payload::text, COALESCE(lease_owner,''), attempt, lease_until, idempotency_key
FROM jobs WHERE id=$1 FOR UPDATE`, b.RecoveryJobID).Scan(
&queueType, &queueState, &payloadRaw, &leaseOwner, &attempt, &leaseUntil, &idempotency)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery queue job %s does not exist", b.RecoveryJobID)
}
if err != nil {
return fmt.Errorf("lock recovery queue job: %w", err)
}
if queueType != "library.recover" || queueState != "running" {
return fmt.Errorf("recovery queue job %s is not a running delivery (type=%s state=%s)",
b.RecoveryJobID, queueType, queueState)
}
if idempotency != "library.recover:"+window {
return fmt.Errorf("recovery queue %s is not idempotency-bound to window %s", b.RecoveryJobID, window)
}
var payload map[string]string
if err := json.Unmarshal([]byte(payloadRaw), &payload); err != nil {
return fmt.Errorf("recovery queue payload is not a delivery claim: %w", err)
}
if payload["scope"] != b.Scope || payload["window_job_id"] != window {
return fmt.Errorf("recovery queue %s carries window %s/%s, not %s/%s",
b.RecoveryJobID, payload["window_job_id"], payload["scope"], window, b.Scope)
}
// 可选的锚点只要出现就必须与不可变绑定相同;缺省表示"由 journal/写集合决定"。
for key, want := range map[string]string{
"source_snapshot_id": b.SourceSnapshot,
"target_id": b.Key.TargetIdentity,
"original_operation_id": b.Key.OperationID,
"backup_snapshot_id": b.BackupSnapshot,
"plan_digest": b.Key.PlanDigest,
} {
if got := payload[key]; got != "" && got != want {
return fmt.Errorf("recovery queue %s declares %s=%s, binding is %s",
b.RecoveryJobID, key, got, want)
}
}
if _, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); !ok {
return fmt.Errorf("recovery queue %s attempt %d is not the current delivery of %s",
b.RecoveryJobID, attempt, b.Owner)
}
if ownerWorker, _ := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); ownerWorker != leaseOwner {
return fmt.Errorf("recovery queue %s is leased by %q, not by the attempt owner %q",
b.RecoveryJobID, leaseOwner, ownerWorker)
}
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
if err != nil {
return err
}
if !live {
return fmt.Errorf("recovery queue %s lease is expired", b.RecoveryJobID)
}
var recoveryKind, recoveryScope, recoveryTarget, requestDigest, operationTarget string
err = tx.QueryRow(ctx, `
SELECT kind, scope, target_snapshot_id, request_digest, COALESCE(target_id,'')
FROM library_operations WHERE id=$1 FOR UPDATE`, b.RecoveryOperationID).Scan(
&recoveryKind, &recoveryScope, &recoveryTarget, &requestDigest, &operationTarget)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery operation %s does not exist", b.RecoveryOperationID)
}
if err != nil {
return fmt.Errorf("lock recovery operation: %w", err)
}
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope {
return fmt.Errorf("recovery operation %s is not a recover operation for scope %s", b.RecoveryOperationID, b.Scope)
}
if recoveryTarget != b.BackupSnapshot {
return fmt.Errorf("recovery operation %s restores %s, not the window backup %s",
b.RecoveryOperationID, recoveryTarget, b.BackupSnapshot)
}
if operationTarget != "" && operationTarget != b.Key.TargetIdentity {
return fmt.Errorf("recovery operation %s targets %s, not %s",
b.RecoveryOperationID, operationTarget, b.Key.TargetIdentity)
}
if requestDigest == "" {
return fmt.Errorf("recovery operation %s has no request digest anchor", b.RecoveryOperationID)
}
// 恢复请求摘要只能由原 window 的不可变锚点重算:queue/operation/window/原 operation/
// 目标/来源/备份/写集合摘要。换任何一项都会得到另一份摘要。
expected, err := governance.LibraryRecoveryDigest(b.Scope, b.RecoveryJobID, window, b.Key.OperationID,
b.Key.TargetIdentity, b.SourceSnapshot, b.BackupSnapshot, b.Key.PlanDigest)
if err != nil {
return fmt.Errorf("recompute recovery request digest: %w", err)
}
if requestDigest != expected {
return fmt.Errorf("recovery operation %s request digest is bound to another window/plan", b.RecoveryOperationID)
}
return nil
}
@@ -0,0 +1,316 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/version"
)
// DS01-F2.b:恢复投递必须在 CAS 事务里与 queue→operation→原 window/摘要绑定。
func TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction(t *testing.T) {
type fixture struct {
store *Store
pool *pgxpool.Pool
binding libraryrelease.RestoreMarkerBinding
}
seeded := func(t *testing.T, jobID string) fixture {
t.Helper()
store, pool := newIdentityStore(t)
now := time.Now().UTC()
windowOp := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
queueID := governance.LibraryRecoveryQueueID(jobID)
owner := governance.LibraryDeliveryOwner("worker-r", queueID, 1)
if _, err := store.BeginLibraryAttempt(context.Background(), jobID, owner, 10*time.Minute, now); err != nil {
t.Fatalf("begin attempt: %v", err)
}
payload, _ := json.Marshal(map[string]string{"scope": "internal", "window_job_id": jobID})
if _, err := pool.Exec(context.Background(), `
INSERT INTO jobs (id, type, payload, state, lease_owner, lease_until, attempt, idempotency_key)
VALUES ($1,'library.recover',$2::jsonb,'running','worker-r',$3,1,$4)`,
queueID, string(payload), now.Add(10*time.Minute), "library.recover:"+jobID); err != nil {
t.Fatalf("seed recovery queue: %v", err)
}
recoveryOp := governance.LibraryOperationID("internal", governance.LibraryJobRecover, queueID)
digest, err := governance.LibraryRecoveryDigest("internal", queueID, jobID, windowOp, "target-ds01",
"snap-target", "snap-backup", "plan-1")
if err != nil {
t.Fatal(err)
}
if _, _, err := store.EnsureLibraryOperation(context.Background(), governance.LibraryOperation{
ID: recoveryOp, Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: digest, OperationKey: queueID,
Actor: "worker-r", CreatedAt: now,
}); err != nil {
t.Fatalf("seed recovery operation: %v", err)
}
binding := ds01Binding()
binding.Key.WindowJobID = jobID
binding.Key.OperationID = windowOp
binding.RecoveryJobID = queueID
binding.RecoveryOperationID = recoveryOp
binding.Owner = owner
binding.Epoch = 1
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
t.Fatalf("activate: %v", err)
}
return fixture{store: store, pool: pool, binding: binding}
}
expectRejected := func(t *testing.T, fx fixture) {
t.Helper()
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err == nil {
t.Fatal("recovery delivery mismatch was accepted")
}
}
t.Run("positive real recovery delivery prepares and acquires", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-ok")
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err != nil {
t.Fatalf("valid recovery delivery: %v", err)
}
permit, _, err := fx.store.AcquireRestoreMarker(context.Background(), fx.binding)
if err != nil || permit == nil {
t.Fatalf("valid recovery delivery acquire: permit=%v err=%v", permit, err)
}
})
t.Run("queue for another window is rejected", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongqueue")
fx.binding.RecoveryJobID = governance.LibraryRecoveryQueueID("another-window")
expectRejected(t, fx)
})
t.Run("payload window differs", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongwindow")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET payload=jsonb_set(payload,'{window_job_id}','"other-window"') WHERE id=$1`,
fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("request digest belongs to another plan", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongdigest")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET request_digest='digest-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("queue leased by another worker", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-owner")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET lease_owner='intruder' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("expired queue lease", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-expired")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET lease_until=now()-interval '1 minute' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("operation restores another backup", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-backup")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET target_snapshot_id='snap-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("binding plan differs from the recovery request", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-plan")
fx.binding.Key.PlanDigest = "plan-other"
expectRejected(t, fx)
})
t.Run("queue is not running", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-state")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET state='pending', lease_owner='', lease_until=NULL WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("operation is not a recovery", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-kind")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET kind='sync' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
}
// DS01-F3.identity:激活与使用点都必须核治理权威域(集群+库 OID)、primary 与完整
// 构建 SHA;同 display 不同不兼容 SHA、unknown、同名错域、错目标一律零激活/零许可。
func TestRestoreDispatchGovernanceAuthorityAndBuildIdentity(t *testing.T) {
store, pool := newIdentityStore(t)
// 使用点会读运行构建身份:测试进程由 runner 注入完整 SHA,这里核一下前提。
if !version.HasInjection() || len(version.CommitSHA) != 40 {
t.Skip("requires an injected full commit sha (run via the version-injecting runner)")
}
ctx := context.Background()
now := time.Now().UTC()
job := "authority-window"
if _, err := pool.Exec(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level,
services, ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,'internal',$3,'instance',$4::jsonb,'closed','needs_restore','','',now()-interval '20 minutes',now())`,
ds01TestInstance, job, ds01TestFingerprint, `["`+ds01TestWriter+`"]`); err != nil {
t.Fatalf("seed maintenance window: %v", err)
}
dir, _, journalDigest := ds01Journal(t, job, nil)
toStore := store.WithRestoreDispatchEvidence(dir)
target := "target-ds01-authority"
base := func() libraryrelease.RestoreDispatchActivationEvidence {
return ds01ActivationEvidence(t, store, target, ds01TestFingerprint, job, journalDigest)
}
activate := func(evidence libraryrelease.RestoreDispatchActivationEvidence) error {
t.Helper()
path, digest := writeDS01ActivationEvidence(t, evidence)
return toStore.ActivateRestoreDispatch(ctx, path, digest, target, ds01TestFingerprint, now)
}
count := func() int {
t.Helper()
var n int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch_activation`).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// 1) 同名库、不同集群:库名相同但 system_identifier 不同。
evidence := base()
evidence.GovernanceAuthority.SystemIdentifier = "1"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
t.Fatalf("same-name other-cluster evidence = %v", err)
}
// 2) 同集群、另一个同名库(OID 不同)。
evidence = base()
evidence.GovernanceAuthority.DatabaseOID = "1"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
t.Fatalf("same-name other-database evidence = %v", err)
}
// 3) unknown 构建身份。
evidence = base()
evidence.ProtocolBuildSHA = "unknown"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "commit sha") {
t.Fatalf("unknown build sha evidence = %v", err)
}
// 4) 同 display、不同 SHA、不同协议批次:不兼容,必须拒绝。
evidence = base()
evidence.ProtocolBuild = version.DisplayVersion()
evidence.ProtocolBuildSHA = strings.Repeat("b", 40)
evidence.ProtocolEpoch = libraryrelease.RestoreDispatchProtocolEpoch - 1
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "epoch") {
t.Fatalf("same-display incompatible build = %v", err)
}
if n := count(); n != 0 {
t.Fatalf("rejected evidence created %d activation rows", n)
}
// 5) 合法兼容升级:不同完整 SHA、同一协议批次 ⇒ 允许,但必须显式记成升级。
compat := base()
compat.ProtocolBuild = "0.1.999-compatible"
compat.ProtocolBuildSHA = strings.Repeat("a", 40)
if err := activate(compat); err != nil {
t.Fatalf("compatible upgrade evidence: %v", err)
}
var stored string
if err := pool.QueryRow(ctx, `SELECT evidence::text FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, target).Scan(&stored); err != nil {
t.Fatal(err)
}
compact := strings.ReplaceAll(stored, " ", "")
if !strings.Contains(compact, `"compatible_upgrade":true`) ||
!strings.Contains(compact, `"system_identifier":"`+compat.GovernanceAuthority.SystemIdentifier+`"`) {
t.Fatalf("activation record does not state the verified authority/upgrade: %s", stored)
}
// 使用点正例:同一兼容批次下真实窗口可以准备/取得许可。
windowOp := seedOperationJob(t, store, pool, "internal", "libsync-ds01-authority", "sync", now)
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-authority", "worker-a#q1#0", 10*time.Minute, now); err != nil {
t.Fatal(err)
}
binding := ds01Binding()
binding.Key.WindowJobID = "libsync-ds01-authority"
binding.Key.OperationID = windowOp
binding.Key.TargetIdentity = target
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
t.Fatalf("use point rejected a compatible build: %v", err)
}
if permit, _, err := store.AcquireRestoreMarker(ctx, binding); err != nil || permit == nil {
t.Fatalf("use point acquire under compatible build: permit=%v err=%v", permit, err)
}
// 使用点反例:把登记行篡改成错域/错批次/unknown 构建,Prepare 必须零许可失败。
tamperTargets := map[string]func(map[string]any){
"target-ds01-tamper-domain": func(v map[string]any) {
authority, _ := v["governance_authority"].(map[string]any)
authority["system_identifier"] = "1"
},
"target-ds01-tamper-epoch": func(v map[string]any) {
v["protocol_epoch"] = float64(libraryrelease.RestoreDispatchProtocolEpoch - 1)
},
"target-ds01-tamper-sha": func(v map[string]any) {
v["protocol_build_sha"] = "unknown"
},
}
for tamperTarget, mutate := range tamperTargets {
var record map[string]any
if err := json.Unmarshal([]byte(stored), &record); err != nil {
t.Fatal(err)
}
verified, _ := record["verified"].(map[string]any)
mutate(verified)
raw, err := json.Marshal(record)
if err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO library_index_restore_dispatch_activation
(target_identity, protocol_version, target_fingerprint, target_instance, governance_database,
protocol_build, activated_by, drain_evidence_digest, evidence, attested_at, activated_at)
SELECT $1, protocol_version, target_fingerprint, target_instance, governance_database,
protocol_build, activated_by, drain_evidence_digest, $2::jsonb, attested_at, activated_at
FROM library_index_restore_dispatch_activation WHERE target_identity=$3`,
tamperTarget, string(raw), target); err != nil {
t.Fatalf("insert tampered activation %s: %v", tamperTarget, err)
}
tampered := binding
tampered.Key.TargetIdentity = tamperTarget
if permit, _, err := store.AcquireRestoreMarker(ctx, tampered); permit != nil || err == nil {
t.Fatalf("use point accepted tampered activation %s: permit=%v err=%v", tamperTarget, permit, err)
}
}
// 使用点错目标:同一激活行但本次观察的目标指纹不同。
wrongTarget := binding
wrongTarget.Key.DocumentID = "doc-other-target"
wrongTarget.TargetFingerprint = ds01TestFingerprintOther
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongTarget); permit != nil || err == nil {
t.Fatalf("use point accepted another target fingerprint: permit=%v err=%v", permit, err)
}
// 运行构建 unknown:即使已有合法登记,使用点也必须失败关闭。
originalSHA := version.CommitSHA
version.CommitSHA = "unknown"
t.Cleanup(func() { version.CommitSHA = originalSHA })
unknown := binding
unknown.Key.DocumentID = "doc-unknown-build"
permit, _, err := store.AcquireRestoreMarker(ctx, unknown)
if permit != nil || err == nil {
t.Fatalf("use point accepted an unknown running build: permit=%v err=%v", permit, err)
}
if errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
t.Log("unknown build rejected as held (expected fail-closed)")
}
}
@@ -190,6 +190,63 @@ func TestRestoreDispatchLeaseExpiryWhileWaitingForLockIsRejected(t *testing.T) {
}
}
// 等锁跨过租约(effect 行):前面的检查发生在拿 effect 锁之前,全部通过;真正等
// effect 行锁时租约到期。CAS 点必须按数据库实时时钟重核,零许可、状态保持 prepared。
func TestRestoreDispatchLeaseExpiryWhileWaitingForEffectLockIsRejected(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
seedOperationJob(t, store, pool, "internal", "libsync-ds01-effect-lease", "sync", now)
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-effect-lease", "worker-a#q1#0", 1500*time.Millisecond, now); err != nil {
t.Fatal(err)
}
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
t.Fatal(err)
}
binding := ds01Binding()
binding.Key.WindowJobID = "libsync-ds01-effect-lease"
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
t.Fatal(err)
}
lock, err := pool.Begin(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := lock.Exec(ctx, `SELECT id FROM library_index_restore_dispatch WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest,
binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil {
t.Fatal(err)
}
type result struct {
permit *libraryrelease.RestoreSendPermit
err error
}
done := make(chan result, 1)
go func() {
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
done <- result{permit, err}
}()
time.Sleep(2200 * time.Millisecond)
if err := lock.Rollback(ctx); err != nil {
t.Fatal(err)
}
got := <-done
if got.permit != nil {
t.Fatal("permit issued although the attempt lease expired while waiting for the effect row lock")
}
if got.err == nil || !strings.Contains(got.err.Error(), "expired while waiting for the dispatch CAS") {
t.Fatalf("expired effect-lock wait error = %v", got.err)
}
var state string
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
t.Fatal(err)
}
if state != libraryrelease.RestoreDispatchPrepared {
t.Fatalf("state = %s after a rejected CAS", state)
}
}
// 错目标/错来源/换 plan 别名/错 UUID 都必须在 CAS 前失败,且台账不动。
func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
store, pool := newIdentityStore(t)
@@ -238,38 +295,43 @@ func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
if count != 1 {
t.Fatalf("dispatch rows = %d, want exactly the original plan row", count)
}
// 恢复投递身份缺失/错误:Prepare 就必须失败,不能等到发送前。
// 恢复投递身份缺失:Prepare 就必须失败,不能等到发送前。
recovery := binding
recovery.RecoveryJobID = "queue-missing"
recovery.RecoveryOperationID = "op-missing"
if _, err := store.PrepareRestoreMarker(ctx, recovery); err == nil {
t.Fatal("unknown recovery delivery identity was accepted")
}
// 真实的 recovery queue/operation 绑定可以被验证通过;换成另一份 backup 的
// recovery operation 则必须拒绝。
// 形状合法但与本 window 无关的 recovery queue/operation:同样零许可。
// 真正绑定成功的正例在 TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction。
queueID := governance.LibraryRecoveryQueueID(binding.Key.WindowJobID)
if _, err := pool.Exec(ctx, `INSERT INTO jobs (id, type, payload, state, lease_owner, idempotency_key)
VALUES ('queue-recovery-1','library.recover','{}'::jsonb,'running','worker-a','queue-recovery-1')`); err != nil {
VALUES ($1,'library.recover',$2::jsonb,'running','worker-a',$3)`,
"queue-recovery-unrelated", `{"scope":"internal","window_job_id":"another-window"}`,
"library.recover:another-window"); err != nil {
t.Fatal(err)
}
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: "op-recovery-1", Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-1",
ID: "op-recovery-unrelated", Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-unrelated",
Actor: "tester", CreatedAt: now.UTC(),
}); err != nil {
t.Fatal(err)
}
validRecovery := binding
validRecovery.Key.DocumentID = "doc-recovery"
validRecovery.RecoveryJobID = "queue-recovery-1"
validRecovery.RecoveryOperationID = "op-recovery-1"
if _, err := store.PrepareRestoreMarker(ctx, validRecovery); err != nil {
t.Fatalf("valid recovery delivery binding: %v", err)
unrelated := binding
unrelated.Key.DocumentID = "doc-recovery"
unrelated.RecoveryJobID = "queue-recovery-unrelated"
unrelated.RecoveryOperationID = "op-recovery-unrelated"
if _, err := store.PrepareRestoreMarker(ctx, unrelated); err == nil {
t.Fatal("recovery delivery for another window was accepted")
}
wrongRecovery := validRecovery
wrongRecovery.Key.DocumentID = "doc-recovery-2"
wrongRecovery.RecoveryOperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
if _, err := store.PrepareRestoreMarker(ctx, wrongRecovery); err == nil {
t.Fatal("recovery operation for another intent was accepted")
// 正确 queue 但 recovery operation 指向原业务 operation:不是恢复请求,必须拒绝。
wrongOperation := binding
wrongOperation.Key.DocumentID = "doc-recovery-2"
wrongOperation.RecoveryJobID = queueID
wrongOperation.RecoveryOperationID = binding.Key.OperationID
if _, err := store.PrepareRestoreMarker(ctx, wrongOperation); err == nil {
t.Fatal("non-recovery operation was accepted as a recovery delivery")
}
}
@@ -433,22 +433,42 @@ func ds01JournalRowJob(t *testing.T, store *Store) string {
return job
}
// ds01ActivationEvidence 组装 v2 证据:只给身份/作业/摘要/writer 集合。
// ds01ActivationEvidence 组装 v3 证据:身份/作业/摘要/writer 集合 + 治理权威域 + 构建 SHA。
func ds01ActivationEvidence(t *testing.T, store *Store, targetIdentity, fingerprint, job, journalDigest string) libraryrelease.RestoreDispatchActivationEvidence {
t.Helper()
var database string
if err := store.db.QueryRow(context.Background(), `SELECT current_database()`).Scan(&database); err != nil {
t.Fatalf("read governance database: %v", err)
authority, primary, database := ds01ObservedAuthority(t, store)
if !primary {
t.Fatal("governance database is not primary")
}
return libraryrelease.RestoreDispatchActivationEvidence{
ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(),
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database,
GovernanceAuthority: authority, ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA,
AttestedBy: "ds01-test", AttestedAt: time.Now().UTC(),
MaintenanceJob: job, MaintenanceJournal: job + ".json", MaintenanceJournalSHA256: journalDigest,
Writers: []libraryrelease.RestoreDispatchWriterDrain{{Name: ds01TestWriter, Kind: "worker", InstanceID: ds01TestWriter}},
}
}
// ds01ObservedAuthority 读回测试进程实际连接的治理库权威域;证据必须携带它,
// 否则"同名异集群"永远无法被拒绝。
func ds01ObservedAuthority(t *testing.T, store *Store) (libraryrelease.RestoreDispatchGovernanceAuthority, bool, string) {
t.Helper()
var authority libraryrelease.RestoreDispatchGovernanceAuthority
var inRecovery bool
var database string
if err := store.db.QueryRow(context.Background(), `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery); err != nil {
t.Fatalf("observe governance authority: %v", err)
}
return authority, !inRecovery, database
}
func writeDS01ActivationEvidence(t *testing.T, evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string) {
t.Helper()
raw, err := json.Marshal(evidence)
@@ -434,9 +434,16 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance)
if expected := os.Getenv("KG_R38_VERSION"); expected != "" && build != expected {
return "", nil, fmt.Errorf("test binary build %q != runner identity %q", build, expected)
}
var database string
if err := gov.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
return "", nil, fmt.Errorf("read governance database: %w", err)
var database, systemIdentifier, databaseOID string
if err := gov.QueryRow(ctx, `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan(
&database, &systemIdentifier, &databaseOID); err != nil {
return "", nil, fmt.Errorf("read governance authority: %w", err)
}
if expected := os.Getenv("KG_R38_COMMIT"); expected != "" && version.CommitSHA != expected {
return "", nil, fmt.Errorf("test binary commit %q != runner identity %q", version.CommitSHA, expected)
}
journalSum := sha256.Sum256(rawJournal)
// 激活后同一窗口还会继续写 journal(recovered 等);把激活那一刻的字节原样留档,
@@ -447,8 +454,12 @@ FROM library_instance_maintenance WHERE instance=$1`, cfg.LibraryTargetInstance)
}
now := time.Now().UTC()
evidence := libraryrelease.RestoreDispatchActivationEvidence{
ProtocolVersion: 2, TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database, ProtocolBuild: build,
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
TargetIdentity: cfg.LibraryTargetIdentity(), TargetFingerprint: windowFingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: cfg.LibraryTargetInstance, GovernanceDatabase: database,
GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID},
ProtocolBuild: build, ProtocolBuildSHA: version.CommitSHA,
AttestedBy: "r38-runner", AttestedAt: now,
MaintenanceJob: job.ID, MaintenanceJournal: job.ID + ".json",
MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]), Writers: writers,
@@ -480,7 +491,10 @@ FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, cfg.Li
facts := map[string]any{
"target": target, "fingerprint": fingerprint, "window_fingerprint": windowFingerprint,
"governance_database": database,
"maintenance_job": job.ID, "protocol_build": build,
"governance_authority": map[string]string{"system_identifier": systemIdentifier, "database_oid": databaseOID,
"primary": "true"},
"protocol_build_sha": version.CommitSHA,
"maintenance_job": job.ID, "protocol_build": build,
"journal_sha256": drainDigest, "journal_snapshot": filepath.Base(journalSnapshot),
"activation_evidence_sha256": hex.EncodeToString(evidenceSum[:]),
"journal_state": journal["state"], "gate": gate, "quiesce": quiesce,
@@ -16,6 +16,7 @@ type ds01StubGate struct {
inspectStatus libraryrelease.RestoreDispatchStatus
inspectFound bool
records []libraryrelease.RestoreDispatchRecord
confirmed int
}
func (g *ds01StubGate) InspectRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
@@ -28,6 +29,7 @@ func (g *ds01StubGate) AcquireRestoreMarker(context.Context, libraryrelease.Rest
return nil, g.inspectStatus, libraryrelease.ErrRestoreDispatchHeld
}
func (g *ds01StubGate) ConfirmRestoreMarker(context.Context, libraryrelease.RestoreMarkerBinding, []byte) error {
g.confirmed++
return nil
}
func (g *ds01StubGate) UnresolvedRestoreDispatches(context.Context, string, string, string) ([]libraryrelease.RestoreDispatchRecord, error) {
@@ -127,6 +129,102 @@ func TestDS01BindingConflictHeldBeforeAnyWrite(t *testing.T) {
}
}
// DS01-F4.binding:绑定冲突是**类型化**分类——prepared/dispatching/marker_confirmed/
// done 任一状态、准入/确认/收尾任一入口都必须拒绝;合法 dispatching + 精确 marker
// (不冲突)仍允许确认,不能用"拒绝全部 Held"掩盖。
func TestDS01BindingConflictRejectedInEveryState(t *testing.T) {
fx, snapshot, target, backups, wire := ds01Fixture(t)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
authority.attach(target)
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
ledger := outcome.Backup.WriteSet
if err := target.AdoptWriteSet(*ledger); err != nil {
t.Fatal(err)
}
ds01FreezeBindingInputs(t, target, outcome.Backup)
change := indexChangeOf(t, ledger, "chunk-target")
body, err := target.restoreIndexClaimBody(change)
if err != nil {
t.Fatal(err)
}
// 在目标上写入精确 marker:这样 dispatching 状态本来是可确认的,冲突检查必须
// 独立地拒绝它,而不是靠"marker 不在"。
applied, err := target.loadIndexEffect(change, "applied")
if err != nil || applied == nil || applied.Receipt == nil {
t.Fatalf("applied receipt missing: %v", err)
}
method, path, payload, err := target.indexMutation(change, body, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, true)
if err != nil {
t.Fatal(err)
}
var marker indexReceipt
if err := target.search.requestJSON(fx.ctx, method, path, payload, &marker); err != nil {
t.Fatalf("exact marker write: %v", err)
}
observed, err := target.observePlannedIndex(fx.ctx, change)
if err != nil {
t.Fatal(err)
}
conflict := func(state string) *ds01StubGate {
return &ds01StubGate{inspectFound: true, inspectStatus: libraryrelease.RestoreDispatchStatus{
State: state, Held: true, BindingConflict: true, Reason: "binding differs"},
records: []libraryrelease.RestoreDispatchRecord{{PlanDigest: ledger.Digest, Index: change.Index,
DocumentID: change.ID, State: libraryrelease.RestoreDispatchMarkerConfirmed}}}
}
for _, state := range []string{
libraryrelease.RestoreDispatchPrepared,
libraryrelease.RestoreDispatchDispatching,
libraryrelease.RestoreDispatchMarkerConfirmed,
} {
gate := conflict(state)
target.config.RestoreDispatch = gate
if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
t.Fatalf("admission accepted a binding conflict in %s: %v", state, err)
}
if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err == nil || !strings.Contains(err.Error(), "冲突") {
t.Fatalf("confirmation accepted a binding conflict in %s: %v", state, err)
}
if gate.confirmed != 0 {
t.Fatalf("binding conflict in %s still recorded a confirmation", state)
}
}
// done 状态:restored 文件已在,准入仍必须因绑定冲突拒绝(旧实现只看状态)。
receipt := indexReceipt{Index: change.Index, ID: change.ID, Result: "created",
SeqNo: marker.SeqNo, PrimaryTerm: marker.PrimaryTerm}
if err := target.saveIndexEffect(change, "restored", &receipt); err != nil {
t.Fatal(err)
}
target.config.RestoreDispatch = conflict(libraryrelease.RestoreDispatchMarkerConfirmed)
if err := target.admitRestorePlan(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
t.Fatalf("admission accepted a binding conflict on a completed effect: %v", err)
}
// 收尾:记录状态看起来是 marker_confirmed、restored 也存在,但绑定冲突必须拒绝。
gate := conflict(libraryrelease.RestoreDispatchMarkerConfirmed)
target.config.RestoreDispatch = gate
if err := target.verifyRestoreEffectsClosed(fx.ctx); err == nil || !strings.Contains(err.Error(), "冲突") {
t.Fatalf("closeout accepted a binding conflict: %v", err)
}
if gate.confirmed != 0 {
t.Fatal("closeout recorded a confirmation for a conflicting binding")
}
// 正例:合法 dispatching + 精确 marker(不冲突)仍可确认,且只确认一次。
okGate := &ds01StubGate{inspectFound: true,
inspectStatus: libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
Reason: "dispatch outcome unknown: no further create is permitted"}}
target.config.RestoreDispatch = okGate
if err := target.confirmDispatchedMarker(fx.ctx, change, body, observed); err != nil {
t.Fatalf("legitimate dispatching marker confirmation: %v", err)
}
if okGate.confirmed != 1 {
t.Fatalf("legitimate marker confirmation count = %d", okGate.confirmed)
}
if wire.dispatch != 0 {
t.Fatalf("confirmation path forwarded %d marker writes", wire.dispatch)
}
}
// DS01-F4:prepared 台账却已在目标上出现精确 marker —— 证据矛盾,必须 held;
// 旧实现会把它当成"已存在的标记"认领并 clean。
func TestDS01PreparedWithMarkerIsContradictionHeld(t *testing.T) {
@@ -549,8 +549,12 @@ func TestDS01RestoreHelper(t *testing.T) {
// 测试替身只负责造这两个真实来源,不替产品做判断。
func activateDS01(ctx context.Context, store *postgresstore.Store, pool *pgxpool.Pool,
targetIdentity, fingerprint string) error {
var database, job string
if err := pool.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
var database, systemIdentifier, databaseOID, job string
if err := pool.QueryRow(ctx, `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database())`).Scan(
&database, &systemIdentifier, &databaseOID); err != nil {
return err
}
if err := pool.QueryRow(ctx, `SELECT owner_job_id FROM library_instance_maintenance WHERE instance=$1`,
@@ -599,8 +603,12 @@ WHERE instance=$1`, ds01TargetInstance, `["worker-ds01"]`, fingerprint); err !=
return err
}
evidence := libraryrelease.RestoreDispatchActivationEvidence{
ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(),
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TargetInstance, GovernanceDatabase: database,
GovernanceAuthority: libraryrelease.RestoreDispatchGovernanceAuthority{SystemIdentifier: systemIdentifier, DatabaseOID: databaseOID},
ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA,
AttestedBy: "ds01-test", AttestedAt: now,
MaintenanceJob: job, MaintenanceJournal: job + ".json",
MaintenanceJournalSHA256: hex.EncodeToString(journalSum[:]),
@@ -137,6 +137,9 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if !found {
return fmt.Errorf("%w: 既存意图没有派发台账(旧协议未知历史)", libraryrelease.ErrRestoreDispatchHeld)
}
if status.BindingConflict {
return heldRestoreMarker(status, "既存意图的派发台账绑定与本次计划冲突")
}
if status.State == libraryrelease.RestoreDispatchPrepared && claimed != nil {
return heldRestoreMarker(status, "prepared 台账却已有 claimed 证据:协议证据矛盾")
}
@@ -161,6 +161,12 @@ func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
}
if found {
// 绑定冲突是类型化分类:任何状态(含已完成 clean 的终态)都必须拒绝。
// 它不能与"仅禁止第二次 create"的 dispatching-held 混为一谈——后者在
// 精确 marker 仍在时是允许确认的。
if status.BindingConflict {
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
}
if done != nil {
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
return heldRestoreMarker(status, "已完成 clean 的 effect 台账却不是 marker_confirmed")
@@ -241,6 +247,9 @@ func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c
if !found {
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
}
if status.BindingConflict {
return heldRestoreMarker(status, "派发台账的不可变绑定与本次计划冲突")
}
if status.Held && status.State == libraryrelease.RestoreDispatchPrepared {
return heldRestoreMarker(status, "派发绑定与不可变计划冲突")
}
@@ -370,6 +379,28 @@ func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context
if done == nil {
return fmt.Errorf("%w: 收尾前仍有未 clean 的 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
// 收尾同样是**使用点**:绑定冲突即使状态看起来是 marker_confirmed 也必须拒绝。
body, err := t.restoreIndexClaimBody(c)
if err != nil {
return err
}
binding, err := t.restoreMarkerBinding(ctx, c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return fmt.Errorf("恢复收尾读不到派发台账:%w", err)
}
if !found {
return fmt.Errorf("%w: 收尾时 effect %s 的派发台账消失了", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
if status.BindingConflict {
return heldRestoreMarker(status, "收尾时派发台账的不可变绑定与本次计划冲突")
}
if status.State != libraryrelease.RestoreDispatchMarkerConfirmed {
return fmt.Errorf("%w: 收尾时 effect %s 的派发状态是 %s", libraryrelease.ErrRestoreDispatchHeld, c.ID, status.State)
}
}
}
for _, c := range t.activePlan.Index {