fix(library): 收紧 DS01 派发租约/投递/治理身份与绑定冲突

F2.a:Acquire/Confirm 在取得 effect 行锁后用数据库时钟重核 job/attempt/
恢复投递租约,等锁跨过到期零许可且状态保持 prepared。

F2.b:同一 CAS 事务锁 recovery queue 与 operation,复核 window/scope、
idempotency、owner/attempt/lease,并按原 window 锚点重算恢复请求摘要。

F3.identity:激活证据 v3 携带 PG 集群 system_identifier + 库 OID、完整构建
SHA 与协议批次;激活与使用点都现场重读并拒绝同名异域/副本/unknown/不兼容 SHA。

F4.binding:区分绑定冲突与仅禁止重发的 Held,准入/确认/续作/收尾在所有状态
拒绝冲突,合法 dispatching + 精确 marker 仍可确认。
This commit is contained in:
姚章浩
2026-11-02 20:36:36 +08:00
parent 1e2bb385c7
commit 245cefd6b0
16 changed files with 1125 additions and 92 deletions
@@ -103,11 +103,17 @@ WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetI
// lockRestoreDispatchActivation 在**同一个 CAS 事务**里锁激活行。撤销是同一行的
// UPDATE:撤销先完成 ⇒ 这里看到 revoked_at;CAS 先取锁 ⇒ 撤销必须等 CAS 提交,
// 之后按"已派发"的窄边界处理。两者只能有一个线性化顺序。
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity string) error {
//
// 这里同时是**使用点**的权威域/构建身份重核:激活行里的 verified 事实必须与当前
// 连接的 PG 权威域(集群 system_identifier + 库 OID + primary)一致;运行构建必须
// 有完整 SHA,且与激活批次兼容;恢复目标指纹必须等于本次观察值。同名库、副本、
// unknown 构建或换目标都在这里失败关闭,而不是只在激活那一刻检查。
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity, targetFingerprint string) error {
var revoked *time.Time
var evidenceRaw, rowFingerprint string
err := tx.QueryRow(ctx, `
SELECT revoked_at FROM library_index_restore_dispatch_activation
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked)
SELECT revoked_at, evidence::text, target_fingerprint FROM library_index_restore_dispatch_activation
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked, &evidenceRaw, &rowFingerprint)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
}
@@ -117,6 +123,39 @@ WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Sca
if revoked != nil {
return fmt.Errorf("%w(目标 %s 的激活已于 %s 撤销)", ErrRestoreDispatchNotActivated, targetIdentity, revoked.UTC().Format(time.RFC3339))
}
if rowFingerprint != targetFingerprint {
return fmt.Errorf("%w(目标 %s 的激活绑定指纹 %s,本次恢复目标指纹 %s)",
ErrRestoreDispatchNotActivated, targetIdentity, rowFingerprint, targetFingerprint)
}
var record restoreDispatchActivationRecord
if err := json.Unmarshal([]byte(evidenceRaw), &record); err != nil {
return fmt.Errorf("parse restore dispatch activation record: %w", err)
}
verified := record.Verified
authority, primary, _, err := observedGovernanceAuthority(ctx, tx)
if err != nil {
return err
}
if verified.GovernanceAuthority.SystemIdentifier == "" || verified.GovernanceAuthority.DatabaseOID == "" {
return fmt.Errorf("%w(目标 %s 的激活登记没有治理权威域身份:拒绝仅凭库名启用)",
ErrRestoreDispatchNotActivated, targetIdentity)
}
if authority != verified.GovernanceAuthority {
return fmt.Errorf("%w(目标 %s 的激活登记属于治理域 %s/%s,当前是 %s/%s)",
ErrRestoreDispatchNotActivated, targetIdentity, verified.GovernanceAuthority.SystemIdentifier,
verified.GovernanceAuthority.DatabaseOID, authority.SystemIdentifier, authority.DatabaseOID)
}
if !primary {
return fmt.Errorf("%w(治理库当前处于恢复/副本角色,拒绝授予派发许可)", ErrRestoreDispatchNotActivated)
}
buildEvidence := libraryrelease.RestoreDispatchActivationEvidence{
ProtocolEpoch: verified.ProtocolEpoch,
ProtocolBuild: verified.ProtocolBuild,
ProtocolBuildSHA: verified.ProtocolBuildSHA,
}
if _, _, _, err := verifyRestoreDispatchBuild(buildEvidence); err != nil {
return fmt.Errorf("%w(目标 %s:%v)", ErrRestoreDispatchNotActivated, targetIdentity, err)
}
return nil
}
@@ -143,7 +182,7 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
}
if stored != digest {
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
return libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
}
switch state {
@@ -223,7 +262,7 @@ func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.Resto
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
return libraryrelease.RestoreDispatchStatus{}, err
}
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return libraryrelease.RestoreDispatchStatus{}, err
}
if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil {
@@ -270,7 +309,7 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
// 撤销与 CAS 在同一事务线性化:这里拿激活行锁,撤销的 UPDATE 必须排队。
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
// 当前代次必须属于这个原 window 作业,且租户/epoch/租约都仍有效。lease 用
@@ -322,13 +361,24 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err)
}
if stored != digest {
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true, BindingConflict: true,
Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld
}
if state != libraryrelease.RestoreDispatchPrepared {
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld
}
// effect 行锁是本次 CAS 前最后一个可能阻塞的锁。job/attempt/恢复投递的租约可能
// 正好在等它的时候过期;前面那几次检查发生在拿锁之前,不能充当 CAS 点的判据。
// 这些行锁都已被本事务持有,这里只重读、不改变锁序。
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
var held *restoreDispatchLeaseExpired
if errors.As(err, &held) {
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true,
Reason: held.Error()}, fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, held.Error())
}
return nil, libraryrelease.RestoreDispatchStatus{}, err
}
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
@@ -384,7 +434,7 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
return err
}
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity, b.TargetFingerprint); err != nil {
return err
}
var epoch int64
@@ -404,6 +454,23 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper
if attemptJob != b.Key.WindowJobID || owner != b.Owner || epoch != b.Epoch || !leaseLive {
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
}
// 与 Acquire 同一个顺序:确认前的最后一把锁也要先拿到,再按数据库实时时钟重核
// 租约。否则"确认"可以在等锁跨过到期之后仍落库,把一次过期 owner 的回执写成事实。
var effectID int64
if err := tx.QueryRow(ctx, `
SELECT id FROM library_index_restore_dispatch
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&effectID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("restore marker confirmation did not match a dispatch record")
}
return fmt.Errorf("lock restore marker for confirmation: %w", err)
}
if err := recheckRestoreDispatchLeases(ctx, tx, b); err != nil {
return fmt.Errorf("%w: %s", governance.ErrLibraryEpochStale, err)
}
tag, err := tx.Exec(ctx, `
UPDATE library_index_restore_dispatch
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=clock_timestamp(), updated_at=clock_timestamp()
@@ -513,23 +580,8 @@ FROM library_instance_maintenance WHERE instance=$1 FOR UPDATE`, b.TargetInstanc
}
if b.RecoveryJobID != "" {
var queueState string
var lease string
if err := tx.QueryRow(ctx, `SELECT state, COALESCE(lease_owner,'') FROM jobs WHERE id=$1`, b.RecoveryJobID).
Scan(&queueState, &lease); err != nil {
return fmt.Errorf("read recovery queue job: %w", err)
}
if queueState != "running" || lease == "" {
return fmt.Errorf("recovery queue job %s is not a live delivery", b.RecoveryJobID)
}
var recoveryKind, recoveryScope, recoveryBackup string
if err := tx.QueryRow(ctx, `
SELECT kind, scope, target_snapshot_id FROM library_operations WHERE id=$1`, b.RecoveryOperationID).
Scan(&recoveryKind, &recoveryScope, &recoveryBackup); err != nil {
return fmt.Errorf("read recovery operation: %w", err)
}
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope || recoveryBackup != b.BackupSnapshot {
return fmt.Errorf("recovery operation %s is not bound to this window and backup", b.RecoveryOperationID)
if err := lockAndVerifyRecoveryDelivery(ctx, tx, b); err != nil {
return err
}
}
return nil
@@ -2,10 +2,12 @@ package postgres
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -42,8 +44,51 @@ type restoreDispatchVerifiedView struct {
Quiesce restoreDispatchQuiesceView `json:"quiesce"`
Writers []restoreDispatchWriterView `json:"writers"`
GovernanceDatabase string `json:"governance_database"`
ProtocolBuild string `json:"protocol_build"`
ObservedAt time.Time `json:"observed_at"`
// 治理权威域是现场读回的 PG 事实:集群 system_identifier + 库 OID + 是否 primary。
// 激活与每次使用点都重读比较,库名只作可读标签。
GovernanceAuthority restoreDispatchAuthorityView `json:"governance_authority"`
GovernancePrimary bool `json:"governance_primary"`
ProtocolBuild string `json:"protocol_build"`
ProtocolBuildSHA string `json:"protocol_build_sha"`
ProtocolEpoch int `json:"protocol_epoch"`
CompatibleUpgrade bool `json:"compatible_upgrade"`
ObservedAt time.Time `json:"observed_at"`
}
// restoreDispatchAuthorityView 只放可比较的权威域字段;不含密钥/连接串。
type restoreDispatchAuthorityView struct {
SystemIdentifier string `json:"system_identifier"`
DatabaseOID string `json:"database_oid"`
}
// lockRestoreDispatchActivationVerification 是使用点重核所需的最小事实。
type restoreDispatchActivationVerification struct {
Authority restoreDispatchAuthorityView
Primary bool
ProtocolBuild string
ProtocolBuildSHA string
ProtocolEpoch int
TargetFingerprint string
}
// observedGovernanceAuthority 读回当前连接的治理库权威域。pg_control_system() 的
// system_identifier 是集群级、跨备份/克隆仍不同的身份;库 OID 在同一集群内区分同名库。
// 没有权限读它时返回错误(能力失败关闭),绝不退回"库名相同就算同一个库"。
func observedGovernanceAuthority(ctx context.Context, q interface {
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
}) (restoreDispatchAuthorityView, bool, string, error) {
var authority restoreDispatchAuthorityView
var database string
var inRecovery bool
err := q.QueryRow(ctx, `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery)
if err != nil {
return authority, false, "", fmt.Errorf("observe governance authority: %w", err)
}
return authority, !inRecovery, database, nil
}
type restoreDispatchGateView struct {
@@ -128,21 +173,26 @@ func (s *Store) verifyRestoreDispatchEvidence(ctx context.Context, evidence libr
if evidence.AttestedAt.Before(journal.Quiesce.PostStopAt) {
return verified, errors.New("restore dispatch attestation predates the maintenance stop observation")
}
var database string
if err := s.db.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
return verified, fmt.Errorf("read governance database identity: %w", err)
authority, primary, database, err := observedGovernanceAuthority(ctx, s.db)
if err != nil {
return verified, err
}
if database == "" || database != evidence.GovernanceDatabase {
if authority != (restoreDispatchAuthorityView{SystemIdentifier: evidence.GovernanceAuthority.SystemIdentifier,
DatabaseOID: evidence.GovernanceAuthority.DatabaseOID}) {
return verified, fmt.Errorf("restore dispatch activation names governance authority %s/%s, live is %s/%s",
evidence.GovernanceAuthority.SystemIdentifier, evidence.GovernanceAuthority.DatabaseOID,
authority.SystemIdentifier, authority.DatabaseOID)
}
if !primary {
return verified, errors.New("restore dispatch activation requires the governance database to be primary, not a replica")
}
if database != evidence.GovernanceDatabase {
return verified, fmt.Errorf("restore dispatch activation evidence names governance database %q, live is %q",
evidence.GovernanceDatabase, database)
}
build := version.DisplayVersion()
if !version.HasInjection() || build == "" || build == "dev" {
return verified, errors.New("restore dispatch activation requires a build with injected version identity")
}
if build != evidence.ProtocolBuild {
return verified, fmt.Errorf("restore dispatch activation evidence was produced by build %q, running build is %q",
evidence.ProtocolBuild, build)
build, buildSHA, compatible, err := verifyRestoreDispatchBuild(evidence)
if err != nil {
return verified, err
}
// 维护窗口指纹与恢复目标指纹是两层:前者已与 journal/维护行比对,后者必须等于
// 激活方刚刚对恢复目标做的那次只读观察。用一个值同时顶替两者会掩盖"配置指向
@@ -230,7 +280,52 @@ FROM library_instance_maintenance WHERE instance=$1`, evidence.TargetInstance).S
Quiesce: restoreDispatchQuiesceView{Depth: journal.Quiesce.Depth, PendingOps: journal.Quiesce.PendingOps,
Unsettled: journal.Quiesce.Unsettled, Digest: journal.Quiesce.Digest, StableMS: journal.Quiesce.StableMS,
Samples: journal.Quiesce.Samples, ProvenAt: journal.Quiesce.ProvenAt.UTC(), PostStopAt: journal.Quiesce.PostStopAt.UTC()},
Writers: verifiedWriters, GovernanceDatabase: database, ProtocolBuild: build, ObservedAt: now.UTC(),
Writers: verifiedWriters, GovernanceDatabase: database,
GovernanceAuthority: authority, GovernancePrimary: primary,
ProtocolBuild: build, ProtocolBuildSHA: buildSHA,
ProtocolEpoch: evidence.ProtocolEpoch, CompatibleUpgrade: compatible,
ObservedAt: now.UTC(),
}
return verified, nil
}
// verifyRestoreDispatchBuild 是构建身份的兼容策略:
// - 运行中的二进制必须有注入的**完整 40 位 SHA**,否则 unknown —— 拒绝;
// - 证据的构建 SHA 也必须是完整 SHA;
// - 同一 SHA ⇒ 必须与运行 display 相符(防止同一二进制自称别的版本);
// - 不同 SHA ⇒ 只有 ProtocolEpoch 相同(已在 Validate 里核过)才算兼容重建,
// 并在台账里显式记 compatible_upgrade=true,不静默等同。
func verifyRestoreDispatchBuild(evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string, bool, error) {
build := version.DisplayVersion()
runningSHA := version.CommitSHA
if !version.HasInjection() || build == "" || build == "dev" || !isFullHexSHA(runningSHA) {
return "", "", false, errors.New("restore dispatch activation requires a build with an injected full commit sha")
}
if !isFullHexSHA(evidence.ProtocolBuildSHA) {
return "", "", false, errors.New("restore dispatch activation evidence carries no full build sha")
}
if evidence.ProtocolEpoch != libraryrelease.RestoreDispatchProtocolEpoch {
return "", "", false, fmt.Errorf("restore dispatch activation build epoch %d is incompatible with %d",
evidence.ProtocolEpoch, libraryrelease.RestoreDispatchProtocolEpoch)
}
if evidence.ProtocolBuildSHA == runningSHA {
if evidence.ProtocolBuild != build {
return "", "", false, fmt.Errorf("restore dispatch activation claims build %q for the running commit, display is %q",
evidence.ProtocolBuild, build)
}
return build, runningSHA, false, nil
}
// 兼容批次内的不同 SHA:允许,但必须记录成"升级",不能伪装成同一个二进制。
if strings.TrimSpace(evidence.ProtocolBuild) == "" {
return "", "", false, errors.New("restore dispatch activation compatible build has no display identity")
}
return build, runningSHA, true, nil
}
func isFullHexSHA(value string) bool {
if len(value) != 40 || strings.ToLower(value) != value {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}
@@ -0,0 +1,220 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
)
// DS01-F2.a:CAS 点的实时租约判据。
//
// 授权事务里的前几次租约检查发生在**拿 effect 行锁之前**;等锁可以跨过租约到期。
// 因此拿到最后一个可能阻塞的锁之后必须按数据库时钟重读一次。这里的行都已经在
// 本事务里持有锁,重读不会新增等待,也不改变 job→operation→maintenance→attempt→effect
// 的锁序。
type restoreDispatchLeaseExpired struct {
reason string
}
func (e *restoreDispatchLeaseExpired) Error() string { return e.reason }
func restoreDispatchLeaseLive(ctx context.Context, tx pgx.Tx, until *time.Time) (bool, error) {
var live bool
if err := tx.QueryRow(ctx,
`SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, until).Scan(&live); err != nil {
return false, fmt.Errorf("read lease clock: %w", err)
}
return live, nil
}
func recheckRestoreDispatchLeases(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
var owner string
var epoch int64
var leaseUntil *time.Time
err := tx.QueryRow(ctx, `
SELECT lease_owner, attempt_epoch, lease_until FROM library_jobs WHERE id=$1`,
b.Key.WindowJobID).Scan(&owner, &epoch, &leaseUntil)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("window job %s disappeared", b.Key.WindowJobID)
}
if err != nil {
return fmt.Errorf("recheck window job lease: %w", err)
}
if owner != b.Owner || epoch != b.Epoch {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"window job lease now belongs to %s#%d, not %s#%d", owner, epoch, b.Owner, b.Epoch)}
}
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "window job lease expired while waiting for the dispatch CAS"}
}
var attemptOwner string
var attemptEpoch int64
var attemptLease *time.Time
err = tx.QueryRow(ctx, `
SELECT owner, epoch, lease_until FROM library_attempts
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
&attemptOwner, &attemptEpoch, &attemptLease)
if errors.Is(err, pgx.ErrNoRows) {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf("no active attempt for operation %s", b.Key.OperationID)}
}
if err != nil {
return fmt.Errorf("recheck active attempt lease: %w", err)
}
if attemptOwner != b.Owner || attemptEpoch != b.Epoch {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"active attempt is now %s#%d, not %s#%d", attemptOwner, attemptEpoch, b.Owner, b.Epoch)}
}
live, err = restoreDispatchLeaseLive(ctx, tx, attemptLease)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "attempt lease expired while waiting for the dispatch CAS"}
}
if b.RecoveryJobID != "" {
var deliveryOwner string
var deliveryAttempt int
var deliveryLease *time.Time
err = tx.QueryRow(ctx, `SELECT lease_owner, attempt, lease_until FROM jobs WHERE id=$1`,
b.RecoveryJobID).Scan(&deliveryOwner, &deliveryAttempt, &deliveryLease)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery queue job %s disappeared", b.RecoveryJobID)
}
if err != nil {
return fmt.Errorf("recheck recovery queue lease: %w", err)
}
worker, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, deliveryAttempt)
if !ok || worker != deliveryOwner {
return &restoreDispatchLeaseExpired{reason: fmt.Sprintf(
"recovery delivery %s attempt %d is not owned by %s", b.RecoveryJobID, deliveryAttempt, b.Owner)}
}
live, err = restoreDispatchLeaseLive(ctx, tx, deliveryLease)
if err != nil {
return err
}
if !live {
return &restoreDispatchLeaseExpired{reason: "recovery queue lease expired while waiting for the dispatch CAS"}
}
}
return nil
}
// DS01-F2.b:恢复投递身份必须在**同一个 CAS 事务**里锁定并复核:
// queue → operation → 原 window/request_digest,再核当前 owner/attempt/lease。
//
// 只验"queue 是 running 且 operation 是 recover 类型"会让一条本属于别的窗口的
// 恢复投递借用本次 effect 的许可;worker 入口的先验校验(ValidateLibraryRecoveryQueue)
// 不能替代 CAS 点的实时授权。
func lockAndVerifyRecoveryDelivery(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
window := b.Key.WindowJobID
if b.RecoveryJobID != governance.LibraryRecoveryQueueID(window) {
return fmt.Errorf("recovery queue %s is not the registered queue for window %s", b.RecoveryJobID, window)
}
var queueType, queueState, payloadRaw, idempotency string
var leaseOwner string
var attempt int
var leaseUntil *time.Time
err := tx.QueryRow(ctx, `
SELECT type, state, payload::text, COALESCE(lease_owner,''), attempt, lease_until, idempotency_key
FROM jobs WHERE id=$1 FOR UPDATE`, b.RecoveryJobID).Scan(
&queueType, &queueState, &payloadRaw, &leaseOwner, &attempt, &leaseUntil, &idempotency)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery queue job %s does not exist", b.RecoveryJobID)
}
if err != nil {
return fmt.Errorf("lock recovery queue job: %w", err)
}
if queueType != "library.recover" || queueState != "running" {
return fmt.Errorf("recovery queue job %s is not a running delivery (type=%s state=%s)",
b.RecoveryJobID, queueType, queueState)
}
if idempotency != "library.recover:"+window {
return fmt.Errorf("recovery queue %s is not idempotency-bound to window %s", b.RecoveryJobID, window)
}
var payload map[string]string
if err := json.Unmarshal([]byte(payloadRaw), &payload); err != nil {
return fmt.Errorf("recovery queue payload is not a delivery claim: %w", err)
}
if payload["scope"] != b.Scope || payload["window_job_id"] != window {
return fmt.Errorf("recovery queue %s carries window %s/%s, not %s/%s",
b.RecoveryJobID, payload["window_job_id"], payload["scope"], window, b.Scope)
}
// 可选的锚点只要出现就必须与不可变绑定相同;缺省表示"由 journal/写集合决定"。
for key, want := range map[string]string{
"source_snapshot_id": b.SourceSnapshot,
"target_id": b.Key.TargetIdentity,
"original_operation_id": b.Key.OperationID,
"backup_snapshot_id": b.BackupSnapshot,
"plan_digest": b.Key.PlanDigest,
} {
if got := payload[key]; got != "" && got != want {
return fmt.Errorf("recovery queue %s declares %s=%s, binding is %s",
b.RecoveryJobID, key, got, want)
}
}
if _, ok := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); !ok {
return fmt.Errorf("recovery queue %s attempt %d is not the current delivery of %s",
b.RecoveryJobID, attempt, b.Owner)
}
if ownerWorker, _ := governance.ParseLibraryDeliveryOwner(b.Owner, b.RecoveryJobID, attempt); ownerWorker != leaseOwner {
return fmt.Errorf("recovery queue %s is leased by %q, not by the attempt owner %q",
b.RecoveryJobID, leaseOwner, ownerWorker)
}
live, err := restoreDispatchLeaseLive(ctx, tx, leaseUntil)
if err != nil {
return err
}
if !live {
return fmt.Errorf("recovery queue %s lease is expired", b.RecoveryJobID)
}
var recoveryKind, recoveryScope, recoveryTarget, requestDigest, operationTarget string
err = tx.QueryRow(ctx, `
SELECT kind, scope, target_snapshot_id, request_digest, COALESCE(target_id,'')
FROM library_operations WHERE id=$1 FOR UPDATE`, b.RecoveryOperationID).Scan(
&recoveryKind, &recoveryScope, &recoveryTarget, &requestDigest, &operationTarget)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("recovery operation %s does not exist", b.RecoveryOperationID)
}
if err != nil {
return fmt.Errorf("lock recovery operation: %w", err)
}
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope {
return fmt.Errorf("recovery operation %s is not a recover operation for scope %s", b.RecoveryOperationID, b.Scope)
}
if recoveryTarget != b.BackupSnapshot {
return fmt.Errorf("recovery operation %s restores %s, not the window backup %s",
b.RecoveryOperationID, recoveryTarget, b.BackupSnapshot)
}
if operationTarget != "" && operationTarget != b.Key.TargetIdentity {
return fmt.Errorf("recovery operation %s targets %s, not %s",
b.RecoveryOperationID, operationTarget, b.Key.TargetIdentity)
}
if requestDigest == "" {
return fmt.Errorf("recovery operation %s has no request digest anchor", b.RecoveryOperationID)
}
// 恢复请求摘要只能由原 window 的不可变锚点重算:queue/operation/window/原 operation/
// 目标/来源/备份/写集合摘要。换任何一项都会得到另一份摘要。
expected, err := governance.LibraryRecoveryDigest(b.Scope, b.RecoveryJobID, window, b.Key.OperationID,
b.Key.TargetIdentity, b.SourceSnapshot, b.BackupSnapshot, b.Key.PlanDigest)
if err != nil {
return fmt.Errorf("recompute recovery request digest: %w", err)
}
if requestDigest != expected {
return fmt.Errorf("recovery operation %s request digest is bound to another window/plan", b.RecoveryOperationID)
}
return nil
}
@@ -0,0 +1,316 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/version"
)
// DS01-F2.b:恢复投递必须在 CAS 事务里与 queue→operation→原 window/摘要绑定。
func TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction(t *testing.T) {
type fixture struct {
store *Store
pool *pgxpool.Pool
binding libraryrelease.RestoreMarkerBinding
}
seeded := func(t *testing.T, jobID string) fixture {
t.Helper()
store, pool := newIdentityStore(t)
now := time.Now().UTC()
windowOp := seedOperationJob(t, store, pool, "internal", jobID, "sync", now)
queueID := governance.LibraryRecoveryQueueID(jobID)
owner := governance.LibraryDeliveryOwner("worker-r", queueID, 1)
if _, err := store.BeginLibraryAttempt(context.Background(), jobID, owner, 10*time.Minute, now); err != nil {
t.Fatalf("begin attempt: %v", err)
}
payload, _ := json.Marshal(map[string]string{"scope": "internal", "window_job_id": jobID})
if _, err := pool.Exec(context.Background(), `
INSERT INTO jobs (id, type, payload, state, lease_owner, lease_until, attempt, idempotency_key)
VALUES ($1,'library.recover',$2::jsonb,'running','worker-r',$3,1,$4)`,
queueID, string(payload), now.Add(10*time.Minute), "library.recover:"+jobID); err != nil {
t.Fatalf("seed recovery queue: %v", err)
}
recoveryOp := governance.LibraryOperationID("internal", governance.LibraryJobRecover, queueID)
digest, err := governance.LibraryRecoveryDigest("internal", queueID, jobID, windowOp, "target-ds01",
"snap-target", "snap-backup", "plan-1")
if err != nil {
t.Fatal(err)
}
if _, _, err := store.EnsureLibraryOperation(context.Background(), governance.LibraryOperation{
ID: recoveryOp, Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: digest, OperationKey: queueID,
Actor: "worker-r", CreatedAt: now,
}); err != nil {
t.Fatalf("seed recovery operation: %v", err)
}
binding := ds01Binding()
binding.Key.WindowJobID = jobID
binding.Key.OperationID = windowOp
binding.RecoveryJobID = queueID
binding.RecoveryOperationID = recoveryOp
binding.Owner = owner
binding.Epoch = 1
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
t.Fatalf("activate: %v", err)
}
return fixture{store: store, pool: pool, binding: binding}
}
expectRejected := func(t *testing.T, fx fixture) {
t.Helper()
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err == nil {
t.Fatal("recovery delivery mismatch was accepted")
}
}
t.Run("positive real recovery delivery prepares and acquires", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-ok")
if _, err := fx.store.PrepareRestoreMarker(context.Background(), fx.binding); err != nil {
t.Fatalf("valid recovery delivery: %v", err)
}
permit, _, err := fx.store.AcquireRestoreMarker(context.Background(), fx.binding)
if err != nil || permit == nil {
t.Fatalf("valid recovery delivery acquire: permit=%v err=%v", permit, err)
}
})
t.Run("queue for another window is rejected", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongqueue")
fx.binding.RecoveryJobID = governance.LibraryRecoveryQueueID("another-window")
expectRejected(t, fx)
})
t.Run("payload window differs", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongwindow")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET payload=jsonb_set(payload,'{window_job_id}','"other-window"') WHERE id=$1`,
fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("request digest belongs to another plan", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-wrongdigest")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET request_digest='digest-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("queue leased by another worker", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-owner")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET lease_owner='intruder' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("expired queue lease", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-expired")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET lease_until=now()-interval '1 minute' WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("operation restores another backup", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-backup")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET target_snapshot_id='snap-other' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("binding plan differs from the recovery request", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-plan")
fx.binding.Key.PlanDigest = "plan-other"
expectRejected(t, fx)
})
t.Run("queue is not running", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-state")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE jobs SET state='pending', lease_owner='', lease_until=NULL WHERE id=$1`, fx.binding.RecoveryJobID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
t.Run("operation is not a recovery", func(t *testing.T) {
fx := seeded(t, "libsync-ds01-recovery-kind")
if _, err := fx.pool.Exec(context.Background(),
`UPDATE library_operations SET kind='sync' WHERE id=$1`, fx.binding.RecoveryOperationID); err != nil {
t.Fatal(err)
}
expectRejected(t, fx)
})
}
// DS01-F3.identity:激活与使用点都必须核治理权威域(集群+库 OID)、primary 与完整
// 构建 SHA;同 display 不同不兼容 SHA、unknown、同名错域、错目标一律零激活/零许可。
func TestRestoreDispatchGovernanceAuthorityAndBuildIdentity(t *testing.T) {
store, pool := newIdentityStore(t)
// 使用点会读运行构建身份:测试进程由 runner 注入完整 SHA,这里核一下前提。
if !version.HasInjection() || len(version.CommitSHA) != 40 {
t.Skip("requires an injected full commit sha (run via the version-injecting runner)")
}
ctx := context.Background()
now := time.Now().UTC()
job := "authority-window"
if _, err := pool.Exec(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level,
services, ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,'internal',$3,'instance',$4::jsonb,'closed','needs_restore','','',now()-interval '20 minutes',now())`,
ds01TestInstance, job, ds01TestFingerprint, `["`+ds01TestWriter+`"]`); err != nil {
t.Fatalf("seed maintenance window: %v", err)
}
dir, _, journalDigest := ds01Journal(t, job, nil)
toStore := store.WithRestoreDispatchEvidence(dir)
target := "target-ds01-authority"
base := func() libraryrelease.RestoreDispatchActivationEvidence {
return ds01ActivationEvidence(t, store, target, ds01TestFingerprint, job, journalDigest)
}
activate := func(evidence libraryrelease.RestoreDispatchActivationEvidence) error {
t.Helper()
path, digest := writeDS01ActivationEvidence(t, evidence)
return toStore.ActivateRestoreDispatch(ctx, path, digest, target, ds01TestFingerprint, now)
}
count := func() int {
t.Helper()
var n int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch_activation`).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// 1) 同名库、不同集群:库名相同但 system_identifier 不同。
evidence := base()
evidence.GovernanceAuthority.SystemIdentifier = "1"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
t.Fatalf("same-name other-cluster evidence = %v", err)
}
// 2) 同集群、另一个同名库(OID 不同)。
evidence = base()
evidence.GovernanceAuthority.DatabaseOID = "1"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "governance authority") {
t.Fatalf("same-name other-database evidence = %v", err)
}
// 3) unknown 构建身份。
evidence = base()
evidence.ProtocolBuildSHA = "unknown"
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "commit sha") {
t.Fatalf("unknown build sha evidence = %v", err)
}
// 4) 同 display、不同 SHA、不同协议批次:不兼容,必须拒绝。
evidence = base()
evidence.ProtocolBuild = version.DisplayVersion()
evidence.ProtocolBuildSHA = strings.Repeat("b", 40)
evidence.ProtocolEpoch = libraryrelease.RestoreDispatchProtocolEpoch - 1
if err := activate(evidence); err == nil || !strings.Contains(err.Error(), "epoch") {
t.Fatalf("same-display incompatible build = %v", err)
}
if n := count(); n != 0 {
t.Fatalf("rejected evidence created %d activation rows", n)
}
// 5) 合法兼容升级:不同完整 SHA、同一协议批次 ⇒ 允许,但必须显式记成升级。
compat := base()
compat.ProtocolBuild = "0.1.999-compatible"
compat.ProtocolBuildSHA = strings.Repeat("a", 40)
if err := activate(compat); err != nil {
t.Fatalf("compatible upgrade evidence: %v", err)
}
var stored string
if err := pool.QueryRow(ctx, `SELECT evidence::text FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, target).Scan(&stored); err != nil {
t.Fatal(err)
}
compact := strings.ReplaceAll(stored, " ", "")
if !strings.Contains(compact, `"compatible_upgrade":true`) ||
!strings.Contains(compact, `"system_identifier":"`+compat.GovernanceAuthority.SystemIdentifier+`"`) {
t.Fatalf("activation record does not state the verified authority/upgrade: %s", stored)
}
// 使用点正例:同一兼容批次下真实窗口可以准备/取得许可。
windowOp := seedOperationJob(t, store, pool, "internal", "libsync-ds01-authority", "sync", now)
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-authority", "worker-a#q1#0", 10*time.Minute, now); err != nil {
t.Fatal(err)
}
binding := ds01Binding()
binding.Key.WindowJobID = "libsync-ds01-authority"
binding.Key.OperationID = windowOp
binding.Key.TargetIdentity = target
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
t.Fatalf("use point rejected a compatible build: %v", err)
}
if permit, _, err := store.AcquireRestoreMarker(ctx, binding); err != nil || permit == nil {
t.Fatalf("use point acquire under compatible build: permit=%v err=%v", permit, err)
}
// 使用点反例:把登记行篡改成错域/错批次/unknown 构建,Prepare 必须零许可失败。
tamperTargets := map[string]func(map[string]any){
"target-ds01-tamper-domain": func(v map[string]any) {
authority, _ := v["governance_authority"].(map[string]any)
authority["system_identifier"] = "1"
},
"target-ds01-tamper-epoch": func(v map[string]any) {
v["protocol_epoch"] = float64(libraryrelease.RestoreDispatchProtocolEpoch - 1)
},
"target-ds01-tamper-sha": func(v map[string]any) {
v["protocol_build_sha"] = "unknown"
},
}
for tamperTarget, mutate := range tamperTargets {
var record map[string]any
if err := json.Unmarshal([]byte(stored), &record); err != nil {
t.Fatal(err)
}
verified, _ := record["verified"].(map[string]any)
mutate(verified)
raw, err := json.Marshal(record)
if err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO library_index_restore_dispatch_activation
(target_identity, protocol_version, target_fingerprint, target_instance, governance_database,
protocol_build, activated_by, drain_evidence_digest, evidence, attested_at, activated_at)
SELECT $1, protocol_version, target_fingerprint, target_instance, governance_database,
protocol_build, activated_by, drain_evidence_digest, $2::jsonb, attested_at, activated_at
FROM library_index_restore_dispatch_activation WHERE target_identity=$3`,
tamperTarget, string(raw), target); err != nil {
t.Fatalf("insert tampered activation %s: %v", tamperTarget, err)
}
tampered := binding
tampered.Key.TargetIdentity = tamperTarget
if permit, _, err := store.AcquireRestoreMarker(ctx, tampered); permit != nil || err == nil {
t.Fatalf("use point accepted tampered activation %s: permit=%v err=%v", tamperTarget, permit, err)
}
}
// 使用点错目标:同一激活行但本次观察的目标指纹不同。
wrongTarget := binding
wrongTarget.Key.DocumentID = "doc-other-target"
wrongTarget.TargetFingerprint = ds01TestFingerprintOther
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongTarget); permit != nil || err == nil {
t.Fatalf("use point accepted another target fingerprint: permit=%v err=%v", permit, err)
}
// 运行构建 unknown:即使已有合法登记,使用点也必须失败关闭。
originalSHA := version.CommitSHA
version.CommitSHA = "unknown"
t.Cleanup(func() { version.CommitSHA = originalSHA })
unknown := binding
unknown.Key.DocumentID = "doc-unknown-build"
permit, _, err := store.AcquireRestoreMarker(ctx, unknown)
if permit != nil || err == nil {
t.Fatalf("use point accepted an unknown running build: permit=%v err=%v", permit, err)
}
if errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
t.Log("unknown build rejected as held (expected fail-closed)")
}
}
@@ -190,6 +190,63 @@ func TestRestoreDispatchLeaseExpiryWhileWaitingForLockIsRejected(t *testing.T) {
}
}
// 等锁跨过租约(effect 行):前面的检查发生在拿 effect 锁之前,全部通过;真正等
// effect 行锁时租约到期。CAS 点必须按数据库实时时钟重核,零许可、状态保持 prepared。
func TestRestoreDispatchLeaseExpiryWhileWaitingForEffectLockIsRejected(t *testing.T) {
store, pool := newIdentityStore(t)
ctx := context.Background()
now := time.Now().UTC()
seedOperationJob(t, store, pool, "internal", "libsync-ds01-effect-lease", "sync", now)
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-effect-lease", "worker-a#q1#0", 1500*time.Millisecond, now); err != nil {
t.Fatal(err)
}
if err := activateRestoreDispatchForTest(t, store, "target-ds01", ds01TestFingerprint); err != nil {
t.Fatal(err)
}
binding := ds01Binding()
binding.Key.WindowJobID = "libsync-ds01-effect-lease"
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
t.Fatal(err)
}
lock, err := pool.Begin(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := lock.Exec(ctx, `SELECT id FROM library_index_restore_dispatch WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest,
binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil {
t.Fatal(err)
}
type result struct {
permit *libraryrelease.RestoreSendPermit
err error
}
done := make(chan result, 1)
go func() {
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
done <- result{permit, err}
}()
time.Sleep(2200 * time.Millisecond)
if err := lock.Rollback(ctx); err != nil {
t.Fatal(err)
}
got := <-done
if got.permit != nil {
t.Fatal("permit issued although the attempt lease expired while waiting for the effect row lock")
}
if got.err == nil || !strings.Contains(got.err.Error(), "expired while waiting for the dispatch CAS") {
t.Fatalf("expired effect-lock wait error = %v", got.err)
}
var state string
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
t.Fatal(err)
}
if state != libraryrelease.RestoreDispatchPrepared {
t.Fatalf("state = %s after a rejected CAS", state)
}
}
// 错目标/错来源/换 plan 别名/错 UUID 都必须在 CAS 前失败,且台账不动。
func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
store, pool := newIdentityStore(t)
@@ -238,38 +295,43 @@ func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
if count != 1 {
t.Fatalf("dispatch rows = %d, want exactly the original plan row", count)
}
// 恢复投递身份缺失/错误:Prepare 就必须失败,不能等到发送前。
// 恢复投递身份缺失:Prepare 就必须失败,不能等到发送前。
recovery := binding
recovery.RecoveryJobID = "queue-missing"
recovery.RecoveryOperationID = "op-missing"
if _, err := store.PrepareRestoreMarker(ctx, recovery); err == nil {
t.Fatal("unknown recovery delivery identity was accepted")
}
// 真实的 recovery queue/operation 绑定可以被验证通过;换成另一份 backup 的
// recovery operation 则必须拒绝。
// 形状合法但与本 window 无关的 recovery queue/operation:同样零许可。
// 真正绑定成功的正例在 TestRestoreDispatchRecoveryDeliveryIsBoundInTheCASTransaction。
queueID := governance.LibraryRecoveryQueueID(binding.Key.WindowJobID)
if _, err := pool.Exec(ctx, `INSERT INTO jobs (id, type, payload, state, lease_owner, idempotency_key)
VALUES ('queue-recovery-1','library.recover','{}'::jsonb,'running','worker-a','queue-recovery-1')`); err != nil {
VALUES ($1,'library.recover',$2::jsonb,'running','worker-a',$3)`,
"queue-recovery-unrelated", `{"scope":"internal","window_job_id":"another-window"}`,
"library.recover:another-window"); err != nil {
t.Fatal(err)
}
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: "op-recovery-1", Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-1",
ID: "op-recovery-unrelated", Scope: "internal", Kind: governance.LibraryJobRecover,
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-unrelated",
Actor: "tester", CreatedAt: now.UTC(),
}); err != nil {
t.Fatal(err)
}
validRecovery := binding
validRecovery.Key.DocumentID = "doc-recovery"
validRecovery.RecoveryJobID = "queue-recovery-1"
validRecovery.RecoveryOperationID = "op-recovery-1"
if _, err := store.PrepareRestoreMarker(ctx, validRecovery); err != nil {
t.Fatalf("valid recovery delivery binding: %v", err)
unrelated := binding
unrelated.Key.DocumentID = "doc-recovery"
unrelated.RecoveryJobID = "queue-recovery-unrelated"
unrelated.RecoveryOperationID = "op-recovery-unrelated"
if _, err := store.PrepareRestoreMarker(ctx, unrelated); err == nil {
t.Fatal("recovery delivery for another window was accepted")
}
wrongRecovery := validRecovery
wrongRecovery.Key.DocumentID = "doc-recovery-2"
wrongRecovery.RecoveryOperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
if _, err := store.PrepareRestoreMarker(ctx, wrongRecovery); err == nil {
t.Fatal("recovery operation for another intent was accepted")
// 正确 queue 但 recovery operation 指向原业务 operation:不是恢复请求,必须拒绝。
wrongOperation := binding
wrongOperation.Key.DocumentID = "doc-recovery-2"
wrongOperation.RecoveryJobID = queueID
wrongOperation.RecoveryOperationID = binding.Key.OperationID
if _, err := store.PrepareRestoreMarker(ctx, wrongOperation); err == nil {
t.Fatal("non-recovery operation was accepted as a recovery delivery")
}
}
@@ -433,22 +433,42 @@ func ds01JournalRowJob(t *testing.T, store *Store) string {
return job
}
// ds01ActivationEvidence 组装 v2 证据:只给身份/作业/摘要/writer 集合。
// ds01ActivationEvidence 组装 v3 证据:身份/作业/摘要/writer 集合 + 治理权威域 + 构建 SHA。
func ds01ActivationEvidence(t *testing.T, store *Store, targetIdentity, fingerprint, job, journalDigest string) libraryrelease.RestoreDispatchActivationEvidence {
t.Helper()
var database string
if err := store.db.QueryRow(context.Background(), `SELECT current_database()`).Scan(&database); err != nil {
t.Fatalf("read governance database: %v", err)
authority, primary, database := ds01ObservedAuthority(t, store)
if !primary {
t.Fatal("governance database is not primary")
}
return libraryrelease.RestoreDispatchActivationEvidence{
ProtocolVersion: 2, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database, ProtocolBuild: version.DisplayVersion(),
ProtocolVersion: libraryrelease.RestoreDispatchActivationProtocolVersion,
ProtocolEpoch: libraryrelease.RestoreDispatchProtocolEpoch,
TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
RestoreTargetFingerprint: fingerprint, TargetInstance: ds01TestInstance, GovernanceDatabase: database,
GovernanceAuthority: authority, ProtocolBuild: version.DisplayVersion(), ProtocolBuildSHA: version.CommitSHA,
AttestedBy: "ds01-test", AttestedAt: time.Now().UTC(),
MaintenanceJob: job, MaintenanceJournal: job + ".json", MaintenanceJournalSHA256: journalDigest,
Writers: []libraryrelease.RestoreDispatchWriterDrain{{Name: ds01TestWriter, Kind: "worker", InstanceID: ds01TestWriter}},
}
}
// ds01ObservedAuthority 读回测试进程实际连接的治理库权威域;证据必须携带它,
// 否则"同名异集群"永远无法被拒绝。
func ds01ObservedAuthority(t *testing.T, store *Store) (libraryrelease.RestoreDispatchGovernanceAuthority, bool, string) {
t.Helper()
var authority libraryrelease.RestoreDispatchGovernanceAuthority
var inRecovery bool
var database string
if err := store.db.QueryRow(context.Background(), `
SELECT current_database(),
(SELECT system_identifier::text FROM pg_control_system()),
(SELECT oid::text FROM pg_database WHERE datname = current_database()),
pg_is_in_recovery()`).Scan(&database, &authority.SystemIdentifier, &authority.DatabaseOID, &inRecovery); err != nil {
t.Fatalf("observe governance authority: %v", err)
}
return authority, !inRecovery, database
}
func writeDS01ActivationEvidence(t *testing.T, evidence libraryrelease.RestoreDispatchActivationEvidence) (string, string) {
t.Helper()
raw, err := json.Marshal(evidence)