为什么:Astra 复核证实五组实现缺陷——正式装配缺 scope/快照且来源与 before-image 混用;激活撤销不在 CAS 事务内、lease 用事务起始时间;激活只接受任意字符串且撤销可 被 SQL 重置/删除;准入忽略绑定冲突与 prepared 矛盾、收尾只看当前 plan;prepared 已 提交但 mark-intent 未落盘时被误判 legacy。 改了什么: - 绑定来自原锚点(scope/来源快照/backup/实例/目标指纹/索引 UUID/恢复投递),每次 重建都实时重读目标身份并与冻结值比较; - activation 行锁与 CAS 同事务,lease 用 clock_timestamp(),同事务核对 job/operation/ attempt/维护窗口/恢复 queue 与 operation,同 effect 换 plan 摘要拒绝; - 激活消费带摘要的证据文件(writer 退出+排空、目标/治理库身份、有效期),撤销终局, down 在 activation 有行时拒绝; - 前置/收尾共用权威窗口全集,绑定冲突与 prepared 矛盾零写 held,合法 dispatching+ 精确 marker 仍可确认; - prepared 无 mark-intent 仅在绑定一致且无矛盾时补同内容意图,旧无台账意图仍 held。 验证:真 PG 锁等待覆盖撤销先胜/CAS 先胜/等锁跨租约/错锚点/plan 别名/恢复投递; COMMIT 已到服务端但回包丢失与未决事务分别读回;真 ES+PG+对象存储的 DS01 用例全部 通过;go test ./... 全绿。未迁移正式库、未部署、未激活。
This commit is contained in:
@@ -19,31 +19,62 @@ import (
|
||||
//
|
||||
// 关键点不是"多了一张表",而是发送许可的产生方式:
|
||||
// - Prepare 只建立 prepared(不返回许可);
|
||||
// - Acquire 在**独立短事务**里按 job→operation→active attempt→effect 的顺序取锁,
|
||||
// 校验当前 epoch/owner/lease 后做 prepared→dispatching 的 CAS,只有 COMMIT 明确
|
||||
// 成功才返回一次性许可;
|
||||
// - Acquire 在**独立短事务**里按 job→operation→activation→active attempt→effect
|
||||
// 的顺序取锁,校验当前 epoch/owner/lease(数据库时钟)、原作业/operation 锚点、
|
||||
// 维护窗口、恢复投递身份与不可变绑定后做 prepared→dispatching 的 CAS,只有
|
||||
// COMMIT 明确成功才返回一次性许可;
|
||||
// - Inspect 纯读,永不返回许可;
|
||||
// - 表本身拒绝状态回退、绑定变化与删行(迁移里的触发器)。
|
||||
// - 表本身拒绝状态回退、绑定变化、删行与激活撤销回退(迁移里的触发器)。
|
||||
//
|
||||
// 外层 WithLibraryExecution 的事务只 rollback、从不 commit,所以这里绝不能用它:
|
||||
// 在那种事务里写 dispatching,发出请求后账会被回滚成"未派发"。
|
||||
|
||||
var ErrRestoreDispatchNotActivated = libraryrelease.ErrRestoreDispatchUnavailable
|
||||
|
||||
// ActivateRestoreDispatch 登记"该目标已完成旧 writer 退出/排空"的部署证据。
|
||||
// ActivateRestoreDispatch 消费一份**可核验的**激活证据文件:文件摘要由调用方给出并
|
||||
// 重算、结构完整、每个受控 writer 都登记了退出与排空、证据在有效期内、目标身份/
|
||||
// 指纹与治理库身份都与现场一致。任何一项不满足都不建立 active 行。
|
||||
//
|
||||
// 它记录的是运维声明,不验证声明为真;代码只能保证没有这条登记就不发放许可。
|
||||
// 撤销是终局:重新启用必须换一个新的目标身份(或由 DBA 归档旧行后重新登记),
|
||||
// 不允许用同一条旧证据把已经撤销的目标悄悄打开。
|
||||
func (s *Store) ActivateRestoreDispatch(ctx context.Context, targetIdentity, activatedBy, drainEvidenceDigest string, now time.Time) error {
|
||||
if targetIdentity == "" || activatedBy == "" || drainEvidenceDigest == "" {
|
||||
return errors.New("restore dispatch activation requires target, actor and drain evidence digest")
|
||||
// 代码核验的是证据文件完整性与身份一致性;"writer 确实已退出"仍是部署流程的受控
|
||||
// attestation(含退出时间与排空摘要),不是本进程能独立观察到的外部事实。
|
||||
func (s *Store) ActivateRestoreDispatch(ctx context.Context, evidencePath, evidenceDigest,
|
||||
targetIdentity, targetFingerprint string, now time.Time) error {
|
||||
evidence, err := libraryrelease.LoadRestoreDispatchActivationEvidence(evidencePath, evidenceDigest,
|
||||
now.UTC(), libraryrelease.RestoreDispatchActivationMaxAge)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if evidence.TargetIdentity != targetIdentity {
|
||||
return fmt.Errorf("restore dispatch activation evidence is for target %q, not %q", evidence.TargetIdentity, targetIdentity)
|
||||
}
|
||||
if evidence.TargetFingerprint != targetFingerprint {
|
||||
return errors.New("restore dispatch activation evidence does not match the live target fingerprint")
|
||||
}
|
||||
var database string
|
||||
var address string
|
||||
var port int
|
||||
if err := s.db.QueryRow(ctx,
|
||||
`SELECT current_database(), COALESCE(host(inet_server_addr()), ''), COALESCE(inet_server_port(), 0)`).
|
||||
Scan(&database, &address, &port); err != nil {
|
||||
return fmt.Errorf("read governance database identity: %w", err)
|
||||
}
|
||||
if database == "" || database != evidence.GovernanceDatabase {
|
||||
return fmt.Errorf("restore dispatch activation evidence names governance database %q, live is %q",
|
||||
evidence.GovernanceDatabase, database)
|
||||
}
|
||||
raw, err := evidence.CanonicalBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
INSERT INTO library_index_restore_dispatch_activation (target_identity, protocol_version, activated_by, drain_evidence_digest, activated_at)
|
||||
VALUES ($1, 1, $2, $3, $4)
|
||||
INSERT INTO library_index_restore_dispatch_activation
|
||||
(target_identity, protocol_version, target_fingerprint, target_instance, governance_database,
|
||||
protocol_build, activated_by, drain_evidence_digest, evidence, attested_at, activated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11)
|
||||
ON CONFLICT (target_identity) DO NOTHING`,
|
||||
targetIdentity, activatedBy, drainEvidenceDigest, now.UTC()); err != nil {
|
||||
evidence.TargetIdentity, evidence.ProtocolVersion, evidence.TargetFingerprint, evidence.TargetInstance,
|
||||
evidence.GovernanceDatabase, evidence.ProtocolBuild, evidence.AttestedBy,
|
||||
evidenceDigest, string(raw), evidence.AttestedAt.UTC(), now.UTC()); err != nil {
|
||||
return fmt.Errorf("activate restore dispatch: %w", err)
|
||||
}
|
||||
return s.requireRestoreDispatchActivation(ctx, targetIdentity)
|
||||
@@ -76,6 +107,26 @@ WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetI
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockRestoreDispatchActivation 在**同一个 CAS 事务**里锁激活行。撤销是同一行的
|
||||
// UPDATE:撤销先完成 ⇒ 这里看到 revoked_at;CAS 先取锁 ⇒ 撤销必须等 CAS 提交,
|
||||
// 之后按"已派发"的窄边界处理。两者只能有一个线性化顺序。
|
||||
func lockRestoreDispatchActivation(ctx context.Context, tx pgx.Tx, targetIdentity string) error {
|
||||
var revoked *time.Time
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT revoked_at FROM library_index_restore_dispatch_activation
|
||||
WHERE target_identity=$1 AND protocol_version=1 FOR UPDATE`, targetIdentity).Scan(&revoked)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock restore dispatch activation: %w", err)
|
||||
}
|
||||
if revoked != nil {
|
||||
return fmt.Errorf("%w(目标 %s 的激活已于 %s 撤销)", ErrRestoreDispatchNotActivated, targetIdentity, revoked.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreDispatchBindingDigest(b libraryrelease.RestoreMarkerBinding) (string, error) {
|
||||
return b.Digest()
|
||||
}
|
||||
@@ -117,6 +168,49 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
|
||||
}
|
||||
}
|
||||
|
||||
// UnresolvedRestoreDispatches 返回原 window 的权威台账全集(跨 plan 摘要)。
|
||||
// 前置与收尾都查同一份记录:只遍历当前 plan 的文档列表会漏掉计划外/别名 effect。
|
||||
func (s *Store) UnresolvedRestoreDispatches(ctx context.Context, targetIdentity, operationID, windowJobID string) ([]libraryrelease.RestoreDispatchRecord, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT plan_digest, index_name, document_id, state FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3
|
||||
ORDER BY plan_digest, index_name, document_id`, targetIdentity, operationID, windowJobID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read restore dispatch window: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var records []libraryrelease.RestoreDispatchRecord
|
||||
for rows.Next() {
|
||||
var record libraryrelease.RestoreDispatchRecord
|
||||
if err := rows.Scan(&record.PlanDigest, &record.Index, &record.DocumentID, &record.State); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, rows.Err()
|
||||
}
|
||||
|
||||
// rejectRestoreDispatchPlanAlias 拒绝同一逻辑 effect 用另一个 plan 摘要另起一行。
|
||||
// key 里虽然有 plan_digest,但唯一约束允许"同 target/operation/window/index/doc/phase、
|
||||
// 不同 plan"共存;那正是"换计划别名重开证明"的旁路。
|
||||
func rejectRestoreDispatchPlanAlias(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
||||
var other string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT plan_digest FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3
|
||||
AND index_name=$4 AND document_id=$5 AND phase=$6 AND plan_digest<>$7
|
||||
LIMIT 1 FOR UPDATE`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase, b.Key.PlanDigest).Scan(&other)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read restore dispatch plan alias: %w", err)
|
||||
}
|
||||
return fmt.Errorf("%w: effect already belongs to another plan digest", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
|
||||
// PrepareRestoreMarker 只建立 prepared:没有发送权,也不把已有 dispatching 抬回 prepared。
|
||||
func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, error) {
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
@@ -136,6 +230,12 @@ func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO library_index_restore_dispatch
|
||||
(target_identity, operation_id, window_job_id, plan_digest, index_name, document_id, phase, binding_digest, state)
|
||||
@@ -168,9 +268,6 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("dispatch restore marker: %w", err)
|
||||
@@ -179,14 +276,19 @@ func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
// 当前代次必须在租约内,且就是许可申请者:旧 owner 晚到时租约/epoch 已变,零发送。
|
||||
// 撤销与 CAS 在同一事务线性化:这里拿激活行锁,撤销的 UPDATE 必须排队。
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
// 当前代次必须属于这个原 window 作业,且租户/epoch/租约都仍有效。lease 用
|
||||
// clock_timestamp():等锁可能跨越租约到期,事务起始的 now() 不是判据。
|
||||
var epoch int64
|
||||
var owner, attemptState string
|
||||
var owner, attemptState, attemptJob string
|
||||
var leaseUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT epoch, owner, state, lease_until FROM library_attempts
|
||||
SELECT epoch, owner, state, job_id, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
|
||||
&epoch, &owner, &attemptState, &leaseUntil)
|
||||
&epoch, &owner, &attemptState, &attemptJob, &leaseUntil)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("%w: no active attempt for operation %s",
|
||||
governance.ErrLibraryEpochStale, b.Key.OperationID)
|
||||
@@ -194,8 +296,13 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read active attempt: %w", err)
|
||||
}
|
||||
if attemptJob != b.Key.WindowJobID {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("%w: active attempt belongs to job %s, not the original window %s",
|
||||
governance.ErrLibraryEpochStale, attemptJob, b.Key.WindowJobID)
|
||||
}
|
||||
var leaseLive bool
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read attempt lease: %w", err)
|
||||
}
|
||||
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
@@ -203,6 +310,9 @@ WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.Oper
|
||||
Reason: fmt.Sprintf("attempt %s#%d is not the live authorized owner", b.Owner, b.Epoch)},
|
||||
fmt.Errorf("%w: attempt %s#%d is not current", governance.ErrLibraryEpochStale, b.Owner, b.Epoch)
|
||||
}
|
||||
if err := rejectRestoreDispatchPlanAlias(ctx, tx, b); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
|
||||
var id int64
|
||||
var state, stored string
|
||||
@@ -235,7 +345,7 @@ WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_diges
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='dispatching', dispatch_owner=$2, dispatch_epoch=$3, dispatch_nonce=$4,
|
||||
request_digest=$5, dispatch_committed_at=now(), updated_at=now()
|
||||
request_digest=$5, dispatch_committed_at=clock_timestamp(), updated_at=clock_timestamp()
|
||||
WHERE id=$1 AND state='prepared'`, id, b.Owner, b.Epoch, nonce, b.RequestDigest)
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("claim restore dispatch: %w", err)
|
||||
@@ -273,9 +383,6 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm restore marker: %w", err)
|
||||
@@ -284,25 +391,29 @@ func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.Resto
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := lockRestoreDispatchActivation(ctx, tx, b.Key.TargetIdentity); err != nil {
|
||||
return err
|
||||
}
|
||||
var epoch int64
|
||||
var owner string
|
||||
var owner, attemptJob string
|
||||
var leaseUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT epoch, owner, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(&epoch, &owner, &leaseUntil)
|
||||
SELECT epoch, owner, job_id, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(&epoch, &owner, &attemptJob, &leaseUntil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm restore marker attempt: %w", err)
|
||||
}
|
||||
var leaseLive bool
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
return fmt.Errorf("confirm restore marker lease: %w", err)
|
||||
}
|
||||
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
if attemptJob != b.Key.WindowJobID || owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=now(), updated_at=now()
|
||||
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=clock_timestamp(), updated_at=clock_timestamp()
|
||||
WHERE target_identity=$2 AND operation_id=$3 AND window_job_id=$4 AND plan_digest=$5
|
||||
AND index_name=$6 AND document_id=$7 AND phase=$8
|
||||
AND state='dispatching' AND binding_digest=$9`,
|
||||
@@ -320,9 +431,18 @@ WHERE target_identity=$2 AND operation_id=$3 AND window_job_id=$4 AND plan_diges
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockRestoreDispatchIdentity 以 job→operation 锁序把"这份许可是不是仍属于原作业"
|
||||
// 钉死在原持久锚点上:目标/scope/来源快照、写集摘要(若已登记)、维护窗口与实例
|
||||
// 指纹、恢复投递的 queue/operation 绑定都必须与不可变绑定一致。
|
||||
func lockRestoreDispatchIdentity(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
||||
var operationID string
|
||||
err := tx.QueryRow(ctx, `SELECT operation_id FROM library_jobs WHERE id=$1 FOR UPDATE`, b.Key.WindowJobID).Scan(&operationID)
|
||||
var operationID, scope, jobSource string
|
||||
var leaseOwner string
|
||||
var jobEpoch int64
|
||||
var jobLease *time.Time
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT operation_id, scope, target_snapshot_id, lease_owner, attempt_epoch, lease_until
|
||||
FROM library_jobs WHERE id=$1 FOR UPDATE`, b.Key.WindowJobID).Scan(
|
||||
&operationID, &scope, &jobSource, &leaseOwner, &jobEpoch, &jobLease)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("window job %s not found", b.Key.WindowJobID)
|
||||
}
|
||||
@@ -332,9 +452,93 @@ func lockRestoreDispatchIdentity(ctx context.Context, tx pgx.Tx, b libraryreleas
|
||||
if operationID != b.Key.OperationID {
|
||||
return errors.New("window job belongs to another operation")
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT id FROM library_operations WHERE id=$1 FOR UPDATE`, b.Key.OperationID); err != nil {
|
||||
if scope != b.Scope {
|
||||
return fmt.Errorf("window job scope %s does not match binding %s", scope, b.Scope)
|
||||
}
|
||||
if jobSource == "" || jobSource != b.SourceSnapshot {
|
||||
return fmt.Errorf("window job source snapshot %q does not match binding %q", jobSource, b.SourceSnapshot)
|
||||
}
|
||||
if leaseOwner != b.Owner || jobEpoch != b.Epoch || jobLease == nil {
|
||||
return fmt.Errorf("%w: window job lease is not held by attempt %s#%d", governance.ErrLibraryEpochStale, b.Owner, b.Epoch)
|
||||
}
|
||||
var jobLeaseLive bool
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > clock_timestamp(), false)`, jobLease).Scan(&jobLeaseLive); err != nil {
|
||||
return fmt.Errorf("read window job lease: %w", err)
|
||||
}
|
||||
if !jobLeaseLive {
|
||||
return fmt.Errorf("%w: window job lease expired while waiting for the dispatch CAS", governance.ErrLibraryEpochStale)
|
||||
}
|
||||
|
||||
var opScope, opKind, opSource, requestDigest, writesetDigest, firstBackup string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT scope, kind, target_snapshot_id, request_digest, COALESCE(writeset_digest,''), COALESCE(first_backup_snapshot_id,'')
|
||||
FROM library_operations WHERE id=$1 FOR UPDATE`, b.Key.OperationID).Scan(
|
||||
&opScope, &opKind, &opSource, &requestDigest, &writesetDigest, &firstBackup)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("operation %s not found", b.Key.OperationID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock operation: %w", err)
|
||||
}
|
||||
if opScope != b.Scope {
|
||||
return fmt.Errorf("operation scope %s does not match binding %s", opScope, b.Scope)
|
||||
}
|
||||
if opSource == "" || opSource != b.SourceSnapshot {
|
||||
return fmt.Errorf("operation source snapshot %q does not match binding %q", opSource, b.SourceSnapshot)
|
||||
}
|
||||
if requestDigest == "" {
|
||||
return errors.New("operation has no request digest anchor")
|
||||
}
|
||||
if writesetDigest != "" && writesetDigest != b.Key.PlanDigest {
|
||||
return fmt.Errorf("operation plan digest %s does not match binding %s", writesetDigest, b.Key.PlanDigest)
|
||||
}
|
||||
if firstBackup != "" && firstBackup != b.BackupSnapshot {
|
||||
return fmt.Errorf("operation backup snapshot %s does not match binding %s", firstBackup, b.BackupSnapshot)
|
||||
}
|
||||
_ = opKind
|
||||
|
||||
if b.TargetInstance != "" {
|
||||
var ownerJob, ownerScope string
|
||||
var released *time.Time
|
||||
var state string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT owner_job_id, owner_scope, state, released_at
|
||||
FROM library_instance_maintenance WHERE instance=$1 FOR UPDATE`, b.TargetInstance).Scan(
|
||||
&ownerJob, &ownerScope, &state, &released)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("target instance %s has no maintenance window", b.TargetInstance)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock instance maintenance: %w", err)
|
||||
}
|
||||
if released != nil || state == "released" {
|
||||
return errors.New("target instance maintenance window is already released")
|
||||
}
|
||||
if ownerJob != b.Key.WindowJobID || ownerScope != b.Scope {
|
||||
return fmt.Errorf("maintenance window belongs to %s (scope=%s), not window %s", ownerJob, ownerScope, b.Key.WindowJobID)
|
||||
}
|
||||
}
|
||||
|
||||
if b.RecoveryJobID != "" {
|
||||
var queueState string
|
||||
var lease string
|
||||
if err := tx.QueryRow(ctx, `SELECT state, COALESCE(lease_owner,'') FROM jobs WHERE id=$1`, b.RecoveryJobID).
|
||||
Scan(&queueState, &lease); err != nil {
|
||||
return fmt.Errorf("read recovery queue job: %w", err)
|
||||
}
|
||||
if queueState != "running" || lease == "" {
|
||||
return fmt.Errorf("recovery queue job %s is not a live delivery", b.RecoveryJobID)
|
||||
}
|
||||
var recoveryKind, recoveryScope, recoveryBackup string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT kind, scope, target_snapshot_id FROM library_operations WHERE id=$1`, b.RecoveryOperationID).
|
||||
Scan(&recoveryKind, &recoveryScope, &recoveryBackup); err != nil {
|
||||
return fmt.Errorf("read recovery operation: %w", err)
|
||||
}
|
||||
if recoveryKind != string(governance.LibraryJobRecover) || recoveryScope != b.Scope || recoveryBackup != b.BackupSnapshot {
|
||||
return fmt.Errorf("recovery operation %s is not bound to this window and backup", b.RecoveryOperationID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// G1:真实 PG 的"COMMIT 已到服务端、客户端收不到成功回包"。beforeCommit 钩子做不到
|
||||
// 这一点(它根本没调用 Commit)。这里用专用 TCP 转发代理:看到客户端 commit 帧后
|
||||
// 原样转发,然后立刻切断连接——服务端已提交,客户端拿到的是错误。
|
||||
|
||||
func startCommitAckDropProxy(t *testing.T, upstreamHostPort string) (string, *sync.WaitGroup) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
client, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer client.Close()
|
||||
server, err := net.Dial("tcp", upstreamHostPort)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer server.Close()
|
||||
var once sync.Once
|
||||
go func() {
|
||||
// 服务端 -> 客户端:正常转发;代理被关闭后自然结束。
|
||||
_, _ = io.Copy(client, server)
|
||||
}()
|
||||
buffer := make([]byte, 32*1024)
|
||||
for {
|
||||
n, err := client.Read(buffer)
|
||||
if n > 0 {
|
||||
chunk := buffer[:n]
|
||||
drop := onceFrames(chunk, &once)
|
||||
if _, werr := server.Write(chunk); werr != nil {
|
||||
return
|
||||
}
|
||||
if drop {
|
||||
// commit 已经交给服务端;不给客户端任何成功回包。
|
||||
_ = client.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
_ = listener.Close()
|
||||
wg.Wait()
|
||||
})
|
||||
return "127.0.0.1:" + strings.Split(listener.Addr().String(), ":")[1], nil
|
||||
}
|
||||
|
||||
// onceFrames 在客户端字节流里找简单查询帧 'Q' 且语句以 commit 开头。找到一次就返回
|
||||
// true(后续帧不再触发)。
|
||||
func onceFrames(chunk []byte, once *sync.Once) bool {
|
||||
for i := 0; i+5 <= len(chunk); i++ {
|
||||
if chunk[i] != 'Q' {
|
||||
continue
|
||||
}
|
||||
length := int(binary.BigEndian.Uint32(chunk[i+1 : i+5]))
|
||||
if length < 5 || i+1+length > len(chunk) {
|
||||
continue
|
||||
}
|
||||
sql := strings.TrimSpace(strings.ToLower(string(chunk[i+5 : i+1+length])))
|
||||
if strings.HasPrefix(sql, "commit") {
|
||||
triggered := false
|
||||
once.Do(func() { triggered = true })
|
||||
return triggered
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func proxyStore(t *testing.T, direct *pgxpool.Pool) (*Store, *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
conn := direct.Config().ConnConfig
|
||||
address, _ := startCommitAckDropProxy(t, net.JoinHostPort(conn.Host, strconv.Itoa(int(conn.Port))))
|
||||
target := url.URL{Scheme: "postgres", User: url.UserPassword(conn.User, conn.Password),
|
||||
Host: address, Path: "/" + conn.Database}
|
||||
query := target.Query()
|
||||
for key, value := range conn.RuntimeParams {
|
||||
query.Set(key, value)
|
||||
}
|
||||
query.Set("sslmode", "disable")
|
||||
target.RawQuery = query.Encode()
|
||||
pool, err := pgxpool.New(context.Background(), target.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
return NewStore(pool), pool
|
||||
}
|
||||
|
||||
func TestRestoreDispatchCommitAckLossKeepsDispatchingAndNoPermit(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-ackloss", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-ackloss", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-ackloss"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
proxied, _ := proxyStore(t, pool)
|
||||
permit, status, err := proxied.AcquireRestoreMarker(ctx, binding)
|
||||
if err == nil {
|
||||
t.Fatal("commit acknowledgement loss reported success")
|
||||
}
|
||||
if permit != nil {
|
||||
t.Fatal("a send permit was issued although the client never saw COMMIT succeed")
|
||||
}
|
||||
if status.State != libraryrelease.RestoreDispatchDispatching || !status.Held {
|
||||
t.Fatalf("status after ack loss = %+v", status)
|
||||
}
|
||||
// 服务端确实已提交:换一条直连读回 durable dispatching。
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchDispatching {
|
||||
t.Fatalf("server-side state after committed-but-unacknowledged commit = %s", state)
|
||||
}
|
||||
// 下一次调用(直连)不能把 dispatching 当 prepared 再发一次。
|
||||
permit, _, err = store.AcquireRestoreMarker(ctx, binding)
|
||||
if permit != nil || !errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Fatalf("next call after ack loss = permit=%v err=%v", permit, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchUndecidedTransactionBlocksThenFollowsRealOutcome(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-undecided", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-undecided", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-undecided"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 原事务仍未决:另一条连接已经把行推进到 dispatching,但没有 commit。
|
||||
open, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='dispatching', dispatch_owner='old', dispatch_epoch=1, dispatch_nonce='old-nonce',
|
||||
request_digest='old-request', dispatch_committed_at=clock_timestamp()
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
||||
binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest,
|
||||
binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
permit *libraryrelease.RestoreSendPermit
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
done <- result{permit, err}
|
||||
}()
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("acquire returned while the original transaction was undecided: permit=%v err=%v", got.permit, got.err)
|
||||
case <-time.After(700 * time.Millisecond):
|
||||
}
|
||||
if err := open.Rollback(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := <-done
|
||||
if got.err != nil || got.permit == nil {
|
||||
t.Fatalf("after a real rollback the call should win once: permit=%v err=%v", got.permit, got.err)
|
||||
}
|
||||
|
||||
// 第二个 effect:原事务选择 COMMIT,后续调用必须看到 dispatching 而不是 prepared。
|
||||
second := binding
|
||||
second.Key.DocumentID = "doc-2"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open2, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open2.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='dispatching', dispatch_owner='old', dispatch_epoch=1, dispatch_nonce='old-nonce-2',
|
||||
request_digest='old-request-2', dispatch_committed_at=clock_timestamp()
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
||||
second.Key.TargetIdentity, second.Key.OperationID, second.Key.WindowJobID, second.Key.PlanDigest,
|
||||
second.Key.Index, second.Key.DocumentID, second.Key.Phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done2 := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, second)
|
||||
done2 <- result{permit, err}
|
||||
}()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if err := open2.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got2 := <-done2
|
||||
if got2.permit != nil || !errors.Is(got2.err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Fatalf("after a real commit the call must not treat dispatching as prepared: permit=%v err=%v", got2.permit, got2.err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// DS01-F2:授权必须在**同一个 CAS 事务**里与撤销线性化,lease 必须用数据库实时
|
||||
// 时钟。以下都用真 PG 的锁等待,不用 scripted fake。
|
||||
|
||||
// 撤销先完成:Acquire 已经在事务里排队等 job 锁,撤销此时提交。CAS 不得再发许可。
|
||||
func TestRestoreDispatchRevocationWinsBeforeCASIsRejected(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-revoke-race", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-revoke-race", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-revoke-race"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lock, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := lock.Exec(ctx, `SELECT id FROM library_jobs WHERE id=$1 FOR UPDATE`, binding.Key.WindowJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
permit *libraryrelease.RestoreSendPermit
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
done <- result{permit, err}
|
||||
}()
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if err := store.RevokeRestoreDispatch(ctx, "target-ds01", now); err != nil {
|
||||
t.Fatalf("revoke: %v", err)
|
||||
}
|
||||
if err := lock.Rollback(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := <-done
|
||||
if got.permit != nil {
|
||||
t.Fatal("revocation committed before the CAS but a send permit was still issued")
|
||||
}
|
||||
if !errors.Is(got.err, libraryrelease.ErrRestoreDispatchUnavailable) {
|
||||
t.Fatalf("revocation race error = %v", got.err)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
t.Fatalf("row advanced to %s after a winning revocation", state)
|
||||
}
|
||||
}
|
||||
|
||||
// CAS 先线性化:Acquire 已经锁住激活行、正在等 effect 行;撤销必须排队。CAS 提交后
|
||||
// 许可属于一次已派发,第二次调用必须 held。
|
||||
func TestRestoreDispatchCASWinsBeforeRevocationIssuesOnePermit(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-cas-race", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-cas-race", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-cas-race"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lock, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := lock.Exec(ctx, `SELECT id FROM library_index_restore_dispatch WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4 AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
|
||||
binding.Key.TargetIdentity, binding.Key.OperationID, binding.Key.WindowJobID, binding.Key.PlanDigest,
|
||||
binding.Key.Index, binding.Key.DocumentID, binding.Key.Phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
permit *libraryrelease.RestoreSendPermit
|
||||
err error
|
||||
}
|
||||
acquired := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
acquired <- result{permit, err}
|
||||
}()
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
revoked := make(chan error, 1)
|
||||
go func() { revoked <- store.RevokeRestoreDispatch(ctx, "target-ds01", now) }()
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if err := lock.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := <-acquired
|
||||
if got.err != nil || got.permit == nil {
|
||||
t.Fatalf("CAS should have won: permit=%v err=%v", got.permit, got.err)
|
||||
}
|
||||
if err := <-revoked; err != nil {
|
||||
t.Fatalf("revocation after CAS: %v", err)
|
||||
}
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
if permit != nil || err == nil {
|
||||
t.Fatalf("second acquire after dispatching = permit=%v err=%v", permit, err)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchDispatching {
|
||||
t.Fatalf("state = %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
// 等锁跨过租约:事务开始时刻仍有效、真正做 CAS 时已过期。必须用数据库实时时钟拒绝。
|
||||
func TestRestoreDispatchLeaseExpiryWhileWaitingForLockIsRejected(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-lease", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-lease", "worker-a#q1#0", 500*time.Millisecond, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-lease"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lock, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := lock.Exec(ctx, `SELECT id FROM library_jobs WHERE id=$1 FOR UPDATE`, binding.Key.WindowJobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
permit *libraryrelease.RestoreSendPermit
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
done <- result{permit, err}
|
||||
}()
|
||||
time.Sleep(900 * time.Millisecond)
|
||||
if err := lock.Rollback(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := <-done
|
||||
if got.permit != nil {
|
||||
t.Fatal("permit issued although the attempt lease expired while waiting for the job lock")
|
||||
}
|
||||
if got.err == nil || (!strings.Contains(got.err.Error(), "not current") &&
|
||||
!strings.Contains(got.err.Error(), "lease expired")) {
|
||||
t.Fatalf("expired lease error = %v", got.err)
|
||||
}
|
||||
}
|
||||
|
||||
// 错目标/错来源/换 plan 别名/错 UUID 都必须在 CAS 前失败,且台账不动。
|
||||
func TestRestoreDispatchRejectsWrongAnchorsAndPlanAlias(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-anchors", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-anchors", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-anchors"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wrongSource := binding
|
||||
wrongSource.SourceSnapshot = "snap-other"
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongSource); permit != nil || err == nil {
|
||||
t.Fatalf("wrong source acquired: permit=%v err=%v", permit, err)
|
||||
}
|
||||
wrongUUID := binding
|
||||
wrongUUID.IndexUUID = "uuid-replaced"
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongUUID); permit != nil ||
|
||||
!errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Fatalf("replaced index uuid acquired: permit=%v err=%v", permit, err)
|
||||
}
|
||||
wrongFingerprint := binding
|
||||
wrongFingerprint.TargetFingerprint = "fingerprint-other"
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, wrongFingerprint); permit != nil || err == nil {
|
||||
t.Fatalf("replaced target fingerprint acquired: permit=%v err=%v", permit, err)
|
||||
}
|
||||
alias := binding
|
||||
alias.Key.PlanDigest = "plan-alias"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, alias); err == nil ||
|
||||
!strings.Contains(err.Error(), "another plan") {
|
||||
t.Fatalf("plan alias was accepted: %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("dispatch rows = %d, want exactly the original plan row", count)
|
||||
}
|
||||
// 恢复投递身份缺失/错误:Prepare 就必须失败,不能等到发送前。
|
||||
recovery := binding
|
||||
recovery.RecoveryJobID = "queue-missing"
|
||||
recovery.RecoveryOperationID = "op-missing"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, recovery); err == nil {
|
||||
t.Fatal("unknown recovery delivery identity was accepted")
|
||||
}
|
||||
// 真实的 recovery queue/operation 绑定可以被验证通过;换成另一份 backup 的
|
||||
// recovery operation 则必须拒绝。
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs (id, type, payload, state, lease_owner, idempotency_key)
|
||||
VALUES ('queue-recovery-1','library.recover','{}'::jsonb,'running','worker-a','queue-recovery-1')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: "op-recovery-1", Scope: "internal", Kind: governance.LibraryJobRecover,
|
||||
TargetSnapshotID: "snap-backup", RequestDigest: "digest-recovery", OperationKey: "recovery-1",
|
||||
Actor: "tester", CreatedAt: now.UTC(),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validRecovery := binding
|
||||
validRecovery.Key.DocumentID = "doc-recovery"
|
||||
validRecovery.RecoveryJobID = "queue-recovery-1"
|
||||
validRecovery.RecoveryOperationID = "op-recovery-1"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, validRecovery); err != nil {
|
||||
t.Fatalf("valid recovery delivery binding: %v", err)
|
||||
}
|
||||
wrongRecovery := validRecovery
|
||||
wrongRecovery.Key.DocumentID = "doc-recovery-2"
|
||||
wrongRecovery.RecoveryOperationID = seededOperationID(t, pool, binding.Key.WindowJobID)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, wrongRecovery); err == nil {
|
||||
t.Fatal("recovery operation for another intent was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// DS01-F3:撤销是终局——SQL 不能把 revoked_at 改回 NULL,也不能删掉整行;down 在
|
||||
// 只有 activation 证据时同样拒绝。
|
||||
func TestRestoreDispatchActivationCannotBeResetOrDeleted(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RevokeRestoreDispatch(ctx, "target-ds01", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_index_restore_dispatch_activation SET revoked_at=NULL WHERE target_identity=$1`, "target-ds01"); err == nil {
|
||||
t.Fatal("revoked_at was reset to NULL")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, "target-ds01"); err == nil {
|
||||
t.Fatal("activation row was deleted")
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000078_library_restore_dispatch.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, string(down)); err == nil ||
|
||||
!strings.Contains(err.Error(), "activation evidence") {
|
||||
t.Fatalf("down with activation-only evidence = %v", err)
|
||||
}
|
||||
var revoked *time.Time
|
||||
if err := pool.QueryRow(ctx, `SELECT revoked_at FROM library_index_restore_dispatch_activation WHERE target_identity=$1`, "target-ds01").Scan(&revoked); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if revoked == nil {
|
||||
t.Fatal("revocation evidence did not survive")
|
||||
}
|
||||
}
|
||||
|
||||
// 激活只接受结构与身份都可核验的证据文件。
|
||||
func TestRestoreDispatchActivationRequiresVerifiableEvidence(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
var database string
|
||||
if err := pool.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
exited := now.Add(-time.Minute)
|
||||
valid := libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolVersion: 1, TargetIdentity: "target-ds01", TargetFingerprint: "fingerprint-1",
|
||||
TargetInstance: "inst-1", GovernanceDatabase: database, ProtocolBuild: "build-1",
|
||||
AttestedBy: "deploy-bot", AttestedAt: now,
|
||||
Writers: []libraryrelease.RestoreDispatchWriterDrain{{
|
||||
Name: "worker-1", Kind: "worker", Stopped: true, ExitedAt: &exited,
|
||||
QueueDrained: true, DrainEvidence: "sha256:drain",
|
||||
}},
|
||||
}
|
||||
write := func(t *testing.T, mutate func(*libraryrelease.RestoreDispatchActivationEvidence), digest string) (string, string) {
|
||||
t.Helper()
|
||||
raw, err := libraryrelease.RestoreDispatchActivationEvidence(valid).CanonicalBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mutate != nil {
|
||||
var copy libraryrelease.RestoreDispatchActivationEvidence
|
||||
if err := json.Unmarshal(raw, ©); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mutate(©)
|
||||
raw, err = copy.CanonicalBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
path := t.TempDir() + "/evidence.json"
|
||||
if err := os.WriteFile(path, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if digest == "" {
|
||||
sum := sha256.Sum256(raw)
|
||||
digest = hex.EncodeToString(sum[:])
|
||||
}
|
||||
return path, digest
|
||||
}
|
||||
// 错摘要
|
||||
path, _ := write(t, nil, "deadbeef")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, "deadbeef", "target-ds01", "fingerprint-1", now); err == nil {
|
||||
t.Fatal("wrong evidence digest was accepted")
|
||||
}
|
||||
// 错目标指纹
|
||||
path, digest := write(t, nil, "")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, digest, "target-ds01", "fingerprint-other", now); err == nil {
|
||||
t.Fatal("evidence for another target fingerprint was accepted")
|
||||
}
|
||||
// 未排空
|
||||
path, digest = write(t, func(e *libraryrelease.RestoreDispatchActivationEvidence) {
|
||||
e.Writers[0].QueueDrained = false
|
||||
}, "")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, digest, "target-ds01", "fingerprint-1", now); err == nil {
|
||||
t.Fatal("evidence with an undrained writer was accepted")
|
||||
}
|
||||
// 治理库身份不符
|
||||
path, digest = write(t, func(e *libraryrelease.RestoreDispatchActivationEvidence) {
|
||||
e.GovernanceDatabase = "some-other-database"
|
||||
}, "")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, digest, "target-ds01", "fingerprint-1", now); err == nil {
|
||||
t.Fatal("evidence for another governance database was accepted")
|
||||
}
|
||||
// 过期
|
||||
path, digest = write(t, func(e *libraryrelease.RestoreDispatchActivationEvidence) {
|
||||
old := now.Add(-24 * time.Hour)
|
||||
e.AttestedAt = old
|
||||
e.Writers[0].ExitedAt = &old
|
||||
}, "")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, digest, "target-ds01", "fingerprint-1", now); err == nil {
|
||||
t.Fatal("stale evidence was accepted")
|
||||
}
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch_activation`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("invalid evidence created %d activation rows", count)
|
||||
}
|
||||
// 有效证据
|
||||
path, digest = write(t, nil, "")
|
||||
if err := store.ActivateRestoreDispatch(ctx, path, digest, "target-ds01", "fingerprint-1", now); err != nil {
|
||||
t.Fatalf("valid evidence rejected: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,12 @@ package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -33,7 +38,8 @@ func ds01Binding() libraryrelease.RestoreMarkerBinding {
|
||||
PlanDigest: "plan-1", Index: "kg-index", DocumentID: "doc-1",
|
||||
Phase: libraryrelease.RestoreMarkerPhaseCreate,
|
||||
},
|
||||
Scope: "internal", SourceSnapshot: "snap-src", BackupSnapshot: "snap-backup",
|
||||
Scope: "internal", SourceSnapshot: "snap-target", BackupSnapshot: "snap-backup",
|
||||
TargetInstance: "inst-1", TargetFingerprint: "fingerprint-1", IndexUUID: "uuid-1",
|
||||
ChangeDigest: "change-1", MarkerDigest: "marker-1", RequestDigest: "request-1",
|
||||
Owner: "worker-a#q1#0", Epoch: 1,
|
||||
}
|
||||
@@ -63,7 +69,7 @@ func TestRestoreDispatchPrepareIsUniqueAndBindingBound(t *testing.T) {
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-prep", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
@@ -101,7 +107,7 @@ func TestRestoreDispatchConcurrentAcquireHasSingleWinner(t *testing.T) {
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-race", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
@@ -156,7 +162,7 @@ func TestRestoreDispatchIsMonotoneAndAppendOnly(t *testing.T) {
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-mono", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
@@ -211,7 +217,7 @@ func TestRestoreDispatchRejectsStaleEpochBeforeSend(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
@@ -258,7 +264,7 @@ func TestRestoreDispatchCommitWithoutExplicitSuccessYieldsNoPermit(t *testing.T)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-commit", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
@@ -310,7 +316,7 @@ func TestRestoreDispatchRevocationIsFinal(t *testing.T) {
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-revoke", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RevokeRestoreDispatch(ctx, "target-ds01", now); err != nil {
|
||||
@@ -322,7 +328,7 @@ func TestRestoreDispatchRevocationIsFinal(t *testing.T) {
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); !errors.Is(err, libraryrelease.ErrRestoreDispatchUnavailable) {
|
||||
t.Fatalf("revoked target still prepared: %v", err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err == nil {
|
||||
if err := activateRestoreDispatchForTest(t, store, "target-ds01", "fingerprint-1"); err == nil {
|
||||
t.Fatal("revoked activation was reopened with the same evidence")
|
||||
}
|
||||
var revoked *time.Time
|
||||
@@ -333,3 +339,38 @@ func TestRestoreDispatchRevocationIsFinal(t *testing.T) {
|
||||
t.Fatal("revocation was silently cleared")
|
||||
}
|
||||
}
|
||||
|
||||
// activateRestoreDispatchForTest 写一份结构完整的激活证据并登记到隔离治理库。
|
||||
// 治理库名取自当前连接:激活 API 会与现场库身份比对。
|
||||
func activateRestoreDispatchForTest(t *testing.T, store *Store, targetIdentity, fingerprint string) error {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
var database string
|
||||
if err := store.db.QueryRow(ctx, `SELECT current_database()`).Scan(&database); err != nil {
|
||||
t.Fatalf("read governance database: %v", err)
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
exited := now.Add(-time.Minute)
|
||||
evidence := libraryrelease.RestoreDispatchActivationEvidence{
|
||||
ProtocolVersion: 1, TargetIdentity: targetIdentity, TargetFingerprint: fingerprint,
|
||||
TargetInstance: "inst-1", GovernanceDatabase: database, ProtocolBuild: "test-build",
|
||||
AttestedBy: "ds01-test", AttestedAt: now,
|
||||
Writers: []libraryrelease.RestoreDispatchWriterDrain{{
|
||||
Name: "worker", Kind: "worker", Stopped: true, ExitedAt: &exited,
|
||||
QueueDrained: true, DrainEvidence: "sha256:drain",
|
||||
}},
|
||||
}
|
||||
raw, err := json.Marshal(evidence)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "activation.json")
|
||||
if err := os.WriteFile(path, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return store.ActivateRestoreDispatch(ctx, path, hex.EncodeToString(sum[:]), targetIdentity, fingerprint, now)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
-- 已有派发记录时拒绝自动回滚:这些行是"是否发送过"的唯一证明,
|
||||
-- 用 down 抹掉等于把 unknown 重新伪装成 prepared。人工处置必须先导出并存档。
|
||||
-- 已有派发记录或激活/撤销证据时拒绝自动回滚:这些行是"是否发送过 / 是否曾被授权"
|
||||
-- 的唯一证明,用 down 抹掉等于把 unknown 重新伪装成 prepared,或把已撤销的激活
|
||||
-- 悄悄删掉。人工处置必须先导出并存档。仅有一条已撤销 activation、尚无任何 effect
|
||||
-- 行时同样拒绝——撤销事实本身也是证据。
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM library_index_restore_dispatch LIMIT 1) THEN
|
||||
IF to_regclass('library_index_restore_dispatch') IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM library_index_restore_dispatch LIMIT 1) THEN
|
||||
RAISE EXCEPTION 'refusing to drop restore dispatch evidence with recorded rows';
|
||||
END IF;
|
||||
IF to_regclass('library_index_restore_dispatch_activation') IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM library_index_restore_dispatch_activation LIMIT 1) THEN
|
||||
RAISE EXCEPTION 'refusing to drop restore dispatch activation evidence';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_monotone ON library_index_restore_dispatch;
|
||||
|
||||
@@ -67,26 +67,46 @@ CREATE TRIGGER library_index_restore_dispatch_monotone
|
||||
BEFORE UPDATE OR DELETE ON library_index_restore_dispatch
|
||||
FOR EACH ROW EXECUTE FUNCTION library_index_restore_dispatch_guard();
|
||||
|
||||
-- 激活登记:配置开关不是独占证明。只有部署流程把"旧 writer 已退出并排空"的证据
|
||||
-- 登记到这个目标上,协议才对该目标生效;撤销后立即 fail closed。
|
||||
-- 激活登记:配置开关不是独占证明。只有部署流程把一份可核验的"旧 writer 已退出并
|
||||
-- 排空"证据文件登记到这个目标上,协议才对该目标生效。
|
||||
--
|
||||
-- 除 revoked_at 外全部不可变;撤销是终局(NULL -> 时间戳之后不能再改回 NULL、不能
|
||||
-- 改时间、不能删除整行)。要重新开放同一物理目标,只能换新的目标身份并重新登记,
|
||||
-- 不能"把旧行归档/删掉再插一行"。
|
||||
CREATE TABLE IF NOT EXISTS library_index_restore_dispatch_activation (
|
||||
target_identity text PRIMARY KEY,
|
||||
protocol_version integer NOT NULL CHECK (protocol_version = 1),
|
||||
target_fingerprint text NOT NULL CHECK (target_fingerprint <> ''),
|
||||
target_instance text NOT NULL CHECK (target_instance <> ''),
|
||||
governance_database text NOT NULL CHECK (governance_database <> ''),
|
||||
protocol_build text NOT NULL CHECK (protocol_build <> ''),
|
||||
activated_by text NOT NULL CHECK (activated_by <> ''),
|
||||
drain_evidence_digest text NOT NULL CHECK (drain_evidence_digest <> ''),
|
||||
evidence jsonb NOT NULL,
|
||||
attested_at timestamptz NOT NULL,
|
||||
activated_at timestamptz NOT NULL DEFAULT now(),
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION library_index_restore_dispatch_activation_guard() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation evidence is append-only';
|
||||
END IF;
|
||||
IF NEW.target_identity <> OLD.target_identity OR NEW.protocol_version <> OLD.protocol_version OR
|
||||
NEW.activated_by <> OLD.activated_by OR NEW.drain_evidence_digest <> OLD.drain_evidence_digest OR
|
||||
NEW.activated_at <> OLD.activated_at THEN
|
||||
NEW.target_fingerprint <> OLD.target_fingerprint OR NEW.target_instance <> OLD.target_instance OR
|
||||
NEW.governance_database <> OLD.governance_database OR NEW.protocol_build <> OLD.protocol_build OR
|
||||
NEW.activated_by <> OLD.activated_by OR
|
||||
NEW.drain_evidence_digest <> OLD.drain_evidence_digest OR NEW.evidence <> OLD.evidence OR
|
||||
NEW.attested_at <> OLD.attested_at OR NEW.activated_at <> OLD.activated_at THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation is immutable except revocation';
|
||||
END IF;
|
||||
IF OLD.revoked_at IS NOT NULL AND NEW.revoked_at IS NOT NULL AND NEW.revoked_at <> OLD.revoked_at THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation revocation is final';
|
||||
-- 终局:已撤销行既不能改回 NULL,也不能改写成另一个时间。
|
||||
IF OLD.revoked_at IS NOT NULL THEN
|
||||
IF NEW.revoked_at IS NULL OR NEW.revoked_at <> OLD.revoked_at THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation revocation is final';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
@@ -94,5 +114,5 @@ $$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_activation_guard_trg ON library_index_restore_dispatch_activation;
|
||||
CREATE TRIGGER library_index_restore_dispatch_activation_guard_trg
|
||||
BEFORE UPDATE ON library_index_restore_dispatch_activation
|
||||
BEFORE UPDATE OR DELETE ON library_index_restore_dispatch_activation
|
||||
FOR EACH ROW EXECUTE FUNCTION library_index_restore_dispatch_activation_guard();
|
||||
Reference in New Issue
Block a user