fad26166a4
为什么:删除型恢复的独占 create 在 ES 侧没有"从未写过"的持久证据, missing/stats/内存计数都不能当未派发证明。改成为每个固定 effect 维护 单调台账:prepared -> dispatching -> marker_confirmed,绑定换不了、 状态回退不了、行删不掉。 许可只在独立短事务里 CAS 并 COMMIT 明确成功后返回,消费一次即作废; 提交结果不明时不发许可。激活登记是配置之外的第二个条件。 验证:真实 PG(隔离 schema)覆盖并发唯一赢家、旧 epoch 拒绝、 单调/绑定/删行拒绝、提交不明零许可、无激活 fail closed、down 拒绝抹证据。
367 lines
16 KiB
Go
367 lines
16 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
"knowledge-governance/internal/libraryrelease"
|
|
)
|
|
|
|
// DS01:恢复标记派发的持久准入。
|
|
//
|
|
// 关键点不是"多了一张表",而是发送许可的产生方式:
|
|
// - Prepare 只建立 prepared(不返回许可);
|
|
// - Acquire 在**独立短事务**里按 job→operation→active attempt→effect 的顺序取锁,
|
|
// 校验当前 epoch/owner/lease 后做 prepared→dispatching 的 CAS,只有 COMMIT 明确
|
|
// 成功才返回一次性许可;
|
|
// - Inspect 纯读,永不返回许可;
|
|
// - 表本身拒绝状态回退、绑定变化与删行(迁移里的触发器)。
|
|
//
|
|
// 外层 WithLibraryExecution 的事务只 rollback、从不 commit,所以这里绝不能用它:
|
|
// 在那种事务里写 dispatching,发出请求后账会被回滚成"未派发"。
|
|
|
|
var ErrRestoreDispatchNotActivated = libraryrelease.ErrRestoreDispatchUnavailable
|
|
|
|
// ActivateRestoreDispatch 登记"该目标已完成旧 writer 退出/排空"的部署证据。
|
|
//
|
|
// 它记录的是运维声明,不验证声明为真;代码只能保证没有这条登记就不发放许可。
|
|
func (s *Store) ActivateRestoreDispatch(ctx context.Context, targetIdentity, activatedBy, drainEvidenceDigest string, now time.Time) error {
|
|
if targetIdentity == "" || activatedBy == "" || drainEvidenceDigest == "" {
|
|
return errors.New("restore dispatch activation requires target, actor and drain evidence digest")
|
|
}
|
|
_, err := s.db.Exec(ctx, `
|
|
INSERT INTO library_index_restore_dispatch_activation (target_identity, protocol_version, activated_by, drain_evidence_digest, activated_at)
|
|
VALUES ($1, 1, $2, $3, $4)
|
|
ON CONFLICT (target_identity) DO UPDATE
|
|
SET revoked_at = NULL
|
|
WHERE library_index_restore_dispatch_activation.revoked_at IS NOT NULL`,
|
|
targetIdentity, activatedBy, drainEvidenceDigest, now.UTC())
|
|
if err != nil {
|
|
return fmt.Errorf("activate restore dispatch: %w", err)
|
|
}
|
|
return s.requireRestoreDispatchActivation(ctx, targetIdentity)
|
|
}
|
|
|
|
func (s *Store) RevokeRestoreDispatch(ctx context.Context, targetIdentity string, now time.Time) error {
|
|
tag, err := s.db.Exec(ctx, `
|
|
UPDATE library_index_restore_dispatch_activation SET revoked_at=$2
|
|
WHERE target_identity=$1 AND revoked_at IS NULL`, targetIdentity, now.UTC())
|
|
if err != nil {
|
|
return fmt.Errorf("revoke restore dispatch: %w", err)
|
|
}
|
|
if tag.RowsAffected() != 1 {
|
|
return errors.New("restore dispatch activation is not active for this target")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) requireRestoreDispatchActivation(ctx context.Context, targetIdentity string) error {
|
|
var one int
|
|
err := s.db.QueryRow(ctx, `
|
|
SELECT 1 FROM library_index_restore_dispatch_activation
|
|
WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetIdentity).Scan(&one)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("read restore dispatch activation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func restoreDispatchBindingDigest(b libraryrelease.RestoreMarkerBinding) (string, error) {
|
|
return b.Digest()
|
|
}
|
|
|
|
func (s *Store) InspectRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
|
|
digest, err := restoreDispatchBindingDigest(b)
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, err
|
|
}
|
|
var state, stored string
|
|
err = s.db.QueryRow(ctx, `
|
|
SELECT state, binding_digest FROM library_index_restore_dispatch
|
|
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
|
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
|
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
|
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&state, &stored)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
|
|
}
|
|
if stored != digest {
|
|
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
|
|
}
|
|
switch state {
|
|
case libraryrelease.RestoreDispatchPrepared:
|
|
return libraryrelease.RestoreDispatchStatus{State: state}, true, nil
|
|
case libraryrelease.RestoreDispatchDispatching:
|
|
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "dispatch outcome unknown: no further create is permitted"}, true, nil
|
|
case libraryrelease.RestoreDispatchMarkerConfirmed:
|
|
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "marker already dispatched; only the original confirmation path may continue"}, true, nil
|
|
default:
|
|
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "unrecognised dispatch state"}, true, nil
|
|
}
|
|
}
|
|
|
|
// PrepareRestoreMarker 只建立 prepared:没有发送权,也不把已有 dispatching 抬回 prepared。
|
|
func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, error) {
|
|
digest, err := restoreDispatchBindingDigest(b)
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
tx, err := s.db.Begin(ctx)
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("prepare restore dispatch: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
// 与 BeginLibraryAttempt 同序:先作业行、再 operation 行。这样"准备"不会与代次切换
|
|
// 交错成一份属于旧 identity 的台账。
|
|
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
INSERT INTO library_index_restore_dispatch
|
|
(target_identity, operation_id, window_job_id, plan_digest, index_name, document_id, phase, binding_digest, state)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'prepared')
|
|
ON CONFLICT ON CONSTRAINT library_index_restore_dispatch_identity DO NOTHING`,
|
|
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
|
b.Key.Index, b.Key.DocumentID, b.Key.Phase, digest); err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("record prepared restore dispatch: %w", err)
|
|
}
|
|
status, found, err := inspectRestoreDispatchTx(ctx, tx, b)
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
if !found {
|
|
return libraryrelease.RestoreDispatchStatus{}, errors.New("prepared restore dispatch was not durable")
|
|
}
|
|
if status.Held {
|
|
return status, fmt.Errorf("%w: %s", libraryrelease.ErrRestoreDispatchHeld, status.Reason)
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("commit prepared restore dispatch: %w", err)
|
|
}
|
|
return status, nil
|
|
}
|
|
|
|
// acquireRestoreMarker 的 beforeCommit 是提交结果不明这一档的测试接缝:钩子返回错误时
|
|
// 事务回滚,调用方不得发送。生产路径传 nil。
|
|
func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding, beforeCommit func() error) (*libraryrelease.RestoreSendPermit, libraryrelease.RestoreDispatchStatus, error) {
|
|
digest, err := restoreDispatchBindingDigest(b)
|
|
if err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
tx, err := s.db.Begin(ctx)
|
|
if err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("dispatch restore marker: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
// 当前代次必须在租约内,且就是许可申请者:旧 owner 晚到时租约/epoch 已变,零发送。
|
|
var epoch int64
|
|
var owner, attemptState string
|
|
var leaseUntil *time.Time
|
|
err = tx.QueryRow(ctx, `
|
|
SELECT epoch, owner, state, lease_until FROM library_attempts
|
|
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
|
|
&epoch, &owner, &attemptState, &leaseUntil)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("%w: no active attempt for operation %s",
|
|
governance.ErrLibraryEpochStale, b.Key.OperationID)
|
|
}
|
|
if err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read active attempt: %w", err)
|
|
}
|
|
var leaseLive bool
|
|
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read attempt lease: %w", err)
|
|
}
|
|
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true,
|
|
Reason: fmt.Sprintf("attempt %s#%d is not the live authorized owner", b.Owner, b.Epoch)},
|
|
fmt.Errorf("%w: attempt %s#%d is not current", governance.ErrLibraryEpochStale, b.Owner, b.Epoch)
|
|
}
|
|
|
|
var id int64
|
|
var state, stored string
|
|
err = tx.QueryRow(ctx, `
|
|
SELECT id, state, binding_digest FROM library_index_restore_dispatch
|
|
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
|
AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
|
|
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
|
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&id, &state, &stored)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, errors.New("no prepared restore dispatch record: refusing to send")
|
|
}
|
|
if err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err)
|
|
}
|
|
if stored != digest {
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld
|
|
}
|
|
if state != libraryrelease.RestoreDispatchPrepared {
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld
|
|
}
|
|
|
|
var raw [16]byte
|
|
if _, err := rand.Read(raw[:]); err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
|
}
|
|
nonce := hex.EncodeToString(raw[:])
|
|
tag, err := tx.Exec(ctx, `
|
|
UPDATE library_index_restore_dispatch
|
|
SET state='dispatching', dispatch_owner=$2, dispatch_epoch=$3, dispatch_nonce=$4,
|
|
request_digest=$5, dispatch_committed_at=now(), updated_at=now()
|
|
WHERE id=$1 AND state='prepared'`, id, b.Owner, b.Epoch, nonce, b.RequestDigest)
|
|
if err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("claim restore dispatch: %w", err)
|
|
}
|
|
if tag.RowsAffected() != 1 {
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "dispatch CAS lost"}, libraryrelease.ErrRestoreDispatchHeld
|
|
}
|
|
if beforeCommit != nil {
|
|
if err := beforeCommit(); err != nil {
|
|
// 提交结果不明:本调用不发送,也不把状态当成 prepared 重试。
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
|
|
Reason: "commit outcome unknown"}, fmt.Errorf("commit restore dispatch: %w", err)
|
|
}
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
|
|
Reason: "commit outcome unknown"}, fmt.Errorf("commit restore dispatch: %w", err)
|
|
}
|
|
return libraryrelease.IssueRestoreSendPermit(b.Key, digest, nonce),
|
|
libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching}, nil
|
|
}
|
|
|
|
func (s *Store) AcquireRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (*libraryrelease.RestoreSendPermit, libraryrelease.RestoreDispatchStatus, error) {
|
|
return s.acquireRestoreMarker(ctx, b, nil)
|
|
}
|
|
|
|
// ConfirmRestoreMarker 只记录真实回执(不再发送)。状态仍是 dispatching 时才允许;
|
|
// 回执本身来自本次受控发送或同计划标记的实时核实,不由客户端布尔值决定。
|
|
func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding, receipt []byte) error {
|
|
if len(receipt) == 0 || !json.Valid(receipt) {
|
|
return errors.New("restore marker confirmation requires a real receipt")
|
|
}
|
|
digest, err := restoreDispatchBindingDigest(b)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.Begin(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("confirm restore marker: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
|
return err
|
|
}
|
|
var epoch int64
|
|
var owner string
|
|
var leaseUntil *time.Time
|
|
err = tx.QueryRow(ctx, `
|
|
SELECT epoch, owner, lease_until FROM library_attempts
|
|
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(&epoch, &owner, &leaseUntil)
|
|
if err != nil {
|
|
return fmt.Errorf("confirm restore marker attempt: %w", err)
|
|
}
|
|
var leaseLive bool
|
|
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
|
return fmt.Errorf("confirm restore marker lease: %w", err)
|
|
}
|
|
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
|
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
|
|
}
|
|
tag, err := tx.Exec(ctx, `
|
|
UPDATE library_index_restore_dispatch
|
|
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=now(), updated_at=now()
|
|
WHERE target_identity=$2 AND operation_id=$3 AND window_job_id=$4 AND plan_digest=$5
|
|
AND index_name=$6 AND document_id=$7 AND phase=$8
|
|
AND state='dispatching' AND binding_digest=$9`,
|
|
string(receipt), b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
|
b.Key.Index, b.Key.DocumentID, b.Key.Phase, digest)
|
|
if err != nil {
|
|
return fmt.Errorf("confirm restore marker: %w", err)
|
|
}
|
|
if tag.RowsAffected() != 1 {
|
|
return errors.New("restore marker confirmation did not match a dispatching record")
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return fmt.Errorf("commit restore marker confirmation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func lockRestoreDispatchIdentity(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
|
var operationID string
|
|
err := tx.QueryRow(ctx, `SELECT operation_id FROM library_jobs WHERE id=$1 FOR UPDATE`, b.Key.WindowJobID).Scan(&operationID)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return fmt.Errorf("window job %s not found", b.Key.WindowJobID)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("lock window job: %w", err)
|
|
}
|
|
if operationID != b.Key.OperationID {
|
|
return errors.New("window job belongs to another operation")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT id FROM library_operations WHERE id=$1 FOR UPDATE`, b.Key.OperationID); err != nil {
|
|
return fmt.Errorf("lock operation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func inspectRestoreDispatchTx(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
|
|
digest, err := restoreDispatchBindingDigest(b)
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, err
|
|
}
|
|
var state, stored string
|
|
err = tx.QueryRow(ctx, `
|
|
SELECT state, binding_digest FROM library_index_restore_dispatch
|
|
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
|
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
|
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
|
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&state, &stored)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
|
|
}
|
|
if stored != digest {
|
|
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
|
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
|
|
}
|
|
return libraryrelease.RestoreDispatchStatus{State: state,
|
|
Held: state != libraryrelease.RestoreDispatchPrepared}, true, nil
|
|
}
|