feat(library): 新增恢复标记派发台账与一次性许可
为什么:删除型恢复的独占 create 在 ES 侧没有"从未写过"的持久证据, missing/stats/内存计数都不能当未派发证明。改成为每个固定 effect 维护 单调台账:prepared -> dispatching -> marker_confirmed,绑定换不了、 状态回退不了、行删不掉。 许可只在独立短事务里 CAS 并 COMMIT 明确成功后返回,消费一次即作废; 提交结果不明时不发许可。激活登记是配置之外的第二个条件。 验证:真实 PG(隔离 schema)覆盖并发唯一赢家、旧 epoch 拒绝、 单调/绑定/删行拒绝、提交不明零许可、无激活 fail closed、down 拒绝抹证据。
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 恢复标记派发协议(DS01 窄协议)。
|
||||
//
|
||||
// 这里定义的唯一持久事实是:某个固定的删除型恢复 effect 是否已经有人拿到过
|
||||
// "向索引发送一次 `_create`"的许可。它不是"ES 历史上从未有人写过这个 ID"的证明,
|
||||
// 也不是目标端 fence。判据是单调的持久状态,不是 missing、stats 或内存布尔值。
|
||||
const (
|
||||
// RestoreMarkerPhaseCreate 是恢复期独占 create(删除型文档的原文标记)的唯一相位。
|
||||
// 其它相位(apply create、compensation DELETE、clean CAS)不得借用它取得发送许可。
|
||||
RestoreMarkerPhaseCreate = "restore-marker-create"
|
||||
|
||||
RestoreDispatchPrepared = "prepared"
|
||||
RestoreDispatchDispatching = "dispatching"
|
||||
RestoreDispatchMarkerConfirmed = "marker_confirmed"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRestoreDispatchHeld = errors.New("restore marker dispatch is held")
|
||||
ErrRestorePermitSpent = errors.New("restore send permit already used")
|
||||
ErrRestorePermitMismatch = errors.New("restore send permit bound to another effect")
|
||||
ErrRestoreDispatchUnavailable = errors.New("restore marker dispatch protocol is not activated for this target")
|
||||
)
|
||||
|
||||
// RestoreMarkerKey 是"同一个逻辑恢复 effect"的身份。epoch、恢复队列 ID、恢复作业 ID
|
||||
// 都不在 key 里:接管与重试必须落到同一行,不能靠换 key 绕开旧的 dispatching。
|
||||
type RestoreMarkerKey struct {
|
||||
TargetIdentity string `json:"target_identity"`
|
||||
OperationID string `json:"operation_id"`
|
||||
WindowJobID string `json:"window_job_id"`
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
Index string `json:"index"`
|
||||
DocumentID string `json:"document_id"`
|
||||
Phase string `json:"phase"`
|
||||
}
|
||||
|
||||
func (k RestoreMarkerKey) Validate() error {
|
||||
for name, value := range map[string]string{
|
||||
"target": k.TargetIdentity, "operation": k.OperationID, "window job": k.WindowJobID,
|
||||
"plan": k.PlanDigest, "index": k.Index, "document": k.DocumentID, "phase": k.Phase,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("restore marker key lacks %s", name)
|
||||
}
|
||||
}
|
||||
if k.Phase != RestoreMarkerPhaseCreate {
|
||||
return fmt.Errorf("restore marker key phase %q is not the single create phase", k.Phase)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreMarkerBinding 是派发许可的不可变绑定。目标、快照、计划、变更体与标记体任一
|
||||
// 变化都必须 held,不能另起一行继续写。
|
||||
type RestoreMarkerBinding struct {
|
||||
Key RestoreMarkerKey `json:"key"`
|
||||
Scope string `json:"scope"`
|
||||
SourceSnapshot string `json:"source_snapshot"`
|
||||
BackupSnapshot string `json:"backup_snapshot"`
|
||||
ChangeDigest string `json:"change_digest"`
|
||||
MarkerDigest string `json:"marker_digest"`
|
||||
RequestDigest string `json:"request_digest"`
|
||||
Owner string `json:"owner"`
|
||||
Epoch int64 `json:"epoch"`
|
||||
}
|
||||
|
||||
func (b RestoreMarkerBinding) Validate() error {
|
||||
if err := b.Key.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
for name, value := range map[string]string{
|
||||
"scope": b.Scope, "source snapshot": b.SourceSnapshot, "backup snapshot": b.BackupSnapshot,
|
||||
"change digest": b.ChangeDigest, "marker digest": b.MarkerDigest,
|
||||
"request digest": b.RequestDigest, "owner": b.Owner,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("restore marker binding lacks %s", name)
|
||||
}
|
||||
}
|
||||
if b.Epoch <= 0 {
|
||||
return errors.New("restore marker binding lacks a positive attempt epoch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Digest 覆盖全部不可变字段;台账用它拒绝"同 key 换绑定"。
|
||||
func (b RestoreMarkerBinding) Digest() (string, error) {
|
||||
if err := b.Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
joined := strings.Join([]string{
|
||||
"v1", b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase, b.Scope, b.SourceSnapshot, b.BackupSnapshot,
|
||||
b.ChangeDigest, b.MarkerDigest, b.RequestDigest,
|
||||
}, "\x00")
|
||||
sum := sha256.Sum256([]byte(joined))
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// RestoreDispatchStatus 是纯读分类结果。Held 为真时永远不能发送。
|
||||
type RestoreDispatchStatus struct {
|
||||
State string `json:"state"`
|
||||
Held bool `json:"held"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// RestoreSendPermit 是"这一次调用、这一个 effect、这一次发送"的许可。
|
||||
//
|
||||
// 它不是可复用凭证:Store 只在 dispatching 明确 commit 之后返回它;消费一次即作废。
|
||||
// 不能从 Load/Get/JSON/重启重建——那正是本协议要排除的"从外观推断未派发"。
|
||||
type RestoreSendPermit struct {
|
||||
key RestoreMarkerKey
|
||||
binding string
|
||||
nonce string
|
||||
used bool
|
||||
}
|
||||
|
||||
// IssueRestoreSendPermit 只允许实现 RestoreMarkerGate 的存储层调用。许可的权威性来自
|
||||
// 数据库 CAS 与提交,不来自这个构造函数;这里只保证调用栈内一次性。
|
||||
func IssueRestoreSendPermit(key RestoreMarkerKey, bindingDigest, nonce string) *RestoreSendPermit {
|
||||
return &RestoreSendPermit{key: key, binding: bindingDigest, nonce: nonce}
|
||||
}
|
||||
|
||||
func (p *RestoreSendPermit) Nonce() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return p.nonce
|
||||
}
|
||||
|
||||
// Consume 校验 effect/绑定并作废许可。绑定不符时不消耗许可(避免被一次误用烧掉)。
|
||||
func (p *RestoreSendPermit) Consume(key RestoreMarkerKey, bindingDigest string) error {
|
||||
if p == nil {
|
||||
return ErrRestorePermitMismatch
|
||||
}
|
||||
if p.used {
|
||||
return ErrRestorePermitSpent
|
||||
}
|
||||
if p.key != key || p.binding != bindingDigest {
|
||||
return ErrRestorePermitMismatch
|
||||
}
|
||||
p.used = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreMarkerGate 是唯一能授予恢复标记 create 发送权的接口。
|
||||
//
|
||||
// Inspect 纯读、不返回许可;Prepare 只建立 prepared;Acquire 在独立短事务里做
|
||||
// prepared→dispatching 的 CAS,只有明确提交后才返回许可;Confirm 只记录真实回执。
|
||||
type RestoreMarkerGate interface {
|
||||
InspectRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (RestoreDispatchStatus, bool, error)
|
||||
PrepareRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (RestoreDispatchStatus, error)
|
||||
AcquireRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (*RestoreSendPermit, RestoreDispatchStatus, error)
|
||||
ConfirmRestoreMarker(ctx context.Context, b RestoreMarkerBinding, receipt []byte) error
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func restoreBinding() RestoreMarkerBinding {
|
||||
return RestoreMarkerBinding{
|
||||
Key: RestoreMarkerKey{TargetIdentity: "target-a", OperationID: "op-1", WindowJobID: "job-1",
|
||||
PlanDigest: "plan-1", Index: "kg-index", DocumentID: "doc-1", Phase: RestoreMarkerPhaseCreate},
|
||||
Scope: "internal", SourceSnapshot: "snap-1", BackupSnapshot: "snap-1",
|
||||
ChangeDigest: "change-1", MarkerDigest: "marker-1", RequestDigest: "request-1",
|
||||
Owner: "worker-a#queue#1", Epoch: 3,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreMarkerBindingRejectsIncompleteIdentity(t *testing.T) {
|
||||
for name, mutate := range map[string]func(*RestoreMarkerBinding){
|
||||
"target": func(b *RestoreMarkerBinding) { b.Key.TargetIdentity = "" },
|
||||
"operation": func(b *RestoreMarkerBinding) { b.Key.OperationID = "" },
|
||||
"window": func(b *RestoreMarkerBinding) { b.Key.WindowJobID = "" },
|
||||
"plan": func(b *RestoreMarkerBinding) { b.Key.PlanDigest = "" },
|
||||
"index": func(b *RestoreMarkerBinding) { b.Key.Index = "" },
|
||||
"document": func(b *RestoreMarkerBinding) { b.Key.DocumentID = "" },
|
||||
"change": func(b *RestoreMarkerBinding) { b.ChangeDigest = "" },
|
||||
"marker": func(b *RestoreMarkerBinding) { b.MarkerDigest = "" },
|
||||
"request": func(b *RestoreMarkerBinding) { b.RequestDigest = "" },
|
||||
"owner": func(b *RestoreMarkerBinding) { b.Owner = "" },
|
||||
"epoch": func(b *RestoreMarkerBinding) { b.Epoch = 0 },
|
||||
"phase": func(b *RestoreMarkerBinding) { b.Key.Phase = "apply-create" },
|
||||
} {
|
||||
binding := restoreBinding()
|
||||
mutate(&binding)
|
||||
if err := binding.Validate(); err == nil {
|
||||
t.Fatalf("%s: incomplete binding accepted", name)
|
||||
}
|
||||
}
|
||||
if err := restoreBinding().Validate(); err != nil {
|
||||
t.Fatalf("complete binding rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreMarkerBindingDigestCoversEveryBoundValue(t *testing.T) {
|
||||
base, err := restoreBinding().Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := restoreBinding().Digest(); again != base {
|
||||
t.Fatal("binding digest is not deterministic")
|
||||
}
|
||||
for name, mutate := range map[string]func(*RestoreMarkerBinding){
|
||||
"target": func(b *RestoreMarkerBinding) { b.Key.TargetIdentity = "target-b" },
|
||||
"index": func(b *RestoreMarkerBinding) { b.Key.Index = "other-index" },
|
||||
"doc": func(b *RestoreMarkerBinding) { b.Key.DocumentID = "doc-2" },
|
||||
"plan": func(b *RestoreMarkerBinding) { b.Key.PlanDigest = "plan-2" },
|
||||
"backup": func(b *RestoreMarkerBinding) { b.BackupSnapshot = "snap-2" },
|
||||
"change": func(b *RestoreMarkerBinding) { b.ChangeDigest = "change-2" },
|
||||
"marker": func(b *RestoreMarkerBinding) { b.MarkerDigest = "marker-2" },
|
||||
"request": func(b *RestoreMarkerBinding) { b.RequestDigest = "request-2" },
|
||||
"scope": func(b *RestoreMarkerBinding) { b.Scope = "external" },
|
||||
"source": func(b *RestoreMarkerBinding) { b.SourceSnapshot = "snap-9" },
|
||||
} {
|
||||
binding := restoreBinding()
|
||||
mutate(&binding)
|
||||
changed, err := binding.Digest()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if changed == base {
|
||||
t.Fatalf("%s: binding change not covered by digest", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 许可只能在本次调用栈用一次;换 key/换绑定都不能复用,也不能把已花的许可再花一次。
|
||||
func TestRestoreSendPermitIsSingleUseAndKeyBound(t *testing.T) {
|
||||
binding := restoreBinding()
|
||||
digest, err := binding.Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
permit := IssueRestoreSendPermit(binding.Key, digest, "nonce-1")
|
||||
other := binding.Key
|
||||
other.DocumentID = "doc-2"
|
||||
if err := permit.Consume(other, digest); err == nil || !errors.Is(err, ErrRestorePermitMismatch) {
|
||||
t.Fatalf("permit usable for another effect: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, "other-binding"); err == nil || !errors.Is(err, ErrRestorePermitMismatch) {
|
||||
t.Fatalf("permit usable for another binding: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, digest); err != nil {
|
||||
t.Fatalf("valid permit rejected: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, digest); err == nil || !errors.Is(err, ErrRestorePermitSpent) {
|
||||
t.Fatalf("permit spent twice: %v", err)
|
||||
}
|
||||
if strings.Contains(permit.Nonce(), digest) {
|
||||
t.Fatal("nonce is not independent audit data")
|
||||
}
|
||||
var nilPermit *RestoreSendPermit
|
||||
if err := nilPermit.Consume(binding.Key, digest); err == nil {
|
||||
t.Fatal("nil permit accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// DS01:恢复标记派发的持久准入。
|
||||
//
|
||||
// 关键点不是"多了一张表",而是发送许可的产生方式:
|
||||
// - Prepare 只建立 prepared(不返回许可);
|
||||
// - Acquire 在**独立短事务**里按 job→operation→active attempt→effect 的顺序取锁,
|
||||
// 校验当前 epoch/owner/lease 后做 prepared→dispatching 的 CAS,只有 COMMIT 明确
|
||||
// 成功才返回一次性许可;
|
||||
// - Inspect 纯读,永不返回许可;
|
||||
// - 表本身拒绝状态回退、绑定变化与删行(迁移里的触发器)。
|
||||
//
|
||||
// 外层 WithLibraryExecution 的事务只 rollback、从不 commit,所以这里绝不能用它:
|
||||
// 在那种事务里写 dispatching,发出请求后账会被回滚成"未派发"。
|
||||
|
||||
var ErrRestoreDispatchNotActivated = libraryrelease.ErrRestoreDispatchUnavailable
|
||||
|
||||
// ActivateRestoreDispatch 登记"该目标已完成旧 writer 退出/排空"的部署证据。
|
||||
//
|
||||
// 它记录的是运维声明,不验证声明为真;代码只能保证没有这条登记就不发放许可。
|
||||
func (s *Store) ActivateRestoreDispatch(ctx context.Context, targetIdentity, activatedBy, drainEvidenceDigest string, now time.Time) error {
|
||||
if targetIdentity == "" || activatedBy == "" || drainEvidenceDigest == "" {
|
||||
return errors.New("restore dispatch activation requires target, actor and drain evidence digest")
|
||||
}
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO library_index_restore_dispatch_activation (target_identity, protocol_version, activated_by, drain_evidence_digest, activated_at)
|
||||
VALUES ($1, 1, $2, $3, $4)
|
||||
ON CONFLICT (target_identity) DO UPDATE
|
||||
SET revoked_at = NULL
|
||||
WHERE library_index_restore_dispatch_activation.revoked_at IS NOT NULL`,
|
||||
targetIdentity, activatedBy, drainEvidenceDigest, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("activate restore dispatch: %w", err)
|
||||
}
|
||||
return s.requireRestoreDispatchActivation(ctx, targetIdentity)
|
||||
}
|
||||
|
||||
func (s *Store) RevokeRestoreDispatch(ctx context.Context, targetIdentity string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch_activation SET revoked_at=$2
|
||||
WHERE target_identity=$1 AND revoked_at IS NULL`, targetIdentity, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("revoke restore dispatch: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("restore dispatch activation is not active for this target")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) requireRestoreDispatchActivation(ctx context.Context, targetIdentity string) error {
|
||||
var one int
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT 1 FROM library_index_restore_dispatch_activation
|
||||
WHERE target_identity=$1 AND protocol_version=1 AND revoked_at IS NULL`, targetIdentity).Scan(&one)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("%w(目标 %s 没有激活登记)", ErrRestoreDispatchNotActivated, targetIdentity)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read restore dispatch activation: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreDispatchBindingDigest(b libraryrelease.RestoreMarkerBinding) (string, error) {
|
||||
return b.Digest()
|
||||
}
|
||||
|
||||
func (s *Store) InspectRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, err
|
||||
}
|
||||
var state, stored string
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT state, binding_digest FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&state, &stored)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
|
||||
}
|
||||
if stored != digest {
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
|
||||
}
|
||||
switch state {
|
||||
case libraryrelease.RestoreDispatchPrepared:
|
||||
return libraryrelease.RestoreDispatchStatus{State: state}, true, nil
|
||||
case libraryrelease.RestoreDispatchDispatching:
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "dispatch outcome unknown: no further create is permitted"}, true, nil
|
||||
case libraryrelease.RestoreDispatchMarkerConfirmed:
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "marker already dispatched; only the original confirmation path may continue"}, true, nil
|
||||
default:
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "unrecognised dispatch state"}, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// PrepareRestoreMarker 只建立 prepared:没有发送权,也不把已有 dispatching 抬回 prepared。
|
||||
func (s *Store) PrepareRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, error) {
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("prepare restore dispatch: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
// 与 BeginLibraryAttempt 同序:先作业行、再 operation 行。这样"准备"不会与代次切换
|
||||
// 交错成一份属于旧 identity 的台账。
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO library_index_restore_dispatch
|
||||
(target_identity, operation_id, window_job_id, plan_digest, index_name, document_id, phase, binding_digest, state)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'prepared')
|
||||
ON CONFLICT ON CONSTRAINT library_index_restore_dispatch_identity DO NOTHING`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase, digest); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("record prepared restore dispatch: %w", err)
|
||||
}
|
||||
status, found, err := inspectRestoreDispatchTx(ctx, tx, b)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if !found {
|
||||
return libraryrelease.RestoreDispatchStatus{}, errors.New("prepared restore dispatch was not durable")
|
||||
}
|
||||
if status.Held {
|
||||
return status, fmt.Errorf("%w: %s", libraryrelease.ErrRestoreDispatchHeld, status.Reason)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("commit prepared restore dispatch: %w", err)
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
// acquireRestoreMarker 的 beforeCommit 是提交结果不明这一档的测试接缝:钩子返回错误时
|
||||
// 事务回滚,调用方不得发送。生产路径传 nil。
|
||||
func (s *Store) acquireRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding, beforeCommit func() error) (*libraryrelease.RestoreSendPermit, libraryrelease.RestoreDispatchStatus, error) {
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("dispatch restore marker: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
// 当前代次必须在租约内,且就是许可申请者:旧 owner 晚到时租约/epoch 已变,零发送。
|
||||
var epoch int64
|
||||
var owner, attemptState string
|
||||
var leaseUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT epoch, owner, state, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(
|
||||
&epoch, &owner, &attemptState, &leaseUntil)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("%w: no active attempt for operation %s",
|
||||
governance.ErrLibraryEpochStale, b.Key.OperationID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read active attempt: %w", err)
|
||||
}
|
||||
var leaseLive bool
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("read attempt lease: %w", err)
|
||||
}
|
||||
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchPrepared, Held: true,
|
||||
Reason: fmt.Sprintf("attempt %s#%d is not the live authorized owner", b.Owner, b.Epoch)},
|
||||
fmt.Errorf("%w: attempt %s#%d is not current", governance.ErrLibraryEpochStale, b.Owner, b.Epoch)
|
||||
}
|
||||
|
||||
var id int64
|
||||
var state, stored string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT id, state, binding_digest FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7 FOR UPDATE`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&id, &state, &stored)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, errors.New("no prepared restore dispatch record: refusing to send")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("lock restore dispatch: %w", err)
|
||||
}
|
||||
if stored != digest {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "restore dispatch binding differs from the immutable plan"}, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "dispatch already claimed; no second send is permitted"}, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
|
||||
var raw [16]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, err
|
||||
}
|
||||
nonce := hex.EncodeToString(raw[:])
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='dispatching', dispatch_owner=$2, dispatch_epoch=$3, dispatch_nonce=$4,
|
||||
request_digest=$5, dispatch_committed_at=now(), updated_at=now()
|
||||
WHERE id=$1 AND state='prepared'`, id, b.Owner, b.Epoch, nonce, b.RequestDigest)
|
||||
if err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{}, fmt.Errorf("claim restore dispatch: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "dispatch CAS lost"}, libraryrelease.ErrRestoreDispatchHeld
|
||||
}
|
||||
if beforeCommit != nil {
|
||||
if err := beforeCommit(); err != nil {
|
||||
// 提交结果不明:本调用不发送,也不把状态当成 prepared 重试。
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
|
||||
Reason: "commit outcome unknown"}, fmt.Errorf("commit restore dispatch: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching, Held: true,
|
||||
Reason: "commit outcome unknown"}, fmt.Errorf("commit restore dispatch: %w", err)
|
||||
}
|
||||
return libraryrelease.IssueRestoreSendPermit(b.Key, digest, nonce),
|
||||
libraryrelease.RestoreDispatchStatus{State: libraryrelease.RestoreDispatchDispatching}, nil
|
||||
}
|
||||
|
||||
func (s *Store) AcquireRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding) (*libraryrelease.RestoreSendPermit, libraryrelease.RestoreDispatchStatus, error) {
|
||||
return s.acquireRestoreMarker(ctx, b, nil)
|
||||
}
|
||||
|
||||
// ConfirmRestoreMarker 只记录真实回执(不再发送)。状态仍是 dispatching 时才允许;
|
||||
// 回执本身来自本次受控发送或同计划标记的实时核实,不由客户端布尔值决定。
|
||||
func (s *Store) ConfirmRestoreMarker(ctx context.Context, b libraryrelease.RestoreMarkerBinding, receipt []byte) error {
|
||||
if len(receipt) == 0 || !json.Valid(receipt) {
|
||||
return errors.New("restore marker confirmation requires a real receipt")
|
||||
}
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.requireRestoreDispatchActivation(ctx, b.Key.TargetIdentity); err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm restore marker: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := lockRestoreDispatchIdentity(ctx, tx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
var epoch int64
|
||||
var owner string
|
||||
var leaseUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT epoch, owner, lease_until FROM library_attempts
|
||||
WHERE operation_id=$1 AND state IN ('claimed','running') FOR UPDATE`, b.Key.OperationID).Scan(&epoch, &owner, &leaseUntil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm restore marker attempt: %w", err)
|
||||
}
|
||||
var leaseLive bool
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE($1::timestamptz > now(), false)`, leaseUntil).Scan(&leaseLive); err != nil {
|
||||
return fmt.Errorf("confirm restore marker lease: %w", err)
|
||||
}
|
||||
if owner != b.Owner || epoch != b.Epoch || !leaseLive {
|
||||
return fmt.Errorf("%w: confirmation by a stale owner", governance.ErrLibraryEpochStale)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_index_restore_dispatch
|
||||
SET state='marker_confirmed', marker_receipt=$1::jsonb, confirmed_at=now(), updated_at=now()
|
||||
WHERE target_identity=$2 AND operation_id=$3 AND window_job_id=$4 AND plan_digest=$5
|
||||
AND index_name=$6 AND document_id=$7 AND phase=$8
|
||||
AND state='dispatching' AND binding_digest=$9`,
|
||||
string(receipt), b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase, digest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm restore marker: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("restore marker confirmation did not match a dispatching record")
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit restore marker confirmation: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockRestoreDispatchIdentity(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) error {
|
||||
var operationID string
|
||||
err := tx.QueryRow(ctx, `SELECT operation_id FROM library_jobs WHERE id=$1 FOR UPDATE`, b.Key.WindowJobID).Scan(&operationID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("window job %s not found", b.Key.WindowJobID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("lock window job: %w", err)
|
||||
}
|
||||
if operationID != b.Key.OperationID {
|
||||
return errors.New("window job belongs to another operation")
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT id FROM library_operations WHERE id=$1 FOR UPDATE`, b.Key.OperationID); err != nil {
|
||||
return fmt.Errorf("lock operation: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inspectRestoreDispatchTx(ctx context.Context, tx pgx.Tx, b libraryrelease.RestoreMarkerBinding) (libraryrelease.RestoreDispatchStatus, bool, error) {
|
||||
digest, err := restoreDispatchBindingDigest(b)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, err
|
||||
}
|
||||
var state, stored string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT state, binding_digest FROM library_index_restore_dispatch
|
||||
WHERE target_identity=$1 AND operation_id=$2 AND window_job_id=$3 AND plan_digest=$4
|
||||
AND index_name=$5 AND document_id=$6 AND phase=$7`,
|
||||
b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase).Scan(&state, &stored)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreDispatchStatus{}, false, fmt.Errorf("inspect restore dispatch: %w", err)
|
||||
}
|
||||
if stored != digest {
|
||||
return libraryrelease.RestoreDispatchStatus{State: state, Held: true,
|
||||
Reason: "restore dispatch binding differs from the immutable plan"}, true, nil
|
||||
}
|
||||
return libraryrelease.RestoreDispatchStatus{State: state,
|
||||
Held: state != libraryrelease.RestoreDispatchPrepared}, true, nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// DS01 的许可竞争必须是真库行为:内存 fake 无法证明"只有唯一赢家拿到许可",
|
||||
// 也无法证明触发器拒绝回退/删行。
|
||||
|
||||
func seededOperationID(t *testing.T, pool *pgxpool.Pool, jobID string) string {
|
||||
t.Helper()
|
||||
var operationID string
|
||||
if err := pool.QueryRow(context.Background(), `SELECT operation_id FROM library_jobs WHERE id=$1`, jobID).Scan(&operationID); err != nil {
|
||||
t.Fatalf("read seeded operation: %v", err)
|
||||
}
|
||||
return operationID
|
||||
}
|
||||
|
||||
func ds01Binding() libraryrelease.RestoreMarkerBinding {
|
||||
return libraryrelease.RestoreMarkerBinding{
|
||||
Key: libraryrelease.RestoreMarkerKey{
|
||||
TargetIdentity: "target-ds01", OperationID: "op", WindowJobID: "job",
|
||||
PlanDigest: "plan-1", Index: "kg-index", DocumentID: "doc-1",
|
||||
Phase: libraryrelease.RestoreMarkerPhaseCreate,
|
||||
},
|
||||
Scope: "internal", SourceSnapshot: "snap-src", BackupSnapshot: "snap-backup",
|
||||
ChangeDigest: "change-1", MarkerDigest: "marker-1", RequestDigest: "request-1",
|
||||
Owner: "worker-a#q1#0", Epoch: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchRequiresActivation(t *testing.T) {
|
||||
store, _ := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
binding := ds01Binding()
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); !errors.Is(err, libraryrelease.ErrRestoreDispatchUnavailable) {
|
||||
t.Fatalf("prepare without activation = %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := store.db.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("prepared rows without activation = %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchPrepareIsUniqueAndBindingBound(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-prep", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-prep", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-prep"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, "libsync-ds01-prep")
|
||||
for i := 0; i < 2; i++ {
|
||||
status, err := store.PrepareRestoreMarker(ctx, binding)
|
||||
if err != nil || status.State != libraryrelease.RestoreDispatchPrepared || status.Held {
|
||||
t.Fatalf("prepare #%d = %+v %v", i, status, err)
|
||||
}
|
||||
}
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM library_index_restore_dispatch`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("prepared rows = %d", count)
|
||||
}
|
||||
other := binding
|
||||
other.MarkerDigest = "marker-2"
|
||||
if _, err := store.PrepareRestoreMarker(ctx, other); err == nil {
|
||||
t.Fatal("binding change reused the prepared row")
|
||||
}
|
||||
status, found, err := store.InspectRestoreMarker(ctx, other)
|
||||
if err != nil || !found || !status.Held {
|
||||
t.Fatalf("binding conflict inspect = %+v found=%v err=%v", status, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchConcurrentAcquireHasSingleWinner(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-race", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-race", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-race"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, "libsync-ds01-race")
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
permits := make(chan *libraryrelease.RestoreSendPermit, 2)
|
||||
errs := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
permits <- permit
|
||||
errs <- err
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(permits)
|
||||
close(errs)
|
||||
winners := 0
|
||||
for permit := range permits {
|
||||
if permit != nil {
|
||||
winners++
|
||||
}
|
||||
}
|
||||
for err := range errs {
|
||||
if err != nil && !errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Fatalf("loser error = %v", err)
|
||||
}
|
||||
}
|
||||
if winners != 1 {
|
||||
t.Fatalf("dispatch winners = %d", winners)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchDispatching {
|
||||
t.Fatalf("state = %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchIsMonotoneAndAppendOnly(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-mono", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-mono", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-mono"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, "libsync-ds01-mono")
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, binding)
|
||||
if err != nil || permit == nil {
|
||||
t.Fatalf("acquire = %v", err)
|
||||
}
|
||||
if permit, _, err = store.AcquireRestoreMarker(ctx, binding); permit != nil || !errors.Is(err, libraryrelease.ErrRestoreDispatchHeld) {
|
||||
t.Fatalf("second acquire permit=%v err=%v", permit, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_index_restore_dispatch SET state='prepared'`); err == nil ||
|
||||
!strings.Contains(err.Error(), "not monotone") {
|
||||
t.Fatalf("state regression accepted: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM library_index_restore_dispatch`); err == nil ||
|
||||
!strings.Contains(err.Error(), "append-only") {
|
||||
t.Fatalf("evidence delete accepted: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_index_restore_dispatch SET document_id='doc-2'`); err == nil ||
|
||||
!strings.Contains(err.Error(), "immutable") {
|
||||
t.Fatalf("binding rewrite accepted: %v", err)
|
||||
}
|
||||
var state, owner string
|
||||
if err := pool.QueryRow(ctx, `SELECT state, dispatch_owner FROM library_index_restore_dispatch`).Scan(&state, &owner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchDispatching || owner != "worker-a#q1#0" {
|
||||
t.Fatalf("row after rejected rewrites = %s/%s", state, owner)
|
||||
}
|
||||
// down 迁移不能把这些行当"没有证据"抹掉。
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, migrations[len(migrations)-1].DownSQL); err == nil ||
|
||||
!strings.Contains(err.Error(), "refusing to drop restore dispatch evidence") {
|
||||
t.Fatalf("dangerous down accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchRejectsStaleEpochBeforeSend(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-epoch", "sync", now)
|
||||
first, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-epoch", "worker-a#q1#0", time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-epoch"
|
||||
binding.Key.OperationID = first.OperationID
|
||||
binding.Owner, binding.Epoch = first.Owner, int64(first.Epoch)
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A 还没赢 CAS,租约过期,B 接管成新 epoch。
|
||||
second, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-epoch", "worker-b#q2#0", 10*time.Minute, now.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("takeover: %v", err)
|
||||
}
|
||||
if second.Epoch <= first.Epoch {
|
||||
t.Fatalf("epoch did not advance: %d -> %d", first.Epoch, second.Epoch)
|
||||
}
|
||||
if permit, _, err := store.AcquireRestoreMarker(ctx, binding); permit != nil || !errors.Is(err, governance.ErrLibraryEpochStale) {
|
||||
t.Fatalf("stale owner acquired permit=%v err=%v", permit, err)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
t.Fatalf("stale owner moved state to %s", state)
|
||||
}
|
||||
current := binding
|
||||
current.Owner, current.Epoch = second.Owner, int64(second.Epoch)
|
||||
permit, _, err := store.AcquireRestoreMarker(ctx, current)
|
||||
if err != nil || permit == nil {
|
||||
t.Fatalf("current owner acquire = %v", err)
|
||||
}
|
||||
if permit, _, err = store.AcquireRestoreMarker(ctx, current); permit != nil || err == nil {
|
||||
t.Fatalf("current owner acquired twice: permit=%v err=%v", permit, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchCommitWithoutExplicitSuccessYieldsNoPermit(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
seedOperationJob(t, store, pool, "internal", "libsync-ds01-commit", "sync", now)
|
||||
if _, err := store.BeginLibraryAttempt(ctx, "libsync-ds01-commit", "worker-a#q1#0", 10*time.Minute, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, "target-ds01", "deploy-bot", "sha256:drain", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding := ds01Binding()
|
||||
binding.Key.WindowJobID = "libsync-ds01-commit"
|
||||
binding.Key.OperationID = seededOperationID(t, pool, "libsync-ds01-commit")
|
||||
if _, err := store.PrepareRestoreMarker(ctx, binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
permit, status, err := store.acquireRestoreMarker(ctx, binding, func() error {
|
||||
return errors.New("commit acknowledgement lost")
|
||||
})
|
||||
if permit != nil || err == nil || !status.Held {
|
||||
t.Fatalf("commit-unknown returned permit=%v status=%+v err=%v", permit, status, err)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != libraryrelease.RestoreDispatchPrepared {
|
||||
t.Fatalf("uncommitted CAS leaked into durable state: %s", state)
|
||||
}
|
||||
// 回滚干净之后,下一调用仍可竞争同一条 prepared(活性下降,安全性不变)。
|
||||
permit, _, err = store.AcquireRestoreMarker(ctx, binding)
|
||||
if err != nil || permit == nil {
|
||||
t.Fatalf("post-rollback acquire = %v", err)
|
||||
}
|
||||
receipt := []byte(`{"_index":"kg-index","_id":"doc-1","result":"created","_seq_no":7,"_primary_term":1}`)
|
||||
if err := store.ConfirmRestoreMarker(ctx, binding, receipt); err != nil {
|
||||
t.Fatalf("confirm: %v", err)
|
||||
}
|
||||
var confirmed string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&confirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if confirmed != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
t.Fatalf("confirmed state = %s", confirmed)
|
||||
}
|
||||
if err := store.ConfirmRestoreMarker(ctx, binding, []byte(`{"partial":true}`)); err == nil {
|
||||
t.Fatal("second confirmation accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
-- 已有派发记录时拒绝自动回滚:这些行是"是否发送过"的唯一证明,
|
||||
-- 用 down 抹掉等于把 unknown 重新伪装成 prepared。人工处置必须先导出并存档。
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM library_index_restore_dispatch LIMIT 1) THEN
|
||||
RAISE EXCEPTION 'refusing to drop restore dispatch evidence with recorded rows';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_monotone ON library_index_restore_dispatch;
|
||||
DROP FUNCTION IF EXISTS library_index_restore_dispatch_guard();
|
||||
DROP TABLE IF EXISTS library_index_restore_dispatch;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_activation_guard_trg ON library_index_restore_dispatch_activation;
|
||||
DROP FUNCTION IF EXISTS library_index_restore_dispatch_activation_guard();
|
||||
DROP TABLE IF EXISTS library_index_restore_dispatch_activation;
|
||||
@@ -0,0 +1,98 @@
|
||||
-- DS01:恢复标记派发的唯一持久台账。
|
||||
--
|
||||
-- 它记录的是"这个固定 effect 是否已经有人取得过向索引发送一次 `_create` 的许可",
|
||||
-- 不是"ES 历史上没写过这个 ID"。判据只有单调状态:prepared -> dispatching ->
|
||||
-- marker_confirmed。行不可删、绑定不可改、状态不可回退;删除型恢复的新能力只在
|
||||
-- 这张表里 prepared 的区段可跨进程续作。
|
||||
CREATE TABLE IF NOT EXISTS library_index_restore_dispatch (
|
||||
id bigserial PRIMARY KEY,
|
||||
protocol_version integer NOT NULL DEFAULT 1 CHECK (protocol_version = 1),
|
||||
target_identity text NOT NULL,
|
||||
operation_id text NOT NULL,
|
||||
window_job_id text NOT NULL,
|
||||
plan_digest text NOT NULL,
|
||||
index_name text NOT NULL,
|
||||
document_id text NOT NULL,
|
||||
phase text NOT NULL CHECK (phase = 'restore-marker-create'),
|
||||
binding_digest text NOT NULL CHECK (binding_digest <> ''),
|
||||
state text NOT NULL CHECK (state IN ('prepared','dispatching','marker_confirmed')),
|
||||
failure_reason text NOT NULL DEFAULT '',
|
||||
dispatch_owner text NOT NULL DEFAULT '',
|
||||
dispatch_epoch integer NOT NULL DEFAULT 0,
|
||||
dispatch_nonce text NOT NULL DEFAULT '',
|
||||
request_digest text NOT NULL DEFAULT '',
|
||||
marker_receipt jsonb,
|
||||
prepared_at timestamptz NOT NULL DEFAULT now(),
|
||||
dispatch_committed_at timestamptz,
|
||||
confirmed_at timestamptz,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT library_index_restore_dispatch_identity UNIQUE
|
||||
(target_identity, operation_id, window_job_id, plan_digest, index_name, document_id, phase),
|
||||
-- dispatching 及之后必须留着当时的派发身份:没有它就不能证明许可属于谁。
|
||||
CONSTRAINT library_index_restore_dispatch_dispatch_proof CHECK (
|
||||
state = 'prepared' OR (
|
||||
dispatch_owner <> '' AND dispatch_epoch > 0 AND dispatch_nonce <> '' AND
|
||||
request_digest <> '' AND dispatch_committed_at IS NOT NULL
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION library_index_restore_dispatch_guard() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' THEN
|
||||
RAISE EXCEPTION 'restore dispatch evidence is append-only';
|
||||
END IF;
|
||||
IF NEW.target_identity <> OLD.target_identity OR NEW.operation_id <> OLD.operation_id OR
|
||||
NEW.window_job_id <> OLD.window_job_id OR NEW.plan_digest <> OLD.plan_digest OR
|
||||
NEW.index_name <> OLD.index_name OR NEW.document_id <> OLD.document_id OR
|
||||
NEW.phase <> OLD.phase OR NEW.binding_digest <> OLD.binding_digest OR
|
||||
NEW.protocol_version <> OLD.protocol_version THEN
|
||||
RAISE EXCEPTION 'restore dispatch binding is immutable';
|
||||
END IF;
|
||||
IF NEW.state = OLD.state THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
IF OLD.state = 'prepared' AND NEW.state = 'dispatching' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
IF OLD.state = 'dispatching' AND NEW.state = 'marker_confirmed' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION 'restore dispatch state % -> % is not monotone', OLD.state, NEW.state;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_monotone ON library_index_restore_dispatch;
|
||||
CREATE TRIGGER library_index_restore_dispatch_monotone
|
||||
BEFORE UPDATE OR DELETE ON library_index_restore_dispatch
|
||||
FOR EACH ROW EXECUTE FUNCTION library_index_restore_dispatch_guard();
|
||||
|
||||
-- 激活登记:配置开关不是独占证明。只有部署流程把"旧 writer 已退出并排空"的证据
|
||||
-- 登记到这个目标上,协议才对该目标生效;撤销后立即 fail closed。
|
||||
CREATE TABLE IF NOT EXISTS library_index_restore_dispatch_activation (
|
||||
target_identity text PRIMARY KEY,
|
||||
protocol_version integer NOT NULL CHECK (protocol_version = 1),
|
||||
activated_by text NOT NULL CHECK (activated_by <> ''),
|
||||
drain_evidence_digest text NOT NULL CHECK (drain_evidence_digest <> ''),
|
||||
activated_at timestamptz NOT NULL DEFAULT now(),
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION library_index_restore_dispatch_activation_guard() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF NEW.target_identity <> OLD.target_identity OR NEW.protocol_version <> OLD.protocol_version OR
|
||||
NEW.activated_by <> OLD.activated_by OR NEW.drain_evidence_digest <> OLD.drain_evidence_digest OR
|
||||
NEW.activated_at <> OLD.activated_at THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation is immutable except revocation';
|
||||
END IF;
|
||||
IF OLD.revoked_at IS NOT NULL AND NEW.revoked_at IS NOT NULL AND NEW.revoked_at <> OLD.revoked_at THEN
|
||||
RAISE EXCEPTION 'restore dispatch activation revocation is final';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS library_index_restore_dispatch_activation_guard_trg ON library_index_restore_dispatch_activation;
|
||||
CREATE TRIGGER library_index_restore_dispatch_activation_guard_trg
|
||||
BEFORE UPDATE ON library_index_restore_dispatch_activation
|
||||
FOR EACH ROW EXECUTE FUNCTION library_index_restore_dispatch_activation_guard();
|
||||
@@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 77 {
|
||||
t.Fatalf("migration count = %d, want 77", len(migrations))
|
||||
if len(migrations) != 78 {
|
||||
t.Fatalf("migration count = %d, want 78", len(migrations))
|
||||
}
|
||||
// 78 是 DS01 恢复标记派发台账:唯一 key、单调状态与激活登记;只动治理库。
|
||||
if dispatch := migrations[77]; dispatch.Version != 78 ||
|
||||
!strings.Contains(dispatch.UpSQL, "library_index_restore_dispatch") ||
|
||||
!strings.Contains(dispatch.UpSQL, "restore-marker-create") ||
|
||||
!strings.Contains(dispatch.DownSQL, "refusing to drop restore dispatch evidence") {
|
||||
t.Fatalf("migration 78 = %#v", dispatch)
|
||||
}
|
||||
if target := migrations[76]; target.Version != 77 || !strings.Contains(target.UpSQL, "PRIMARY KEY (scope, target_id)") {
|
||||
t.Fatalf("migration 77 = %#v", target)
|
||||
|
||||
Reference in New Issue
Block a user