feat(library): 删除型恢复按台账准入并唯一发送

为什么:恢复在发现 index held 之前可能已经回放 database/resources;
"最后数据相同"不等于"本次零派发"。改为 Restore 在任何写之前按整份
冻结计划做纯读准入,收尾前再拒绝未决 effect。

删除型恢复的 _create 只从唯一 dispatcher 发出:preflight/故障钩子
在 CAS 之前,许可来自 COMMIT 明确的短事务;发送请求 GetBody=nil,
传输层无法重放。旧意图无台账、dispatching 无精确 marker 一律 held。

验证:真 ES+真 PG 覆盖旧意图零写 held、prepared 续作到终态
(create 恰 1 次)、dispatching+missing 两轮零写、新 OS 进程续作正例。
This commit is contained in:
姚章浩
2026-11-02 16:27:53 +08:00
parent fad26166a4
commit e09fddcb74
7 changed files with 945 additions and 35 deletions
@@ -25,7 +25,7 @@ func TestR10ProductionAdmissionIsolated(t *testing.T) {
fx.Exec(c.sql)
}
before := fx.State()
target, backups, e := librarystack.OpenProductionStack(fx.Context, fx.Config, "internal", fx.Snapshot, "")
target, backups, e := librarystack.OpenProductionStack(fx.Context, fx.Config, "internal", fx.Snapshot, "", nil)
if target != nil {
target.Close()
}
@@ -0,0 +1,506 @@
package v071
import (
"context"
"encoding/json"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/postgres/pgtest"
)
// DS01:删除型恢复派发协议的 adapter 层验收。
//
// 前置是真 ES、真 PG;治理台账放在**独立的治理 schema**(不是业务目标 schema)。
// 这里不制造"prepared 且从未派发"的假正例:prepared 一律由真实 Prepare 产生。
const ds01TargetIdentity = "kg-ds01-test-target"
type ds01Authority struct {
store *postgresstore.Store
pool *pgxpool.Pool
dsn string
operationID string
jobID string
owner string
epoch int64
targetIdentity string
snapshot string
}
func newDS01Authority(t *testing.T, jobID, snapshot string) *ds01Authority {
t.Helper()
base := os.Getenv("KG_DS01_GOVERNANCE_DSN")
if base == "" {
t.Skip("requires an isolated governance database (KG_DS01_GOVERNANCE_DSN)")
}
dsn := pgtest.IsolatedDatabaseURL(t, base)
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect governance: %v", err)
}
t.Cleanup(pool.Close)
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("apply governance migrations: %v", err)
}
store := postgresstore.NewStore(pool)
now := time.Now().UTC()
op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
ID: "op-" + jobID, Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: snapshot,
RequestDigest: "digest-" + jobID, OperationKey: jobID, Actor: "ds01-test", CreatedAt: now,
})
if err != nil {
t.Fatalf("ensure operation: %v", err)
}
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
ID: jobID, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: op.ID,
IdempotencyKey: "library.job:" + op.ID, RequestDigest: "digest-" + jobID,
TargetSnapshotID: snapshot, Actor: "ds01-test",
}, governance.SyncStages(), now); err != nil {
t.Fatalf("create job: %v", err)
}
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker-ds01#queue#1", 10*time.Minute, now)
if err != nil {
t.Fatalf("begin attempt: %v", err)
}
if err := store.ActivateRestoreDispatch(ctx, ds01TargetIdentity, "ds01-test", "sha256:drain-evidence", now); err != nil {
t.Fatalf("activate dispatch: %v", err)
}
return &ds01Authority{store: store, pool: pool, dsn: dsn, operationID: attempt.OperationID,
jobID: attempt.JobID, owner: attempt.Owner, epoch: int64(attempt.Epoch),
targetIdentity: ds01TargetIdentity, snapshot: snapshot}
}
// attach 把真实治理台账与本次 attempt 身份接到目标上。
func (a *ds01Authority) attach(target *ProductionLibraryTarget) {
target.config.RestoreDispatch = a.store
target.config.RestoreTargetIdentity = a.targetIdentity
target.config.RestoreScope = "internal"
target.config.RestoreOperationID = a.operationID
target.config.RestoreWindowJobID = a.jobID
target.config.RestoreAttemptOwner = a.owner
target.config.RestoreAttemptEpoch = a.epoch
target.config.RestoreSnapshotID = a.snapshot
}
func (a *ds01Authority) dispatchRow(t *testing.T) (state, owner string) {
t.Helper()
if err := a.pool.QueryRow(context.Background(),
`SELECT state, dispatch_owner FROM library_index_restore_dispatch`).Scan(&state, &owner); err != nil {
t.Fatalf("read dispatch row: %v", err)
}
return state, owner
}
func (a *ds01Authority) dispatchRows(t *testing.T) int {
t.Helper()
var count int
if err := a.pool.QueryRow(context.Background(), `SELECT count(*) FROM library_index_restore_dispatch`).Scan(&count); err != nil {
t.Fatal(err)
}
return count
}
// ds01RunToBoundary 走真实应用→wiki 失败→自动 rollback:在删除型恢复标记的边界上
// 用控制流故障拒绝,留下 prepared + 意图 + 0 次转发。
func ds01RunToBoundary(t *testing.T, fx *writeSetFixture, snapshot string, target *ProductionLibraryTarget,
backups *ProductionBackupStore, wire *restoreMarkerWire) libraryrelease.Outcome {
t.Helper()
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
if c.ID != "chunk-target" {
return nil
}
return errDS01BoundaryRefusal
}
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
if outcome.State != "rollback_failed" || !strings.Contains(outcome.Rollback, errDS01BoundaryRefusal.Error()) {
t.Fatalf("rollback_failed with DS01 boundary refusal expected, got state=%q rollback=%q", outcome.State, outcome.Rollback)
}
if wire.dispatch != 0 {
t.Fatalf("boundary refusal dispatched %d marker writes", wire.dispatch)
}
return outcome
}
var errDS01BoundaryRefusal = &ds01BoundaryError{}
type ds01BoundaryError struct{}
func (*ds01BoundaryError) Error() string {
return "injected DS01 boundary refusal before dispatch CAS"
}
func ds01Fixture(t *testing.T) (*writeSetFixture, string, *ProductionLibraryTarget, *ProductionBackupStore, *restoreMarkerWire) {
t.Helper()
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
client := *target.search.client
inner := client.Transport
if inner == nil {
inner = http.DefaultTransport
}
wire := &restoreMarkerWire{inner: inner}
client.Transport = wire
target.search.client = &client
return fx, snapshot, target, backups, wire
}
// A7:旧 intent + missing、无台账 ⇒ 在任何 PG/resources 写之前 held。
func TestDS01LegacyIntentWithoutLedgerStaysHeldBeforeAnyWrite(t *testing.T) {
fx, snapshot, target, backups, wire := ds01Fixture(t)
// 先用旧协议(未接线)造出"既存意图、无台账"的现场。
legacy := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
ledger := legacy.Backup.WriteSet
change := indexChangeOf(t, ledger, "chunk-target")
requireIndexEffect(t, fx, ledger, change, "restore-mark-intent", true)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
if rows := authority.dispatchRows(t); rows != 0 {
t.Fatalf("legacy run created %d dispatch rows", rows)
}
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectVersionSet(t, fx)
beforeDispatch := wire.dispatch
authority.attach(target)
target.restoreMarkerFault = nil
err := target.Restore(fx.ctx, legacy.Backup)
if err == nil || !strings.Contains(err.Error(), "held") {
t.Fatalf("legacy intent without ledger must stay held, got %v", err)
}
if wire.dispatch != beforeDispatch {
t.Fatalf("held legacy restoration dispatched %d marker writes", wire.dispatch-beforeDispatch)
}
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("held legacy restoration wrote PG before admission: %s", diff)
}
if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(beforeIndex, after) {
t.Fatal("held legacy restoration changed the index")
}
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
t.Fatal("held legacy restoration changed objects")
}
if rows := authority.dispatchRows(t); rows != 0 {
t.Fatalf("legacy path wrote %d dispatch rows", rows)
}
}
// A9(同进程):prepared + 意图,边界拒绝 0 转发;新调用用同一 window/backup/plan
// 完成恢复,create 恰好转发一次,台账 marker_confirmed。
func TestDS01PreparedEffectResumesToTerminalState(t *testing.T) {
fx, snapshot, target, backups, wire := ds01Fixture(t)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
authority.attach(target)
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
if state, owner := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchPrepared || owner != "" {
t.Fatalf("after boundary refusal row = %s/%s", state, owner)
}
ledger := outcome.Backup.WriteSet
change := indexChangeOf(t, ledger, "chunk-target")
requireIndexEffect(t, fx, ledger, change, "restore-claimed", false)
target.restoreMarkerFault = nil
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("prepared effect did not resume: %v", err)
}
if wire.dispatch != 1 {
t.Fatalf("marker create forwarded %d times, want exactly 1", wire.dispatch)
}
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchMarkerConfirmed {
t.Fatalf("terminal dispatch state = %s", state)
}
requireIndexEffect(t, fx, ledger, change, "restored", true)
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("restoration verify: %v", err)
}
// 重放同一次恢复:不再产生新的 create。
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("replayed restoration: %v", err)
}
if wire.dispatch != 1 {
t.Fatalf("replayed restoration dispatched %d marker creates", wire.dispatch)
}
}
// A8:commit 已落地、Do 之前进程死掉 ⇒ dispatching+missing 永久 held,零新发送,
// PG/对象/索引不变;第二次仍 held。
func TestDS01DispatchingMissingStaysHeldWithZeroWrites(t *testing.T) {
fx, snapshot, target, backups, wire := ds01Fixture(t)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
authority.attach(target)
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
// 模拟"CAS 提交成功、发送前进程死掉":真实 Acquire 把行推进到 dispatching,但不发送。
ledger := outcome.Backup.WriteSet
if err := target.AdoptWriteSet(*ledger); err != nil {
t.Fatalf("adopt plan: %v", err)
}
change := indexChangeOf(t, ledger, "chunk-target")
body, err := target.restoreIndexClaimBody(change)
if err != nil {
t.Fatal(err)
}
binding, err := target.restoreMarkerBinding(change, body)
if err != nil {
t.Fatal(err)
}
permit, _, err := authority.store.AcquireRestoreMarker(fx.ctx, binding)
if err != nil || permit == nil {
t.Fatalf("simulate committed dispatch: %v", err)
}
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchDispatching {
t.Fatalf("simulated state = %s", state)
}
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectVersionSet(t, fx)
beforeDispatch := wire.dispatch
target.restoreMarkerFault = nil
for attempt := 0; attempt < 2; attempt++ {
err := target.Restore(fx.ctx, outcome.Backup)
if err == nil || !strings.Contains(err.Error(), "held") {
t.Fatalf("attempt %d: dispatching+missing must stay held, got %v", attempt, err)
}
}
if wire.dispatch != beforeDispatch {
t.Fatalf("dispatching+missing dispatched %d marker writes", wire.dispatch-beforeDispatch)
}
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("dispatching+missing changed PG: %s", diff)
}
if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(beforeIndex, after) {
t.Fatal("dispatching+missing changed the index")
}
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
t.Fatal("dispatching+missing changed objects")
}
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchDispatching {
t.Fatalf("state regressed to %s", state)
}
}
// A11(单元级):收尾检查拒绝仍停在 dispatching 的 effect,即使内容看起来已恢复。
func TestDS01CloseoutRejectsUnresolvedDispatch(t *testing.T) {
fx, snapshot, target, backups, wire := ds01Fixture(t)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
authority.attach(target)
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
ledger := outcome.Backup.WriteSet
if err := target.AdoptWriteSet(*ledger); err != nil {
t.Fatal(err)
}
change := indexChangeOf(t, ledger, "chunk-target")
body, err := target.restoreIndexClaimBody(change)
if err != nil {
t.Fatal(err)
}
binding, err := target.restoreMarkerBinding(change, body)
if err != nil {
t.Fatal(err)
}
if permit, _, err := authority.store.AcquireRestoreMarker(fx.ctx, binding); err != nil || permit == nil {
t.Fatalf("acquire: %v", err)
}
if err := target.verifyRestoreEffectsClosed(fx.ctx); err == nil ||
!strings.Contains(err.Error(), "held") {
t.Fatalf("closeout accepted an unresolved dispatch: %v", err)
}
}
// A10:唯一发送必须不可被传输层重放(GetBody=nil),且默认路径仍可重放。
func TestDS01MarkerRequestIsNotReplayable(t *testing.T) {
var replayable []bool
transport := ds01RoundTripFunc(func(r *http.Request) (*http.Response, error) {
replayable = append(replayable, r.GetBody != nil)
body := `{"_index":"i","_id":"d","result":"created","_seq_no":1,"_primary_term":1}`
return &http.Response{StatusCode: http.StatusOK, Request: r,
Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{"Content-Type": {"application/json"}}}, nil
})
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: "http://127.0.0.1:19200", Index: "i", PageSize: 10,
HTTPClient: &http.Client{Transport: transport}})
if err != nil {
t.Fatal(err)
}
var receipt indexReceipt
if err := exporter.requestJSONOnce(context.Background(), http.MethodPut, "/i/_create/d?refresh=true", map[string]any{"content": "x"}, &receipt); err != nil {
t.Fatal(err)
}
if err := exporter.requestJSON(context.Background(), http.MethodPut, "/i/_doc/d?refresh=true", map[string]any{"content": "x"}, &receipt); err != nil {
t.Fatal(err)
}
if len(replayable) != 2 {
t.Fatalf("requests = %d", len(replayable))
}
if replayable[0] {
t.Fatal("single-send request is still replayable by the transport")
}
if !replayable[1] {
t.Fatal("default request lost its normal replay semantics")
}
}
type ds01RoundTripFunc func(*http.Request) (*http.Response, error)
func (f ds01RoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
// A9(新操作系统进程):父进程留下 prepared+意图后退出该 attempt 的进程,子进程用
// 同一 archive/index/backup/ledger 与同一 attempt 身份完成恢复。
func TestDS01FreshProcessResumesPreparedEffect(t *testing.T) {
if os.Getenv("KG_DS01_CHILD") == "1" {
t.Skip("child process only")
}
fx, snapshot, target, backups, wire := ds01Fixture(t)
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
authority.attach(target)
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchPrepared {
t.Fatalf("parent state = %s", state)
}
raw, err := json.Marshal(outcome.Backup)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
backupPath := filepath.Join(dir, "backup.json")
if err := os.WriteFile(backupPath, raw, 0o600); err != nil {
t.Fatal(err)
}
resultPath := filepath.Join(dir, "result.json")
command := exec.Command(os.Args[0], "-test.run=^TestDS01RestoreHelper$", "-test.v")
command.Env = append(os.Environ(),
"KG_DS01_CHILD=1",
"KG_DS01_ARCHIVE="+fx.archiveRoot,
"KG_DS01_INDEX="+fx.index,
"KG_DS01_BACKUP="+backupPath,
"KG_DS01_RESULT="+resultPath,
"KG_DS01_GOV_DSN="+authority.dsn,
"KG_DS01_TARGET_ID="+authority.targetIdentity,
"KG_DS01_OP="+authority.operationID,
"KG_DS01_JOB="+authority.jobID,
"KG_DS01_OWNER="+authority.owner,
"KG_DS01_EPOCH="+strconv.FormatInt(authority.epoch, 10),
"KG_DS01_SNAPSHOT="+snapshot,
)
output, err := command.CombinedOutput()
t.Logf("child output:\n%s", output)
if err != nil {
t.Fatalf("fresh process restore failed: %v", err)
}
var result struct {
Dispatch int `json:"dispatch"`
State string `json:"state"`
Restored bool `json:"restored_doc"`
}
rawResult, err := os.ReadFile(resultPath)
if err != nil {
t.Fatalf("child result missing: %v", err)
}
if err := json.Unmarshal(rawResult, &result); err != nil {
t.Fatal(err)
}
if result.Dispatch != 1 || result.State != libraryrelease.RestoreDispatchMarkerConfirmed {
t.Fatalf("child result = %+v", result)
}
if !result.Restored {
t.Fatal("child did not restore chunk-target to its frozen before-image")
}
if wire.dispatch != 0 {
t.Fatalf("parent sent %d marker writes", wire.dispatch)
}
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchMarkerConfirmed {
t.Fatalf("persisted state after child = %s", state)
}
}
func TestDS01RestoreHelper(t *testing.T) {
if os.Getenv("KG_DS01_CHILD") != "1" {
t.Skip("isolated subprocess only")
}
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{
archiveRoot: os.Getenv("KG_DS01_ARCHIVE"), index: os.Getenv("KG_DS01_INDEX")})
raw, err := os.ReadFile(os.Getenv("KG_DS01_BACKUP"))
if err != nil {
t.Fatal(err)
}
var backup libraryrelease.BackupRef
if err := json.Unmarshal(raw, &backup); err != nil {
t.Fatal(err)
}
ledger, err := LoadWriteSet(fx.archiveRoot, backup.WriteSet.Digest)
if err != nil {
t.Fatal(err)
}
backup.WriteSet = ledger
ctx := context.Background()
pool, err := pgxpool.New(ctx, os.Getenv("KG_DS01_GOV_DSN"))
if err != nil {
t.Fatalf("child governance connect: %v", err)
}
defer pool.Close()
store := postgresstore.NewStore(pool)
epoch, err := strconv.ParseInt(os.Getenv("KG_DS01_EPOCH"), 10, 64)
if err != nil {
t.Fatal(err)
}
target := fx.target(t)
target.config.RestoreDispatch = store
target.config.RestoreTargetIdentity = os.Getenv("KG_DS01_TARGET_ID")
target.config.RestoreScope = "internal"
target.config.RestoreOperationID = os.Getenv("KG_DS01_OP")
target.config.RestoreWindowJobID = os.Getenv("KG_DS01_JOB")
target.config.RestoreAttemptOwner = os.Getenv("KG_DS01_OWNER")
target.config.RestoreAttemptEpoch = epoch
target.config.RestoreSnapshotID = os.Getenv("KG_DS01_SNAPSHOT")
client := *target.search.client
inner := client.Transport
if inner == nil {
inner = http.DefaultTransport
}
wire := &restoreMarkerWire{inner: inner}
client.Transport = wire
target.search.client = &client
if err := target.Restore(ctx, backup); err != nil {
t.Fatalf("fresh process restore: %v", err)
}
if wire.dispatch != 1 {
t.Fatalf("fresh process dispatched %d marker creates", wire.dispatch)
}
var state string
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
t.Fatal(err)
}
change := indexChangeOf(t, ledger, "chunk-target")
observed, err := target.observePlannedIndex(ctx, change)
if err != nil {
t.Fatal(err)
}
result, err := json.Marshal(map[string]any{"dispatch": wire.dispatch, "state": state,
"restored_doc": indexBytesEqual(observed, change.Before)})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(os.Getenv("KG_DS01_RESULT"), result, 0o600); err != nil {
t.Fatal(err)
}
}
@@ -179,7 +179,7 @@ func (t *ProductionLibraryTarget) applyPlannedIndex(ctx context.Context) error {
}
return nil
}
func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) {
func (t *ProductionLibraryTarget) indexMutation(c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (string, string, any, error) {
method := http.MethodPut
endpoint := "_doc"
if create {
@@ -191,7 +191,7 @@ func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libr
query := url.Values{"refresh": {"true"}}
if !create {
if seq == nil || term == nil {
return indexReceipt{}, errors.New("index mutation lacks CAS")
return "", "", nil, errors.New("index mutation lacks CAS")
}
query.Set("if_seq_no", fmt.Sprint(*seq))
query.Set("if_primary_term", fmt.Sprint(*term))
@@ -201,15 +201,30 @@ func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libr
if len(body) > 0 {
payload = body
}
return method, path, payload, nil
}
func validateIndexReceipt(c libraryrelease.IndexWrite, hasBody bool, receipt indexReceipt) error {
if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil {
return errors.New("index mutation receipt incomplete")
}
if !hasBody && receipt.Result != "deleted" || hasBody && receipt.Result != "created" && receipt.Result != "updated" {
return errors.New("index mutation receipt not confirmed")
}
return nil
}
func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) {
method, path, payload, err := t.indexMutation(c, body, seq, term, create)
if err != nil {
return indexReceipt{}, err
}
var receipt indexReceipt
if err := t.search.requestJSON(ctx, method, path, payload, &receipt); err != nil {
return receipt, fmt.Errorf("index effect outcome unknown: %w", err)
}
if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil {
return receipt, errors.New("index mutation receipt incomplete")
}
if len(body) == 0 && receipt.Result != "deleted" || len(body) > 0 && receipt.Result != "created" && receipt.Result != "updated" {
return receipt, errors.New("index mutation receipt not confirmed")
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
return receipt, err
}
return receipt, nil
}
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"knowledge-governance/internal/libraryrelease"
)
@@ -85,12 +86,45 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if !start {
return errors.New("legacy unknown index restoration lacks attributable marker intent")
}
// 新协议:先在治理库建立 prepared(文件意图之前),再落文件意图。文件失败只留
// prepared(零派发,下一进程可补写同内容意图);反过来"先文件后台账"则会造出
// 无法证明未派发的旧证据。
if t.config.RestoreDispatch != nil && len(c.After) == 0 {
binding, err := t.restoreMarkerBinding(c, body)
if err != nil {
return err
}
status, err := t.config.RestoreDispatch.PrepareRestoreMarker(ctx, binding)
if err != nil {
return err
}
if status.Held {
return heldRestoreMarker(status, "首次恢复不能建立 prepared")
}
}
if err = t.beginRestoreMarker(ctx, c, applied); err != nil {
return err
}
if markIntent, err = t.loadIndexEffect(c, "restore-mark-intent"); err != nil {
return err
}
} else if t.config.RestoreDispatch != nil && len(c.After) == 0 {
// 既有意图必须能在新台账里找到同一 effect:找不到就是旧协议的未知历史,
// 不能借"文件看起来完整"补一份 prepared。
binding, err := t.restoreMarkerBinding(c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return err
}
if !found {
return fmt.Errorf("%w: 既存意图没有派发台账(旧协议未知历史)", libraryrelease.ErrRestoreDispatchHeld)
}
if status.State == libraryrelease.RestoreDispatchPrepared && claimed != nil {
return heldRestoreMarker(status, "prepared 台账却已有 claimed 证据:协议证据矛盾")
}
}
finalizing, err := t.loadIndexEffect(c, "restore-finalize-intent")
if err != nil {
@@ -100,6 +134,11 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if err != nil {
return err
}
// 台账停在 dispatching 时,只有"精确 marker+完整内容+可靠世代"才允许沿原确认
// 分支继续;确认失败即 held,绝不为了活性重发 create。
if err := t.confirmDispatchedMarker(ctx, c, body, observed); err != nil {
return err
}
if finalizing != nil {
// 干净写入在派发边界上被拒绝时,标记体会原样停在自己那份世代上;落地过一次
// 干净写入必然换掉内容并推进世代。两条同时成立才证明没写过,否则 held。
@@ -111,33 +150,47 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if !indexBytesEqual(observed, body) {
// The marker is absent: either the intent never reached the index (a
// deterministic refusal before dispatch) or its write outcome is unknown.
// Only a server-side generation that every successful write would move —
// available for updates — separates the two. The exclusive-create path has
// no durable, non-reusable evidence (ES counters are shard-lifetime
// statistics and are cleared on shard relocation), so it stays held (B19)
// and is never adopted by appearance or by counter equality.
unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, freshIntent)
if err != nil {
return err
}
if !unsent {
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
}
if err := t.refuseRestoreWrite(c); err != nil {
return err
}
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
if err != nil {
return err
}
receipt.Marked = true
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
return err
}
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
observed, err = t.observePlannedIndex(ctx, c)
if err != nil {
return err
// 删除型的恢复写入是独占 create:服务端没有"从未写过"的可持久证据。
// 新协议只允许台账里 prepared 的 effect 重新竞争一次发送许可;旧意图没有
// 台账、或台账已 dispatching 的,一律 held(上面的分支已拒绝)。
if len(c.After) == 0 && t.config.RestoreDispatch != nil {
receipt, err := t.dispatchRestoreMarkerCreate(ctx, c, body)
if err != nil {
return err
}
receipt.Marked = true
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
return err
}
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
observed, err = t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
} else {
unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, freshIntent)
if err != nil {
return err
}
if !unsent {
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
}
if err := t.refuseRestoreWrite(c); err != nil {
return err
}
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
if err != nil {
return err
}
receipt.Marked = true
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
return err
}
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
observed, err = t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
}
}
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
@@ -0,0 +1,299 @@
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"knowledge-governance/internal/libraryrelease"
)
// DS01:删除型恢复(Before 非空、After 为空)的标记 create 是唯一会"无中生有"的
// 恢复写:ES 里文档缺失与"从未写过"完全同形,所以它不能靠外观、stats 或内存计数
// 放行。新协议把"是否已经有人取得发送许可"放进治理库的单调台账;本文件是 adapter
// 侧的全部接入点——准入、唯一派发、确认与收尾检查。
//
// 明确不保证:ES 端 fence;已 dispatching 的旧 owner 绝不迟到一次。协议只保证
// 遵守协议的派发者不会得到第二次许可,且 dispatching 永不回退。
func (t *ProductionLibraryTarget) restoreMarkerBinding(c libraryrelease.IndexWrite, body json.RawMessage) (libraryrelease.RestoreMarkerBinding, error) {
if t.config.RestoreDispatch == nil {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch protocol is not wired")
}
if t.activePlan == nil {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch requires the frozen plan")
}
if t.config.RestoreTargetIdentity == "" || t.config.RestoreOperationID == "" ||
t.config.RestoreWindowJobID == "" || t.config.RestoreAttemptOwner == "" ||
t.config.RestoreAttemptEpoch <= 0 {
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original job/attempt identity")
}
changeRaw, err := json.Marshal(c)
if err != nil {
return libraryrelease.RestoreMarkerBinding{}, err
}
changeDigest := sha256.Sum256(changeRaw)
markerDigest := sha256.Sum256(body)
method, path, _, err := t.indexMutation(c, body, nil, nil, true)
if err != nil {
return libraryrelease.RestoreMarkerBinding{}, err
}
requestDigest := sha256.Sum256([]byte(method + "\x00" + path))
return libraryrelease.RestoreMarkerBinding{
Key: libraryrelease.RestoreMarkerKey{
TargetIdentity: t.config.RestoreTargetIdentity,
OperationID: t.config.RestoreOperationID,
WindowJobID: t.config.RestoreWindowJobID,
PlanDigest: t.activePlan.Digest,
Index: c.Index,
DocumentID: c.ID,
Phase: libraryrelease.RestoreMarkerPhaseCreate,
},
Scope: t.config.RestoreScope,
SourceSnapshot: t.config.RestoreSnapshotID,
BackupSnapshot: t.config.RestoreSnapshotID,
ChangeDigest: hex.EncodeToString(changeDigest[:]),
MarkerDigest: hex.EncodeToString(markerDigest[:]),
RequestDigest: hex.EncodeToString(requestDigest[:]),
Owner: t.config.RestoreAttemptOwner,
Epoch: t.config.RestoreAttemptEpoch,
}, nil
}
func heldRestoreMarker(status libraryrelease.RestoreDispatchStatus, context string) error {
return fmt.Errorf("%w: %s(state=%s)", libraryrelease.ErrRestoreDispatchHeld, context, status.State)
}
// admitRestorePlan 是 Restore 在**任何** PG/对象/索引写之前的纯读准入。
//
// 删除型恢复一旦开始写 recovery 的行/对象,后面才发现某个 effect 的派发结果未知,
// 目标已经从"未动"变成"半恢复"。所以先按整份冻结计划逐 effect 判定:
// - 台账不存在(旧世界):只有"没有任何恢复意图"才算首次恢复;有意图就 held;
// - prepared:唯一可续作区段;
// - dispatching:只有精确 marker+世代才可走确认分支,否则 held;
// - marker_confirmed:只允许确认后的 clean 路径。
func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
for _, c := range t.activePlan.Index {
if len(c.After) != 0 {
continue
}
body, err := t.restoreIndexClaimBody(c)
if err != nil {
return err
}
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
if err != nil {
return err
}
// 已完成 clean 的 effect 是终态:恢复内容与证据都在时,重放不是未决。
done, err := t.loadIndexEffect(c, "restored")
if err != nil {
return err
}
if done != nil {
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if done.Receipt == nil || !indexBytesEqual(observed, c.Before) ||
!sameIndexGeneration(observed, done.Receipt.SeqNo, done.Receipt.PrimaryTerm) {
return fmt.Errorf("%w: 已标记 restored 的 effect 与目标现状不符", libraryrelease.ErrRestoreDispatchHeld)
}
continue
}
if t.config.RestoreDispatch != nil {
binding, bindErr := t.restoreMarkerBinding(c, body)
if bindErr != nil {
return bindErr
}
status, found, inspErr := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if inspErr != nil {
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
}
if found {
switch status.State {
case libraryrelease.RestoreDispatchPrepared:
continue
case libraryrelease.RestoreDispatchDispatching, libraryrelease.RestoreDispatchMarkerConfirmed:
observed, obsErr := t.observePlannedIndex(ctx, c)
if obsErr != nil {
return obsErr
}
if len(c.After) == 0 && indexBytesEqual(observed, body) && observed.SeqNo != nil && observed.PrimaryTerm != nil {
continue // 只允许原确认分支继续,不重发 create
}
return heldRestoreMarker(status, "目标上缺少可归属的精确恢复标记")
default:
return heldRestoreMarker(status, "派发台账状态不可识别")
}
}
}
if intent == nil {
continue
}
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if indexBytesEqual(observed, body) {
continue // 旧标记已在目标上:按既有确认/CAS 规则处理,不派发新 create
}
return fmt.Errorf("%w: 既存恢复意图没有新协议派发台账,无法证明未派发", libraryrelease.ErrRestoreDispatchHeld)
}
return nil
}
// confirmDispatchedMarker 只在"marker 与冻结内容逐位相同且世代齐全"时记录回执。
// 它不发送任何请求;确认失败即 held,绝不因为"看起来已经写好了"就继续 clean。
func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, observed indexWriteGuard) error {
if t.config.RestoreDispatch == nil || len(c.After) != 0 {
return nil
}
binding, err := t.restoreMarkerBinding(c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return err
}
if !found {
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
}
switch status.State {
case libraryrelease.RestoreDispatchPrepared:
// 还没派发:由调用方按 prepared 竞争许可。
return nil
case libraryrelease.RestoreDispatchMarkerConfirmed:
return nil
case libraryrelease.RestoreDispatchDispatching:
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
return heldRestoreMarker(status, "dispatching 结果未知且目标上没有精确标记")
}
receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: "created", SeqNo: observed.SeqNo, PrimaryTerm: observed.PrimaryTerm}
raw, err := json.Marshal(receipt)
if err != nil {
return err
}
return t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw)
default:
return heldRestoreMarker(status, "派发台账状态不可识别")
}
}
// dispatchRestoreMarkerCreate 是恢复期唯一获得发送许可的入口:preflight 与故障钩子
// 都在 CAS 之前,Acquire 只在 dispatching 明确提交后返回许可。
func (t *ProductionLibraryTarget) dispatchRestoreMarkerCreate(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (indexReceipt, error) {
if t.config.RestoreDispatch == nil {
return indexReceipt{}, errors.New("恢复标记派发协议未接线:拒绝发送")
}
binding, err := t.restoreMarkerBinding(c, body)
if err != nil {
return indexReceipt{}, err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return indexReceipt{}, err
}
if !found || status.State != libraryrelease.RestoreDispatchPrepared {
if err != nil {
return indexReceipt{}, err
}
return indexReceipt{}, heldRestoreMarker(status, "没有可竞争的 prepared 许可")
}
// 控制流故障注入必须在 CAS 之前:拒绝时状态仍是 prepared,可安全重试,零转发。
if err := t.refuseRestoreWrite(c); err != nil {
return indexReceipt{}, err
}
permit, acquireStatus, err := t.config.RestoreDispatch.AcquireRestoreMarker(ctx, binding)
if err != nil {
return indexReceipt{}, err
}
if permit == nil {
return indexReceipt{}, heldRestoreMarker(acquireStatus, "许可竞争未胜出")
}
receipt, err := t.sendRestoreMarkerOnce(ctx, c, body, binding, permit)
if err != nil {
return indexReceipt{}, err
}
raw, err := json.Marshal(receipt)
if err != nil {
return indexReceipt{}, err
}
// 回执来自这次唯一发送:必须落进台账才算确认。写失败时 marker 已在目标上,
// 下一进程走"精确 marker+世代"确认分支,不会重发 create。
if err := t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw); err != nil {
return indexReceipt{}, fmt.Errorf("恢复标记已发送但确认未落库:%w", err)
}
return receipt, nil
}
// sendRestoreMarkerOnce 是唯一的真实网络发送点:许可消费一次、请求不可被传输层重放。
func (t *ProductionLibraryTarget) sendRestoreMarkerOnce(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage,
binding libraryrelease.RestoreMarkerBinding, permit *libraryrelease.RestoreSendPermit) (indexReceipt, error) {
digest, err := binding.Digest()
if err != nil {
return indexReceipt{}, err
}
if err := permit.Consume(binding.Key, digest); err != nil {
return indexReceipt{}, err
}
method, path, payload, err := t.indexMutation(c, body, nil, nil, true)
if err != nil {
return indexReceipt{}, err
}
var receipt indexReceipt
if err := t.search.requestJSONOnce(ctx, method, path, payload, &receipt); err != nil {
return receipt, fmt.Errorf("restore marker outcome unknown: %w", err)
}
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
return receipt, err
}
return receipt, nil
}
// verifyRestoreEffectsClosed 是 Restore 收尾前的最后一道检查:任何一个删除型 effect
// 只要留下恢复意图,就必须已经完成 clean(restored 证据),且台账不得停在 dispatching。
func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context) error {
for _, c := range t.activePlan.Index {
if len(c.After) != 0 {
continue
}
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
if err != nil {
return err
}
if intent == nil {
continue
}
done, err := t.loadIndexEffect(c, "restored")
if err != nil {
return err
}
if done == nil {
return fmt.Errorf("%w: 恢复收尾前仍有未决 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
}
if t.config.RestoreDispatch == nil {
continue
}
body, err := t.restoreIndexClaimBody(c)
if err != nil {
return err
}
binding, err := t.restoreMarkerBinding(c, body)
if err != nil {
return err
}
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
if err != nil {
return err
}
if found && status.State == libraryrelease.RestoreDispatchDispatching {
return heldRestoreMarker(status, "收尾前台账仍停在 dispatching")
}
}
return nil
}
@@ -57,6 +57,18 @@ type ProductionLibraryConfig struct {
MaxDocuments int
HTTPClient *http.Client
Now func() time.Time
// 以下字段只用于 DS01 恢复标记派发协议:RestoreDispatch 为 nil 时该能力整体
// fail closed(既不对旧证据放行,也不产生新证据)。身份来自本次真实 attempt
// 与原作业/计划绑定,不能由调用方临时编造。
RestoreDispatch libraryrelease.RestoreMarkerGate
RestoreTargetIdentity string
RestoreScope string
RestoreOperationID string
RestoreWindowJobID string
RestoreAttemptOwner string
RestoreAttemptEpoch int64
// RestoreSnapshotID 是本次恢复要用的那份预备份(与原 window/plan 绑定)。
RestoreSnapshotID string
}
func (config ProductionLibraryConfig) validate(scope ProductionScope) error {
@@ -968,6 +980,11 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
if err := ensureNoResourceGrants(ctx, t.pool, backup.WriteSet.CreatedIDs("resources")); err != nil {
return err
}
// DS01:整份计划的未决派发准入必须在这里——在任何 PG/对象/索引写之前。
// 否则会先回放 database/resources,最后才在 index 上说 held,目标已经半写。
if err := t.admitRestorePlan(ctx); err != nil {
return err
}
previous := t.preserveChunkSequence
t.preserveChunkSequence = true
defer func() { t.preserveChunkSequence = previous }()
@@ -1008,5 +1025,10 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
if diff := universeDiff(backup.WriteSet.Universe, after); diff != "" {
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
}
// 收尾前再拒绝一次未决 effect:任何残留的 dispatching/未 clean 的删除型恢复
// 都不允许被报成"已恢复",也不允许随后释放维护窗口。
if err := t.verifyRestoreEffectsClosed(ctx); err != nil {
return err
}
return nil
}
+15
View File
@@ -275,6 +275,18 @@ func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, erro
}
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
return exporter.requestJSONWithReplay(ctx, method, path, payload, result, true)
}
// requestJSONOnce 关闭传输层对该请求的重放能力(GetBody=nil):恢复标记 create
// 每个许可只能落地至多一次真实请求,不能让 keep-alive 重连把它悄悄发第二遍。
// Go 的 Transport 只会自动重试"可重放"的请求;Body 非空且 GetBody 为 nil 时,
// PUT/POST 不再满足该条件。
func (exporter *SearchExporter) requestJSONOnce(ctx context.Context, method, path string, payload any, result any) error {
return exporter.requestJSONWithReplay(ctx, method, path, payload, result, false)
}
func (exporter *SearchExporter) requestJSONWithReplay(ctx context.Context, method, path string, payload any, result any, replayable bool) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
@@ -298,6 +310,9 @@ func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path st
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if !replayable {
request.GetBody = nil
}
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}