feat(library): 删除型恢复按台账准入并唯一发送
为什么:恢复在发现 index held 之前可能已经回放 database/resources; "最后数据相同"不等于"本次零派发"。改为 Restore 在任何写之前按整份 冻结计划做纯读准入,收尾前再拒绝未决 effect。 删除型恢复的 _create 只从唯一 dispatcher 发出:preflight/故障钩子 在 CAS 之前,许可来自 COMMIT 明确的短事务;发送请求 GetBody=nil, 传输层无法重放。旧意图无台账、dispatching 无精确 marker 一律 held。 验证:真 ES+真 PG 覆盖旧意图零写 held、prepared 续作到终态 (create 恰 1 次)、dispatching+missing 两轮零写、新 OS 进程续作正例。
This commit is contained in:
@@ -25,7 +25,7 @@ func TestR10ProductionAdmissionIsolated(t *testing.T) {
|
||||
fx.Exec(c.sql)
|
||||
}
|
||||
before := fx.State()
|
||||
target, backups, e := librarystack.OpenProductionStack(fx.Context, fx.Config, "internal", fx.Snapshot, "")
|
||||
target, backups, e := librarystack.OpenProductionStack(fx.Context, fx.Config, "internal", fx.Snapshot, "", nil)
|
||||
if target != nil {
|
||||
target.Close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/postgres/pgtest"
|
||||
)
|
||||
|
||||
// DS01:删除型恢复派发协议的 adapter 层验收。
|
||||
//
|
||||
// 前置是真 ES、真 PG;治理台账放在**独立的治理 schema**(不是业务目标 schema)。
|
||||
// 这里不制造"prepared 且从未派发"的假正例:prepared 一律由真实 Prepare 产生。
|
||||
|
||||
const ds01TargetIdentity = "kg-ds01-test-target"
|
||||
|
||||
type ds01Authority struct {
|
||||
store *postgresstore.Store
|
||||
pool *pgxpool.Pool
|
||||
dsn string
|
||||
operationID string
|
||||
jobID string
|
||||
owner string
|
||||
epoch int64
|
||||
targetIdentity string
|
||||
snapshot string
|
||||
}
|
||||
|
||||
func newDS01Authority(t *testing.T, jobID, snapshot string) *ds01Authority {
|
||||
t.Helper()
|
||||
base := os.Getenv("KG_DS01_GOVERNANCE_DSN")
|
||||
if base == "" {
|
||||
t.Skip("requires an isolated governance database (KG_DS01_GOVERNANCE_DSN)")
|
||||
}
|
||||
dsn := pgtest.IsolatedDatabaseURL(t, base)
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect governance: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("apply governance migrations: %v", err)
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
now := time.Now().UTC()
|
||||
op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{
|
||||
ID: "op-" + jobID, Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: snapshot,
|
||||
RequestDigest: "digest-" + jobID, OperationKey: jobID, Actor: "ds01-test", CreatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ensure operation: %v", err)
|
||||
}
|
||||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
|
||||
ID: jobID, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: op.ID,
|
||||
IdempotencyKey: "library.job:" + op.ID, RequestDigest: "digest-" + jobID,
|
||||
TargetSnapshotID: snapshot, Actor: "ds01-test",
|
||||
}, governance.SyncStages(), now); err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
attempt, err := store.BeginLibraryAttempt(ctx, jobID, "worker-ds01#queue#1", 10*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatalf("begin attempt: %v", err)
|
||||
}
|
||||
if err := store.ActivateRestoreDispatch(ctx, ds01TargetIdentity, "ds01-test", "sha256:drain-evidence", now); err != nil {
|
||||
t.Fatalf("activate dispatch: %v", err)
|
||||
}
|
||||
return &ds01Authority{store: store, pool: pool, dsn: dsn, operationID: attempt.OperationID,
|
||||
jobID: attempt.JobID, owner: attempt.Owner, epoch: int64(attempt.Epoch),
|
||||
targetIdentity: ds01TargetIdentity, snapshot: snapshot}
|
||||
}
|
||||
|
||||
// attach 把真实治理台账与本次 attempt 身份接到目标上。
|
||||
func (a *ds01Authority) attach(target *ProductionLibraryTarget) {
|
||||
target.config.RestoreDispatch = a.store
|
||||
target.config.RestoreTargetIdentity = a.targetIdentity
|
||||
target.config.RestoreScope = "internal"
|
||||
target.config.RestoreOperationID = a.operationID
|
||||
target.config.RestoreWindowJobID = a.jobID
|
||||
target.config.RestoreAttemptOwner = a.owner
|
||||
target.config.RestoreAttemptEpoch = a.epoch
|
||||
target.config.RestoreSnapshotID = a.snapshot
|
||||
}
|
||||
|
||||
func (a *ds01Authority) dispatchRow(t *testing.T) (state, owner string) {
|
||||
t.Helper()
|
||||
if err := a.pool.QueryRow(context.Background(),
|
||||
`SELECT state, dispatch_owner FROM library_index_restore_dispatch`).Scan(&state, &owner); err != nil {
|
||||
t.Fatalf("read dispatch row: %v", err)
|
||||
}
|
||||
return state, owner
|
||||
}
|
||||
|
||||
func (a *ds01Authority) dispatchRows(t *testing.T) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := a.pool.QueryRow(context.Background(), `SELECT count(*) FROM library_index_restore_dispatch`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// ds01RunToBoundary 走真实应用→wiki 失败→自动 rollback:在删除型恢复标记的边界上
|
||||
// 用控制流故障拒绝,留下 prepared + 意图 + 0 次转发。
|
||||
func ds01RunToBoundary(t *testing.T, fx *writeSetFixture, snapshot string, target *ProductionLibraryTarget,
|
||||
backups *ProductionBackupStore, wire *restoreMarkerWire) libraryrelease.Outcome {
|
||||
t.Helper()
|
||||
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
|
||||
if c.ID != "chunk-target" {
|
||||
return nil
|
||||
}
|
||||
return errDS01BoundaryRefusal
|
||||
}
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
|
||||
if outcome.State != "rollback_failed" || !strings.Contains(outcome.Rollback, errDS01BoundaryRefusal.Error()) {
|
||||
t.Fatalf("rollback_failed with DS01 boundary refusal expected, got state=%q rollback=%q", outcome.State, outcome.Rollback)
|
||||
}
|
||||
if wire.dispatch != 0 {
|
||||
t.Fatalf("boundary refusal dispatched %d marker writes", wire.dispatch)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
var errDS01BoundaryRefusal = &ds01BoundaryError{}
|
||||
|
||||
type ds01BoundaryError struct{}
|
||||
|
||||
func (*ds01BoundaryError) Error() string {
|
||||
return "injected DS01 boundary refusal before dispatch CAS"
|
||||
}
|
||||
|
||||
func ds01Fixture(t *testing.T) (*writeSetFixture, string, *ProductionLibraryTarget, *ProductionBackupStore, *restoreMarkerWire) {
|
||||
t.Helper()
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshot := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
client := *target.search.client
|
||||
inner := client.Transport
|
||||
if inner == nil {
|
||||
inner = http.DefaultTransport
|
||||
}
|
||||
wire := &restoreMarkerWire{inner: inner}
|
||||
client.Transport = wire
|
||||
target.search.client = &client
|
||||
return fx, snapshot, target, backups, wire
|
||||
}
|
||||
|
||||
// A7:旧 intent + missing、无台账 ⇒ 在任何 PG/resources 写之前 held。
|
||||
func TestDS01LegacyIntentWithoutLedgerStaysHeldBeforeAnyWrite(t *testing.T) {
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
// 先用旧协议(未接线)造出"既存意图、无台账"的现场。
|
||||
legacy := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
ledger := legacy.Backup.WriteSet
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
requireIndexEffect(t, fx, ledger, change, "restore-mark-intent", true)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
if rows := authority.dispatchRows(t); rows != 0 {
|
||||
t.Fatalf("legacy run created %d dispatch rows", rows)
|
||||
}
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
beforeIndex := recoveryIndexReadback(t, fx)
|
||||
beforeObjects := objectVersionSet(t, fx)
|
||||
beforeDispatch := wire.dispatch
|
||||
|
||||
authority.attach(target)
|
||||
target.restoreMarkerFault = nil
|
||||
err := target.Restore(fx.ctx, legacy.Backup)
|
||||
if err == nil || !strings.Contains(err.Error(), "held") {
|
||||
t.Fatalf("legacy intent without ledger must stay held, got %v", err)
|
||||
}
|
||||
if wire.dispatch != beforeDispatch {
|
||||
t.Fatalf("held legacy restoration dispatched %d marker writes", wire.dispatch-beforeDispatch)
|
||||
}
|
||||
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("held legacy restoration wrote PG before admission: %s", diff)
|
||||
}
|
||||
if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(beforeIndex, after) {
|
||||
t.Fatal("held legacy restoration changed the index")
|
||||
}
|
||||
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
|
||||
t.Fatal("held legacy restoration changed objects")
|
||||
}
|
||||
if rows := authority.dispatchRows(t); rows != 0 {
|
||||
t.Fatalf("legacy path wrote %d dispatch rows", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A9(同进程):prepared + 意图,边界拒绝 0 转发;新调用用同一 window/backup/plan
|
||||
// 完成恢复,create 恰好转发一次,台账 marker_confirmed。
|
||||
func TestDS01PreparedEffectResumesToTerminalState(t *testing.T) {
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
authority.attach(target)
|
||||
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
if state, owner := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchPrepared || owner != "" {
|
||||
t.Fatalf("after boundary refusal row = %s/%s", state, owner)
|
||||
}
|
||||
ledger := outcome.Backup.WriteSet
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
requireIndexEffect(t, fx, ledger, change, "restore-claimed", false)
|
||||
|
||||
target.restoreMarkerFault = nil
|
||||
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
|
||||
t.Fatalf("prepared effect did not resume: %v", err)
|
||||
}
|
||||
if wire.dispatch != 1 {
|
||||
t.Fatalf("marker create forwarded %d times, want exactly 1", wire.dispatch)
|
||||
}
|
||||
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
t.Fatalf("terminal dispatch state = %s", state)
|
||||
}
|
||||
requireIndexEffect(t, fx, ledger, change, "restored", true)
|
||||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||||
t.Fatalf("restoration verify: %v", err)
|
||||
}
|
||||
// 重放同一次恢复:不再产生新的 create。
|
||||
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
|
||||
t.Fatalf("replayed restoration: %v", err)
|
||||
}
|
||||
if wire.dispatch != 1 {
|
||||
t.Fatalf("replayed restoration dispatched %d marker creates", wire.dispatch)
|
||||
}
|
||||
}
|
||||
|
||||
// A8:commit 已落地、Do 之前进程死掉 ⇒ dispatching+missing 永久 held,零新发送,
|
||||
// PG/对象/索引不变;第二次仍 held。
|
||||
func TestDS01DispatchingMissingStaysHeldWithZeroWrites(t *testing.T) {
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
authority.attach(target)
|
||||
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
|
||||
// 模拟"CAS 提交成功、发送前进程死掉":真实 Acquire 把行推进到 dispatching,但不发送。
|
||||
ledger := outcome.Backup.WriteSet
|
||||
if err := target.AdoptWriteSet(*ledger); err != nil {
|
||||
t.Fatalf("adopt plan: %v", err)
|
||||
}
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
body, err := target.restoreIndexClaimBody(change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding, err := target.restoreMarkerBinding(change, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
permit, _, err := authority.store.AcquireRestoreMarker(fx.ctx, binding)
|
||||
if err != nil || permit == nil {
|
||||
t.Fatalf("simulate committed dispatch: %v", err)
|
||||
}
|
||||
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchDispatching {
|
||||
t.Fatalf("simulated state = %s", state)
|
||||
}
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
beforeIndex := recoveryIndexReadback(t, fx)
|
||||
beforeObjects := objectVersionSet(t, fx)
|
||||
beforeDispatch := wire.dispatch
|
||||
|
||||
target.restoreMarkerFault = nil
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
err := target.Restore(fx.ctx, outcome.Backup)
|
||||
if err == nil || !strings.Contains(err.Error(), "held") {
|
||||
t.Fatalf("attempt %d: dispatching+missing must stay held, got %v", attempt, err)
|
||||
}
|
||||
}
|
||||
if wire.dispatch != beforeDispatch {
|
||||
t.Fatalf("dispatching+missing dispatched %d marker writes", wire.dispatch-beforeDispatch)
|
||||
}
|
||||
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("dispatching+missing changed PG: %s", diff)
|
||||
}
|
||||
if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(beforeIndex, after) {
|
||||
t.Fatal("dispatching+missing changed the index")
|
||||
}
|
||||
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
|
||||
t.Fatal("dispatching+missing changed objects")
|
||||
}
|
||||
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchDispatching {
|
||||
t.Fatalf("state regressed to %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
// A11(单元级):收尾检查拒绝仍停在 dispatching 的 effect,即使内容看起来已恢复。
|
||||
func TestDS01CloseoutRejectsUnresolvedDispatch(t *testing.T) {
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
authority.attach(target)
|
||||
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
ledger := outcome.Backup.WriteSet
|
||||
if err := target.AdoptWriteSet(*ledger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
body, err := target.restoreIndexClaimBody(change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding, err := target.restoreMarkerBinding(change, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if permit, _, err := authority.store.AcquireRestoreMarker(fx.ctx, binding); err != nil || permit == nil {
|
||||
t.Fatalf("acquire: %v", err)
|
||||
}
|
||||
if err := target.verifyRestoreEffectsClosed(fx.ctx); err == nil ||
|
||||
!strings.Contains(err.Error(), "held") {
|
||||
t.Fatalf("closeout accepted an unresolved dispatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A10:唯一发送必须不可被传输层重放(GetBody=nil),且默认路径仍可重放。
|
||||
func TestDS01MarkerRequestIsNotReplayable(t *testing.T) {
|
||||
var replayable []bool
|
||||
transport := ds01RoundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
replayable = append(replayable, r.GetBody != nil)
|
||||
body := `{"_index":"i","_id":"d","result":"created","_seq_no":1,"_primary_term":1}`
|
||||
return &http.Response{StatusCode: http.StatusOK, Request: r,
|
||||
Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{"Content-Type": {"application/json"}}}, nil
|
||||
})
|
||||
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: "http://127.0.0.1:19200", Index: "i", PageSize: 10,
|
||||
HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var receipt indexReceipt
|
||||
if err := exporter.requestJSONOnce(context.Background(), http.MethodPut, "/i/_create/d?refresh=true", map[string]any{"content": "x"}, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := exporter.requestJSON(context.Background(), http.MethodPut, "/i/_doc/d?refresh=true", map[string]any{"content": "x"}, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(replayable) != 2 {
|
||||
t.Fatalf("requests = %d", len(replayable))
|
||||
}
|
||||
if replayable[0] {
|
||||
t.Fatal("single-send request is still replayable by the transport")
|
||||
}
|
||||
if !replayable[1] {
|
||||
t.Fatal("default request lost its normal replay semantics")
|
||||
}
|
||||
}
|
||||
|
||||
type ds01RoundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f ds01RoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
// A9(新操作系统进程):父进程留下 prepared+意图后退出该 attempt 的进程,子进程用
|
||||
// 同一 archive/index/backup/ledger 与同一 attempt 身份完成恢复。
|
||||
func TestDS01FreshProcessResumesPreparedEffect(t *testing.T) {
|
||||
if os.Getenv("KG_DS01_CHILD") == "1" {
|
||||
t.Skip("child process only")
|
||||
}
|
||||
fx, snapshot, target, backups, wire := ds01Fixture(t)
|
||||
authority := newDS01Authority(t, requestFor(snapshot).JobID, snapshot)
|
||||
authority.attach(target)
|
||||
outcome := ds01RunToBoundary(t, fx, snapshot, target, backups, wire)
|
||||
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchPrepared {
|
||||
t.Fatalf("parent state = %s", state)
|
||||
}
|
||||
raw, err := json.Marshal(outcome.Backup)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
backupPath := filepath.Join(dir, "backup.json")
|
||||
if err := os.WriteFile(backupPath, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resultPath := filepath.Join(dir, "result.json")
|
||||
command := exec.Command(os.Args[0], "-test.run=^TestDS01RestoreHelper$", "-test.v")
|
||||
command.Env = append(os.Environ(),
|
||||
"KG_DS01_CHILD=1",
|
||||
"KG_DS01_ARCHIVE="+fx.archiveRoot,
|
||||
"KG_DS01_INDEX="+fx.index,
|
||||
"KG_DS01_BACKUP="+backupPath,
|
||||
"KG_DS01_RESULT="+resultPath,
|
||||
"KG_DS01_GOV_DSN="+authority.dsn,
|
||||
"KG_DS01_TARGET_ID="+authority.targetIdentity,
|
||||
"KG_DS01_OP="+authority.operationID,
|
||||
"KG_DS01_JOB="+authority.jobID,
|
||||
"KG_DS01_OWNER="+authority.owner,
|
||||
"KG_DS01_EPOCH="+strconv.FormatInt(authority.epoch, 10),
|
||||
"KG_DS01_SNAPSHOT="+snapshot,
|
||||
)
|
||||
output, err := command.CombinedOutput()
|
||||
t.Logf("child output:\n%s", output)
|
||||
if err != nil {
|
||||
t.Fatalf("fresh process restore failed: %v", err)
|
||||
}
|
||||
var result struct {
|
||||
Dispatch int `json:"dispatch"`
|
||||
State string `json:"state"`
|
||||
Restored bool `json:"restored_doc"`
|
||||
}
|
||||
rawResult, err := os.ReadFile(resultPath)
|
||||
if err != nil {
|
||||
t.Fatalf("child result missing: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(rawResult, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Dispatch != 1 || result.State != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
t.Fatalf("child result = %+v", result)
|
||||
}
|
||||
if !result.Restored {
|
||||
t.Fatal("child did not restore chunk-target to its frozen before-image")
|
||||
}
|
||||
if wire.dispatch != 0 {
|
||||
t.Fatalf("parent sent %d marker writes", wire.dispatch)
|
||||
}
|
||||
if state, _ := authority.dispatchRow(t); state != libraryrelease.RestoreDispatchMarkerConfirmed {
|
||||
t.Fatalf("persisted state after child = %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDS01RestoreHelper(t *testing.T) {
|
||||
if os.Getenv("KG_DS01_CHILD") != "1" {
|
||||
t.Skip("isolated subprocess only")
|
||||
}
|
||||
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{
|
||||
archiveRoot: os.Getenv("KG_DS01_ARCHIVE"), index: os.Getenv("KG_DS01_INDEX")})
|
||||
raw, err := os.ReadFile(os.Getenv("KG_DS01_BACKUP"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var backup libraryrelease.BackupRef
|
||||
if err := json.Unmarshal(raw, &backup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ledger, err := LoadWriteSet(fx.archiveRoot, backup.WriteSet.Digest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backup.WriteSet = ledger
|
||||
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, os.Getenv("KG_DS01_GOV_DSN"))
|
||||
if err != nil {
|
||||
t.Fatalf("child governance connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
store := postgresstore.NewStore(pool)
|
||||
epoch, err := strconv.ParseInt(os.Getenv("KG_DS01_EPOCH"), 10, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := fx.target(t)
|
||||
target.config.RestoreDispatch = store
|
||||
target.config.RestoreTargetIdentity = os.Getenv("KG_DS01_TARGET_ID")
|
||||
target.config.RestoreScope = "internal"
|
||||
target.config.RestoreOperationID = os.Getenv("KG_DS01_OP")
|
||||
target.config.RestoreWindowJobID = os.Getenv("KG_DS01_JOB")
|
||||
target.config.RestoreAttemptOwner = os.Getenv("KG_DS01_OWNER")
|
||||
target.config.RestoreAttemptEpoch = epoch
|
||||
target.config.RestoreSnapshotID = os.Getenv("KG_DS01_SNAPSHOT")
|
||||
|
||||
client := *target.search.client
|
||||
inner := client.Transport
|
||||
if inner == nil {
|
||||
inner = http.DefaultTransport
|
||||
}
|
||||
wire := &restoreMarkerWire{inner: inner}
|
||||
client.Transport = wire
|
||||
target.search.client = &client
|
||||
|
||||
if err := target.Restore(ctx, backup); err != nil {
|
||||
t.Fatalf("fresh process restore: %v", err)
|
||||
}
|
||||
if wire.dispatch != 1 {
|
||||
t.Fatalf("fresh process dispatched %d marker creates", wire.dispatch)
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM library_index_restore_dispatch`).Scan(&state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
change := indexChangeOf(t, ledger, "chunk-target")
|
||||
observed, err := target.observePlannedIndex(ctx, change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := json.Marshal(map[string]any{"dispatch": wire.dispatch, "state": state,
|
||||
"restored_doc": indexBytesEqual(observed, change.Before)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(os.Getenv("KG_DS01_RESULT"), result, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -179,7 +179,7 @@ func (t *ProductionLibraryTarget) applyPlannedIndex(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) {
|
||||
func (t *ProductionLibraryTarget) indexMutation(c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (string, string, any, error) {
|
||||
method := http.MethodPut
|
||||
endpoint := "_doc"
|
||||
if create {
|
||||
@@ -191,7 +191,7 @@ func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libr
|
||||
query := url.Values{"refresh": {"true"}}
|
||||
if !create {
|
||||
if seq == nil || term == nil {
|
||||
return indexReceipt{}, errors.New("index mutation lacks CAS")
|
||||
return "", "", nil, errors.New("index mutation lacks CAS")
|
||||
}
|
||||
query.Set("if_seq_no", fmt.Sprint(*seq))
|
||||
query.Set("if_primary_term", fmt.Sprint(*term))
|
||||
@@ -201,15 +201,30 @@ func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libr
|
||||
if len(body) > 0 {
|
||||
payload = body
|
||||
}
|
||||
return method, path, payload, nil
|
||||
}
|
||||
|
||||
func validateIndexReceipt(c libraryrelease.IndexWrite, hasBody bool, receipt indexReceipt) error {
|
||||
if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil {
|
||||
return errors.New("index mutation receipt incomplete")
|
||||
}
|
||||
if !hasBody && receipt.Result != "deleted" || hasBody && receipt.Result != "created" && receipt.Result != "updated" {
|
||||
return errors.New("index mutation receipt not confirmed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) {
|
||||
method, path, payload, err := t.indexMutation(c, body, seq, term, create)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
var receipt indexReceipt
|
||||
if err := t.search.requestJSON(ctx, method, path, payload, &receipt); err != nil {
|
||||
return receipt, fmt.Errorf("index effect outcome unknown: %w", err)
|
||||
}
|
||||
if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil {
|
||||
return receipt, errors.New("index mutation receipt incomplete")
|
||||
}
|
||||
if len(body) == 0 && receipt.Result != "deleted" || len(body) > 0 && receipt.Result != "created" && receipt.Result != "updated" {
|
||||
return receipt, errors.New("index mutation receipt not confirmed")
|
||||
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
|
||||
return receipt, err
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
@@ -85,12 +86,45 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if !start {
|
||||
return errors.New("legacy unknown index restoration lacks attributable marker intent")
|
||||
}
|
||||
// 新协议:先在治理库建立 prepared(文件意图之前),再落文件意图。文件失败只留
|
||||
// prepared(零派发,下一进程可补写同内容意图);反过来"先文件后台账"则会造出
|
||||
// 无法证明未派发的旧证据。
|
||||
if t.config.RestoreDispatch != nil && len(c.After) == 0 {
|
||||
binding, err := t.restoreMarkerBinding(c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := t.config.RestoreDispatch.PrepareRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status.Held {
|
||||
return heldRestoreMarker(status, "首次恢复不能建立 prepared")
|
||||
}
|
||||
}
|
||||
if err = t.beginRestoreMarker(ctx, c, applied); err != nil {
|
||||
return err
|
||||
}
|
||||
if markIntent, err = t.loadIndexEffect(c, "restore-mark-intent"); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if t.config.RestoreDispatch != nil && len(c.After) == 0 {
|
||||
// 既有意图必须能在新台账里找到同一 effect:找不到就是旧协议的未知历史,
|
||||
// 不能借"文件看起来完整"补一份 prepared。
|
||||
binding, err := t.restoreMarkerBinding(c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%w: 既存意图没有派发台账(旧协议未知历史)", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
if status.State == libraryrelease.RestoreDispatchPrepared && claimed != nil {
|
||||
return heldRestoreMarker(status, "prepared 台账却已有 claimed 证据:协议证据矛盾")
|
||||
}
|
||||
}
|
||||
finalizing, err := t.loadIndexEffect(c, "restore-finalize-intent")
|
||||
if err != nil {
|
||||
@@ -100,6 +134,11 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 台账停在 dispatching 时,只有"精确 marker+完整内容+可靠世代"才允许沿原确认
|
||||
// 分支继续;确认失败即 held,绝不为了活性重发 create。
|
||||
if err := t.confirmDispatchedMarker(ctx, c, body, observed); err != nil {
|
||||
return err
|
||||
}
|
||||
if finalizing != nil {
|
||||
// 干净写入在派发边界上被拒绝时,标记体会原样停在自己那份世代上;落地过一次
|
||||
// 干净写入必然换掉内容并推进世代。两条同时成立才证明没写过,否则 held。
|
||||
@@ -111,33 +150,47 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if !indexBytesEqual(observed, body) {
|
||||
// The marker is absent: either the intent never reached the index (a
|
||||
// deterministic refusal before dispatch) or its write outcome is unknown.
|
||||
// Only a server-side generation that every successful write would move —
|
||||
// available for updates — separates the two. The exclusive-create path has
|
||||
// no durable, non-reusable evidence (ES counters are shard-lifetime
|
||||
// statistics and are cleared on shard relocation), so it stays held (B19)
|
||||
// and is never adopted by appearance or by counter equality.
|
||||
unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, freshIntent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !unsent {
|
||||
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
|
||||
}
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Marked = true
|
||||
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
||||
observed, err = t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
// 删除型的恢复写入是独占 create:服务端没有"从未写过"的可持久证据。
|
||||
// 新协议只允许台账里 prepared 的 effect 重新竞争一次发送许可;旧意图没有
|
||||
// 台账、或台账已 dispatching 的,一律 held(上面的分支已拒绝)。
|
||||
if len(c.After) == 0 && t.config.RestoreDispatch != nil {
|
||||
receipt, err := t.dispatchRestoreMarkerCreate(ctx, c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Marked = true
|
||||
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
||||
observed, err = t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, freshIntent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !unsent {
|
||||
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
|
||||
}
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Marked = true
|
||||
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
||||
observed, err = t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// DS01:删除型恢复(Before 非空、After 为空)的标记 create 是唯一会"无中生有"的
|
||||
// 恢复写:ES 里文档缺失与"从未写过"完全同形,所以它不能靠外观、stats 或内存计数
|
||||
// 放行。新协议把"是否已经有人取得发送许可"放进治理库的单调台账;本文件是 adapter
|
||||
// 侧的全部接入点——准入、唯一派发、确认与收尾检查。
|
||||
//
|
||||
// 明确不保证:ES 端 fence;已 dispatching 的旧 owner 绝不迟到一次。协议只保证
|
||||
// 遵守协议的派发者不会得到第二次许可,且 dispatching 永不回退。
|
||||
|
||||
func (t *ProductionLibraryTarget) restoreMarkerBinding(c libraryrelease.IndexWrite, body json.RawMessage) (libraryrelease.RestoreMarkerBinding, error) {
|
||||
if t.config.RestoreDispatch == nil {
|
||||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch protocol is not wired")
|
||||
}
|
||||
if t.activePlan == nil {
|
||||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch requires the frozen plan")
|
||||
}
|
||||
if t.config.RestoreTargetIdentity == "" || t.config.RestoreOperationID == "" ||
|
||||
t.config.RestoreWindowJobID == "" || t.config.RestoreAttemptOwner == "" ||
|
||||
t.config.RestoreAttemptEpoch <= 0 {
|
||||
return libraryrelease.RestoreMarkerBinding{}, errors.New("restore dispatch lacks the original job/attempt identity")
|
||||
}
|
||||
changeRaw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreMarkerBinding{}, err
|
||||
}
|
||||
changeDigest := sha256.Sum256(changeRaw)
|
||||
markerDigest := sha256.Sum256(body)
|
||||
method, path, _, err := t.indexMutation(c, body, nil, nil, true)
|
||||
if err != nil {
|
||||
return libraryrelease.RestoreMarkerBinding{}, err
|
||||
}
|
||||
requestDigest := sha256.Sum256([]byte(method + "\x00" + path))
|
||||
return libraryrelease.RestoreMarkerBinding{
|
||||
Key: libraryrelease.RestoreMarkerKey{
|
||||
TargetIdentity: t.config.RestoreTargetIdentity,
|
||||
OperationID: t.config.RestoreOperationID,
|
||||
WindowJobID: t.config.RestoreWindowJobID,
|
||||
PlanDigest: t.activePlan.Digest,
|
||||
Index: c.Index,
|
||||
DocumentID: c.ID,
|
||||
Phase: libraryrelease.RestoreMarkerPhaseCreate,
|
||||
},
|
||||
Scope: t.config.RestoreScope,
|
||||
SourceSnapshot: t.config.RestoreSnapshotID,
|
||||
BackupSnapshot: t.config.RestoreSnapshotID,
|
||||
ChangeDigest: hex.EncodeToString(changeDigest[:]),
|
||||
MarkerDigest: hex.EncodeToString(markerDigest[:]),
|
||||
RequestDigest: hex.EncodeToString(requestDigest[:]),
|
||||
Owner: t.config.RestoreAttemptOwner,
|
||||
Epoch: t.config.RestoreAttemptEpoch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func heldRestoreMarker(status libraryrelease.RestoreDispatchStatus, context string) error {
|
||||
return fmt.Errorf("%w: %s(state=%s)", libraryrelease.ErrRestoreDispatchHeld, context, status.State)
|
||||
}
|
||||
|
||||
// admitRestorePlan 是 Restore 在**任何** PG/对象/索引写之前的纯读准入。
|
||||
//
|
||||
// 删除型恢复一旦开始写 recovery 的行/对象,后面才发现某个 effect 的派发结果未知,
|
||||
// 目标已经从"未动"变成"半恢复"。所以先按整份冻结计划逐 effect 判定:
|
||||
// - 台账不存在(旧世界):只有"没有任何恢复意图"才算首次恢复;有意图就 held;
|
||||
// - prepared:唯一可续作区段;
|
||||
// - dispatching:只有精确 marker+世代才可走确认分支,否则 held;
|
||||
// - marker_confirmed:只允许确认后的 clean 路径。
|
||||
func (t *ProductionLibraryTarget) admitRestorePlan(ctx context.Context) error {
|
||||
for _, c := range t.activePlan.Index {
|
||||
if len(c.After) != 0 {
|
||||
continue
|
||||
}
|
||||
body, err := t.restoreIndexClaimBody(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 已完成 clean 的 effect 是终态:恢复内容与证据都在时,重放不是未决。
|
||||
done, err := t.loadIndexEffect(c, "restored")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done != nil {
|
||||
observed, err := t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done.Receipt == nil || !indexBytesEqual(observed, c.Before) ||
|
||||
!sameIndexGeneration(observed, done.Receipt.SeqNo, done.Receipt.PrimaryTerm) {
|
||||
return fmt.Errorf("%w: 已标记 restored 的 effect 与目标现状不符", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.config.RestoreDispatch != nil {
|
||||
binding, bindErr := t.restoreMarkerBinding(c, body)
|
||||
if bindErr != nil {
|
||||
return bindErr
|
||||
}
|
||||
status, found, inspErr := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if inspErr != nil {
|
||||
return fmt.Errorf("恢复准入读不到派发台账:%w", inspErr)
|
||||
}
|
||||
if found {
|
||||
switch status.State {
|
||||
case libraryrelease.RestoreDispatchPrepared:
|
||||
continue
|
||||
case libraryrelease.RestoreDispatchDispatching, libraryrelease.RestoreDispatchMarkerConfirmed:
|
||||
observed, obsErr := t.observePlannedIndex(ctx, c)
|
||||
if obsErr != nil {
|
||||
return obsErr
|
||||
}
|
||||
if len(c.After) == 0 && indexBytesEqual(observed, body) && observed.SeqNo != nil && observed.PrimaryTerm != nil {
|
||||
continue // 只允许原确认分支继续,不重发 create
|
||||
}
|
||||
return heldRestoreMarker(status, "目标上缺少可归属的精确恢复标记")
|
||||
default:
|
||||
return heldRestoreMarker(status, "派发台账状态不可识别")
|
||||
}
|
||||
}
|
||||
}
|
||||
if intent == nil {
|
||||
continue
|
||||
}
|
||||
observed, err := t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if indexBytesEqual(observed, body) {
|
||||
continue // 旧标记已在目标上:按既有确认/CAS 规则处理,不派发新 create
|
||||
}
|
||||
return fmt.Errorf("%w: 既存恢复意图没有新协议派发台账,无法证明未派发", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// confirmDispatchedMarker 只在"marker 与冻结内容逐位相同且世代齐全"时记录回执。
|
||||
// 它不发送任何请求;确认失败即 held,绝不因为"看起来已经写好了"就继续 clean。
|
||||
func (t *ProductionLibraryTarget) confirmDispatchedMarker(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, observed indexWriteGuard) error {
|
||||
if t.config.RestoreDispatch == nil || len(c.After) != 0 {
|
||||
return nil
|
||||
}
|
||||
binding, err := t.restoreMarkerBinding(c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%w: 删除型恢复缺少派发台账,拒绝按外观继续", libraryrelease.ErrRestoreDispatchHeld)
|
||||
}
|
||||
switch status.State {
|
||||
case libraryrelease.RestoreDispatchPrepared:
|
||||
// 还没派发:由调用方按 prepared 竞争许可。
|
||||
return nil
|
||||
case libraryrelease.RestoreDispatchMarkerConfirmed:
|
||||
return nil
|
||||
case libraryrelease.RestoreDispatchDispatching:
|
||||
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
|
||||
return heldRestoreMarker(status, "dispatching 结果未知且目标上没有精确标记")
|
||||
}
|
||||
receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: "created", SeqNo: observed.SeqNo, PrimaryTerm: observed.PrimaryTerm}
|
||||
raw, err := json.Marshal(receipt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw)
|
||||
default:
|
||||
return heldRestoreMarker(status, "派发台账状态不可识别")
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchRestoreMarkerCreate 是恢复期唯一获得发送许可的入口:preflight 与故障钩子
|
||||
// 都在 CAS 之前,Acquire 只在 dispatching 明确提交后返回许可。
|
||||
func (t *ProductionLibraryTarget) dispatchRestoreMarkerCreate(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage) (indexReceipt, error) {
|
||||
if t.config.RestoreDispatch == nil {
|
||||
return indexReceipt{}, errors.New("恢复标记派发协议未接线:拒绝发送")
|
||||
}
|
||||
binding, err := t.restoreMarkerBinding(c, body)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
if !found || status.State != libraryrelease.RestoreDispatchPrepared {
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
return indexReceipt{}, heldRestoreMarker(status, "没有可竞争的 prepared 许可")
|
||||
}
|
||||
// 控制流故障注入必须在 CAS 之前:拒绝时状态仍是 prepared,可安全重试,零转发。
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
permit, acquireStatus, err := t.config.RestoreDispatch.AcquireRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
if permit == nil {
|
||||
return indexReceipt{}, heldRestoreMarker(acquireStatus, "许可竞争未胜出")
|
||||
}
|
||||
receipt, err := t.sendRestoreMarkerOnce(ctx, c, body, binding, permit)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
raw, err := json.Marshal(receipt)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
// 回执来自这次唯一发送:必须落进台账才算确认。写失败时 marker 已在目标上,
|
||||
// 下一进程走"精确 marker+世代"确认分支,不会重发 create。
|
||||
if err := t.config.RestoreDispatch.ConfirmRestoreMarker(ctx, binding, raw); err != nil {
|
||||
return indexReceipt{}, fmt.Errorf("恢复标记已发送但确认未落库:%w", err)
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
// sendRestoreMarkerOnce 是唯一的真实网络发送点:许可消费一次、请求不可被传输层重放。
|
||||
func (t *ProductionLibraryTarget) sendRestoreMarkerOnce(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage,
|
||||
binding libraryrelease.RestoreMarkerBinding, permit *libraryrelease.RestoreSendPermit) (indexReceipt, error) {
|
||||
digest, err := binding.Digest()
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
if err := permit.Consume(binding.Key, digest); err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
method, path, payload, err := t.indexMutation(c, body, nil, nil, true)
|
||||
if err != nil {
|
||||
return indexReceipt{}, err
|
||||
}
|
||||
var receipt indexReceipt
|
||||
if err := t.search.requestJSONOnce(ctx, method, path, payload, &receipt); err != nil {
|
||||
return receipt, fmt.Errorf("restore marker outcome unknown: %w", err)
|
||||
}
|
||||
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
|
||||
return receipt, err
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
// verifyRestoreEffectsClosed 是 Restore 收尾前的最后一道检查:任何一个删除型 effect
|
||||
// 只要留下恢复意图,就必须已经完成 clean(restored 证据),且台账不得停在 dispatching。
|
||||
func (t *ProductionLibraryTarget) verifyRestoreEffectsClosed(ctx context.Context) error {
|
||||
for _, c := range t.activePlan.Index {
|
||||
if len(c.After) != 0 {
|
||||
continue
|
||||
}
|
||||
intent, err := t.loadIndexEffect(c, "restore-mark-intent")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if intent == nil {
|
||||
continue
|
||||
}
|
||||
done, err := t.loadIndexEffect(c, "restored")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done == nil {
|
||||
return fmt.Errorf("%w: 恢复收尾前仍有未决 effect %s", libraryrelease.ErrRestoreDispatchHeld, c.ID)
|
||||
}
|
||||
if t.config.RestoreDispatch == nil {
|
||||
continue
|
||||
}
|
||||
body, err := t.restoreIndexClaimBody(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
binding, err := t.restoreMarkerBinding(c, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, found, err := t.config.RestoreDispatch.InspectRestoreMarker(ctx, binding)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found && status.State == libraryrelease.RestoreDispatchDispatching {
|
||||
return heldRestoreMarker(status, "收尾前台账仍停在 dispatching")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -57,6 +57,18 @@ type ProductionLibraryConfig struct {
|
||||
MaxDocuments int
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
// 以下字段只用于 DS01 恢复标记派发协议:RestoreDispatch 为 nil 时该能力整体
|
||||
// fail closed(既不对旧证据放行,也不产生新证据)。身份来自本次真实 attempt
|
||||
// 与原作业/计划绑定,不能由调用方临时编造。
|
||||
RestoreDispatch libraryrelease.RestoreMarkerGate
|
||||
RestoreTargetIdentity string
|
||||
RestoreScope string
|
||||
RestoreOperationID string
|
||||
RestoreWindowJobID string
|
||||
RestoreAttemptOwner string
|
||||
RestoreAttemptEpoch int64
|
||||
// RestoreSnapshotID 是本次恢复要用的那份预备份(与原 window/plan 绑定)。
|
||||
RestoreSnapshotID string
|
||||
}
|
||||
|
||||
func (config ProductionLibraryConfig) validate(scope ProductionScope) error {
|
||||
@@ -968,6 +980,11 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
|
||||
if err := ensureNoResourceGrants(ctx, t.pool, backup.WriteSet.CreatedIDs("resources")); err != nil {
|
||||
return err
|
||||
}
|
||||
// DS01:整份计划的未决派发准入必须在这里——在任何 PG/对象/索引写之前。
|
||||
// 否则会先回放 database/resources,最后才在 index 上说 held,目标已经半写。
|
||||
if err := t.admitRestorePlan(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
previous := t.preserveChunkSequence
|
||||
t.preserveChunkSequence = true
|
||||
defer func() { t.preserveChunkSequence = previous }()
|
||||
@@ -1008,5 +1025,10 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
|
||||
if diff := universeDiff(backup.WriteSet.Universe, after); diff != "" {
|
||||
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
|
||||
}
|
||||
// 收尾前再拒绝一次未决 effect:任何残留的 dispatching/未 clean 的删除型恢复
|
||||
// 都不允许被报成"已恢复",也不允许随后释放维护窗口。
|
||||
if err := t.verifyRestoreEffectsClosed(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -275,6 +275,18 @@ func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, erro
|
||||
}
|
||||
|
||||
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
|
||||
return exporter.requestJSONWithReplay(ctx, method, path, payload, result, true)
|
||||
}
|
||||
|
||||
// requestJSONOnce 关闭传输层对该请求的重放能力(GetBody=nil):恢复标记 create
|
||||
// 每个许可只能落地至多一次真实请求,不能让 keep-alive 重连把它悄悄发第二遍。
|
||||
// Go 的 Transport 只会自动重试"可重放"的请求;Body 非空且 GetBody 为 nil 时,
|
||||
// PUT/POST 不再满足该条件。
|
||||
func (exporter *SearchExporter) requestJSONOnce(ctx context.Context, method, path string, payload any, result any) error {
|
||||
return exporter.requestJSONWithReplay(ctx, method, path, payload, result, false)
|
||||
}
|
||||
|
||||
func (exporter *SearchExporter) requestJSONWithReplay(ctx context.Context, method, path string, payload any, result any, replayable bool) error {
|
||||
var body io.Reader
|
||||
if payload != nil {
|
||||
encoded, err := json.Marshal(payload)
|
||||
@@ -298,6 +310,9 @@ func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path st
|
||||
if payload != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if !replayable {
|
||||
request.GetBody = nil
|
||||
}
|
||||
if exporter.authorizationToken != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user