fix(release): 索引恢复用写计数器栅栏与世代证明续作
build / build (push) Successful in 1m31s

- 删除型恢复写入(独占 _create)不再把"文档缺失"当成"证明从未发送":
  marker 写成功、回执丢失、随后被删除与从未写过完全同形。现在在第一次
  派发前把索引写计数器(uuid/index_total/delete_total)与意图一起原子
  落盘;只有栅栏逐位未变才允许重做,推进/缺失/读取失败一律 held。
- restore-finalize-intent 已落盘、干净写入在派发前被拒绝后,标记体仍在
  自己那份世代即可证明干净写入没有落地,按同一 CAS 继续收尾。
- 删除补偿(Before 为空)在派发前被拒绝后可按 applied 世代重放 DELETE;
  文档已变或世代不符仍 held。
- 对象验收证据改用 ListObjectVersions,不再用 ListObjectsV2 冒充版本枚举。

真实 PG/ES/MinIO 隔离夹具:3 个新用例先 RED 后 GREEN;索引/恢复回归 12 项、
回滚回归 2 项全绿;make verify 16 项 success(identity 仅 CI 可核)。
This commit is contained in:
姚章浩
2026-11-02 00:49:30 +08:00
parent de867078c6
commit ff257064c3
5 changed files with 619 additions and 47 deletions
+21
View File
@@ -1,5 +1,26 @@
# Changelog
## 开发快照(恢复"确定没写进去"的判据改为可证明的栅栏/世代)
### Fixed
- 索引恢复不再把"文档缺失"当成"证明从未发送":恢复 marker 写成功、回执丢失、随后又被删除,
与从未写过在索引里完全同形。删除型恢复写入是**独占 `_create`**,现在只有在第一次派发前
冻结的**索引写计数器栅栏**(索引 uuid + `index_total` + `delete_total`)逐位未变时才允许重做;
栅栏推进、栅栏缺失或读取失败一律 held,不认领(B19 不放宽)。
- `restore-finalize-intent` 已落盘、干净收尾写入在派发前被拒绝后,重试不再永久 held:标记体
仍停在自己那份世代(内容 == 冻结标记体,`_seq_no`/`_primary_term` == 意图回执)即证明干净
写入没有落地,按同一 CAS 继续收尾;内容或世代对不上仍然 held。
- 删除补偿(`Before` 为空、恢复写入是 `DELETE`)在派发前被拒绝后同样可以续作:文档仍是
applied 结果、世代仍是 applied 回执才重放 `DELETE`;文档已变或世代不符仍然 held。
- 下一条旧快照里"删除即缺失、判据只有世代"的表述仅适用于更新型恢复;删除型以本条冻结的
索引写计数器栅栏为准。
### Changed
- 对象验收证据改用 `ListObjectVersions`(键 + versionId + ETag/字节数/删除标记):`ListObjectsV2`
只枚举当前可见对象,不能证明历史版本。
## 开发快照(索引恢复在"确定没写进去"时可按证据重做)
### Fixed
@@ -29,6 +29,65 @@ type indexEffect struct {
Plan string `json:"plan"`
Change libraryrelease.IndexWrite `json:"change"`
Receipt *indexReceipt `json:"receipt,omitempty"`
// Fence 是第一次恢复写入派发前冻结的索引写计数器(仅 restore-mark-intent 使用)。
// 文档缺失本身不是"证明未发送":marker 写成功、回执丢失、随后被删除,与从未
// 写过在索引里完全同形。只有计数器也没动,才证明那次写入确实没有落地。
Fence *indexWriteFence `json:"fence,omitempty"`
}
type indexWriteFence struct {
IndexUUID string `json:"index_uuid"`
Indexed int64 `json:"indexed"`
Deleted int64 `json:"deleted"`
}
func (f indexWriteFence) equal(other indexWriteFence) bool {
return f.IndexUUID != "" && f.IndexUUID == other.IndexUUID && f.Indexed == other.Indexed && f.Deleted == other.Deleted
}
// indexWriteFence 读取索引主分片的写计数器。任何成功的 index/create/update/delete
// 都会推进它们,所以"与派发前冻结的值完全相同"能证明这份恢复写入没有落地;
// 计数器推进(或读取失败/UUID 不符)一律不能证明,调用方必须 held。
func (exporter *SearchExporter) indexWriteFence(ctx context.Context, index string) (indexWriteFence, error) {
var response struct {
Indices map[string]struct {
UUID string `json:"uuid"`
} `json:"indices"`
All struct {
Primaries struct {
Indexing struct {
IndexTotal *int64 `json:"index_total"`
DeleteTotal *int64 `json:"delete_total"`
} `json:"indexing"`
} `json:"primaries"`
} `json:"_all"`
}
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+url.PathEscape(index)+"/_stats?level=indices", nil, &response); err != nil {
return indexWriteFence{}, err
}
meta, ok := response.Indices[index]
if !ok || meta.UUID == "" || response.All.Primaries.Indexing.IndexTotal == nil || response.All.Primaries.Indexing.DeleteTotal == nil {
return indexWriteFence{}, errors.New("index write fence missing")
}
return indexWriteFence{
IndexUUID: meta.UUID,
Indexed: *response.All.Primaries.Indexing.IndexTotal,
Deleted: *response.All.Primaries.Indexing.DeleteTotal,
}, nil
}
// beginRestoreMarker 把"索引写计数器栅栏"和恢复标记意图一起原子落盘:先读栅栏,
// 再写不可变证据,之后才允许派发第一次恢复写入。
func (t *ProductionLibraryTarget) beginRestoreMarker(ctx context.Context, c libraryrelease.IndexWrite, applied indexReceipt) error {
fence, err := t.search.indexWriteFence(ctx, c.Index)
if err != nil {
return err
}
raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &applied, Fence: &fence})
if err != nil {
return err
}
return persistImmutableJSON(t.indexEffectPath(c, "restore-mark-intent"), raw)
}
func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, phase string) string {
@@ -36,7 +95,7 @@ func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, p
return filepath.Join(t.config.ArchiveRoot, "effects", t.activePlan.Digest, "index", hex.EncodeToString(sum[:]), phase+".json")
}
func (t *ProductionLibraryTarget) saveIndexEffect(c libraryrelease.IndexWrite, phase string, r *indexReceipt) error {
raw, err := json.Marshal(indexEffect{t.activePlan.Digest, c, r})
raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: r})
if err != nil {
return err
}
@@ -260,7 +319,36 @@ func (t *ProductionLibraryTarget) restorePlannedIndex(ctx context.Context) error
}
continue
}
return errors.New("unknown index compensation requires reconciliation")
// 删除型补偿:删除落地必然移除文档并推进世代;文档仍是 applied 结果、
// 世代仍是 applied 回执,才证明那次 DELETE 没有落地,可按同一 CAS 重放。
expected := c.After
if applied.Receipt.Marked {
expected, err = t.indexClaimBody(c)
if err != nil {
return err
}
}
if !indexBytesEqual(g, expected) || !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm) {
return errors.New("unknown index compensation requires reconciliation")
}
if err := t.refuseRestoreWrite(c); err != nil {
return err
}
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, false)
if err != nil {
return err
}
if err = t.saveIndexEffect(c, "restored", &receipt); err != nil {
return err
}
g, err = t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if !indexBytesEqual(g, c.Before) {
return errors.New("index compensation readback failed")
}
continue
}
expected := c.After
if applied.Receipt.Marked {
@@ -10,13 +10,9 @@ import (
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"knowledge-governance/internal/libraryrelease"
)
@@ -44,28 +40,6 @@ func (w *restoreMarkerWire) RoundTrip(r *http.Request) (*http.Response, error) {
return w.inner.RoundTrip(r)
}
// objectKeySet 是对象存储的精确集合证据:键 -> 字节长度/ETag。
// 比"某个对象在不在"强:多一个残留键、少一个原有键、或原有对象被改写都会红。
func objectKeySet(t *testing.T, fx *writeSetFixture) map[string]string {
t.Helper()
out := map[string]string{}
var token *string
for {
list, err := fx.objects.s3.client.ListObjectsV2(fx.ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(fx.bucket), ContinuationToken: token})
if err != nil {
t.Fatalf("list objects: %v", err)
}
for _, object := range list.Contents {
out[aws.ToString(object.Key)] = aws.ToString(object.ETag) + "/" + strconv.FormatInt(aws.ToInt64(object.Size), 10)
}
if !aws.ToBool(list.IsTruncated) {
return out
}
token = list.NextContinuationToken
}
}
func indexChangeOf(t *testing.T, plan *libraryrelease.WriteSet, id string) libraryrelease.IndexWrite {
t.Helper()
if plan == nil {
@@ -113,7 +87,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing.
seedTargetData(t, fx)
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectKeySet(t, fx)
beforeObjects := objectVersionSet(t, fx)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
@@ -216,7 +190,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing.
}
// 真实对象存储:精确键集合 + 每个对象字节身份回到预备份时刻。
if afterObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) {
if afterObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) {
t.Fatalf("object set differs from the before-image:\n before=%v\n after=%v", beforeObjects, afterObjects)
}
@@ -242,7 +216,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing.
if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) {
t.Fatal("replayed restoration rewrote index generations")
}
if replayObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) {
if replayObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) {
t.Fatal("replayed restoration changed the object set")
}
}
@@ -0,0 +1,462 @@
package v071
import (
"bytes"
"encoding/json"
"errors"
"io"
"net/http"
"os"
"reflect"
"strconv"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"knowledge-governance/internal/libraryrelease"
)
// restorePhaseWire 只旁路计数,不改写请求/响应:
// - creates / createWithCAS:真正派发的 `_create`(独占 create)次数,以及其中带
// if_seq_no 的次数。独占 create 是恢复"删除型"变更时唯一允许的重放操作:
// 目标已存在则 409,永远不覆盖别人的字节。
// - clean:标记体之外的 PUT(干净收尾写入)。
// - deletes:DELETE(删除补偿)。
// - marker:带恢复标记前缀的 PUT。
type restorePhaseWire struct {
inner http.RoundTripper
creates int
createWithCAS int
clean int
deletes int
marker int
}
func (w *restorePhaseWire) RoundTrip(r *http.Request) (*http.Response, error) {
var raw []byte
if r.Body != nil {
body, err := io.ReadAll(r.Body)
if err != nil {
return nil, err
}
r.Body = io.NopCloser(bytes.NewReader(body))
raw = body
}
create := strings.Contains(r.URL.Path, "/_create/")
doc := create || strings.Contains(r.URL.Path, "/_doc/")
switch {
case !doc:
// `_pit` 之类的控制面请求不是文档写入,不能算进派发次数。
case r.Method == http.MethodDelete:
w.deletes++
case r.Method == http.MethodPut && create:
w.creates++
if r.URL.Query().Get("if_seq_no") != "" || r.URL.Query().Get("if_primary_term") != "" {
w.createWithCAS++
}
if bytes.Contains(raw, []byte(indexRestoreClaimPrefix)) {
w.marker++
}
case r.Method == http.MethodPut && bytes.Contains(raw, []byte(indexRestoreClaimPrefix)):
w.marker++
case r.Method == http.MethodPut:
w.clean++
}
return w.inner.RoundTrip(r)
}
type wireCounts struct {
creates, createWithCAS, marker, clean, deletes int
}
func (w *restorePhaseWire) counts() wireCounts {
return wireCounts{w.creates, w.createWithCAS, w.marker, w.clean, w.deletes}
}
// since 返回基线之后的增量:Run 的 apply 阶段也会写索引,绝对计数不是恢复阶段的证据。
func (w *restorePhaseWire) since(base wireCounts) wireCounts {
now := w.counts()
return wireCounts{
creates: now.creates - base.creates,
createWithCAS: now.createWithCAS - base.createWithCAS,
marker: now.marker - base.marker,
clean: now.clean - base.clean,
deletes: now.deletes - base.deletes,
}
}
// objectVersionSet 是对象存储的版本级精确证据:键 + versionId -> ETag/字节数/是否当前版本。
// ListObjectsV2 只枚举"当前可见对象",不能证明历史版本;这里显式枚举版本与删除标记。
func objectVersionSet(t *testing.T, fx *writeSetFixture) map[string]string {
t.Helper()
out := map[string]string{}
pages := s3.NewListObjectVersionsPaginator(fx.objects.s3.client, &s3.ListObjectVersionsInput{
Bucket: aws.String(fx.bucket),
})
for pages.HasMorePages() {
page, err := pages.NextPage(fx.ctx)
if err != nil {
t.Fatalf("list object versions: %v", err)
}
for _, version := range page.Versions {
out[aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)] = "v/" + aws.ToString(version.ETag) + "/" + strconv.FormatInt(aws.ToInt64(version.Size), 10)
}
for _, marker := range page.DeleteMarkers {
out[aws.ToString(marker.Key)+"@"+aws.ToString(marker.VersionId)] = "delete-marker/" + strconv.FormatBool(aws.ToBool(marker.IsLatest))
}
}
return out
}
// indexCounters 是测试侧独立读取的索引写计数器(不经过生产代码的类型),
// 用来核对落盘栅栏是否真的冻结了派发之前的值。
type indexCounters struct {
IndexUUID string
Indexed int64
Deleted int64
}
func readIndexCounters(t *testing.T, fx *writeSetFixture, index string) indexCounters {
t.Helper()
response, err := fx.client.Get(fx.searchBase + "/" + index + "/_stats?level=indices")
if err != nil {
t.Fatalf("read index counters: %v", err)
}
defer response.Body.Close()
var body struct {
Indices map[string]struct {
UUID string `json:"uuid"`
} `json:"indices"`
All struct {
Primaries struct {
Indexing struct {
IndexTotal int64 `json:"index_total"`
DeleteTotal int64 `json:"delete_total"`
} `json:"indexing"`
} `json:"primaries"`
} `json:"_all"`
}
if err := json.NewDecoder(response.Body).Decode(&body); err != nil {
t.Fatalf("decode index counters: %v", err)
}
meta, ok := body.Indices[index]
if !ok || meta.UUID == "" {
t.Fatalf("index %s has no uuid in stats", index)
}
return indexCounters{IndexUUID: meta.UUID, Indexed: body.All.Primaries.Indexing.IndexTotal, Deleted: body.All.Primaries.Indexing.DeleteTotal}
}
// rawEffectKeys 读取不可变证据文件的顶层字段名:用来证明"判据所需证据真的落盘了",
// 而不是只看某个 Go 字段(字段不存在时测试仍然要红)。
func rawEffectKeys(t *testing.T, fx *writeSetFixture, plan *libraryrelease.WriteSet, change libraryrelease.IndexWrite, phase string) map[string]json.RawMessage {
t.Helper()
raw, err := os.ReadFile(indexEffectFile(fx, plan, change, phase))
if err != nil {
t.Fatalf("read %s evidence: %v", phase, err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(raw, &fields); err != nil {
t.Fatalf("decode %s evidence: %v", phase, err)
}
return fields
}
// 反例:恢复 marker 已真实写入、回执丢失,随后 marker 被删除 —— 观测到的"缺失"
// 与"从未发送"在索引里完全同形。缺失本身不是"证明未发送",因此必须 held,
// 不能凭 !Found 直接重建(那等于把 CAS create 的不覆盖性质冒充成历史证明)。
func TestContractRestoreMarkerWrittenThenDeletedIsNotProvablyUnsentIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
client := *target.search.client
inner := client.Transport
if inner == nil {
inner = http.DefaultTransport
}
wire := &restorePhaseWire{inner: inner}
client.Transport = wire
target.search.client = &client
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
if c.ID != "chunk-target" {
return nil
}
return errors.New("injected restore boundary fault: dispatch refused before any index write")
}
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
if outcome.State != "rollback_failed" {
t.Fatalf("state = %q, want rollback_failed (held)", outcome.State)
}
plan := outcome.Backup.WriteSet
change := indexChangeOf(t, plan, "chunk-target")
marker, err := target.restoreIndexClaimBody(change)
if err != nil {
t.Fatal(err)
}
requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true)
// marker 写入真实成功(服务器已回执),但调用方没收到:不保存 restore-claimed。
written, err := target.mutatePlannedIndex(fx.ctx, change, marker, nil, nil, true)
if err != nil {
t.Fatalf("real marker create: %v", err)
}
// 随后 marker 被删除:现在观测为"缺失",与从未写入不可区分。
if _, err := target.mutatePlannedIndex(fx.ctx, change, nil, written.SeqNo, written.PrimaryTerm, false); err != nil {
t.Fatalf("delete the marker after its lost response: %v", err)
}
observed, err := target.observePlannedIndex(fx.ctx, change)
if err != nil || observed.Found {
t.Fatalf("precondition: marker must be absent, observed=%+v err=%v", observed, err)
}
requireIndexEffect(t, fx, plan, change, "restore-claimed", false)
requireIndexEffect(t, fx, plan, change, "restored", false)
heldPG := universeDump(t, fx.pool)
heldIndex := recoveryIndexReadback(t, fx)
heldObjects := objectVersionSet(t, fx)
// 撤掉故障后只允许"可证明未发送"的重做;这里历史已经发生,必须 held。
target.restoreMarkerFault = nil
base := wire.counts()
if err := target.Restore(fx.ctx, outcome.Backup); err == nil {
t.Fatal("absence after a lost marker write is indistinguishable from never-sent; restoration must be held, not rebuilt")
} else if !strings.Contains(err.Error(), "held") {
t.Fatalf("held error must say held, got: %v", err)
}
// 判据证据必须真的落盘:落盘前冻结的索引写计数器栅栏。
fence := rawEffectKeys(t, fx, plan, change, "restore-mark-intent")["fence"]
if len(fence) == 0 || string(fence) == "null" {
t.Fatalf("restore-mark-intent must freeze the index write fence before dispatch, got %s", fence)
}
var frozen struct {
IndexUUID string `json:"index_uuid"`
Indexed int64 `json:"indexed"`
Deleted int64 `json:"deleted"`
}
if err := json.Unmarshal(fence, &frozen); err != nil || frozen.IndexUUID == "" {
t.Fatalf("frozen fence invalid: %s (%v)", fence, err)
}
current := readIndexCounters(t, fx, change.Index)
if frozen.IndexUUID == current.IndexUUID && frozen.Indexed == current.Indexed && frozen.Deleted == current.Deleted {
t.Fatalf("fence did not advance although the marker was created and deleted: %+v", current)
}
// held 不许偷偷写入,也不许改动任何真实存储。
if sent := wire.since(base); sent.creates != 0 || sent.marker != 0 || sent.clean != 0 || sent.deletes != 0 {
t.Fatalf("held restoration dispatched writes: %+v", sent)
}
if diff := diffUniverse(heldPG, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("held restoration changed PG: %s", diff)
}
if afterIndex := recoveryIndexReadback(t, fx); !reflect.DeepEqual(heldIndex, afterIndex) {
t.Fatal("held restoration changed the index")
}
if after := objectVersionSet(t, fx); !reflect.DeepEqual(heldObjects, after) {
t.Fatalf("held restoration changed object versions:\n before=%v\n after=%v", heldObjects, after)
}
}
// 缺口 2a:restore-finalize-intent 已落盘、干净收尾写入在派发边界上被拒绝。
// 重试必须能凭"标记体仍停在自己那份世代上"证明干净写入没有落地,并继续收尾;
// 不能因为 finalize-intent 存在就永久 held(那是同一状态的死路)。
func TestContractKnownIndexRecoveryAfterCleanDispatchFaultIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectVersionSet(t, fx)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
client := *target.search.client
inner := client.Transport
if inner == nil {
inner = http.DefaultTransport
}
wire := &restorePhaseWire{inner: inner}
client.Transport = wire
target.search.client = &client
refusals := 0
var base wireCounts
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
if c.ID != "chunk-target" {
return nil
}
refusals++
if refusals == 1 {
// 标记写入派发前的基线:之后的增量只属于恢复阶段。
base = wire.counts()
return nil
}
if refusals == 2 {
return errors.New("injected clean dispatch fault: refused before any clean write")
}
return nil
}
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
if outcome.State != "rollback_failed" {
t.Fatalf("state = %q (rollback=%q error=%q), want rollback_failed", outcome.State, outcome.Rollback, outcome.Error)
}
if refusals != 2 {
t.Fatalf("refusals = %d, want marker allowed + clean refused = 2", refusals)
}
plan := outcome.Backup.WriteSet
change := indexChangeOf(t, plan, "chunk-target")
marker, err := target.restoreIndexClaimBody(change)
if err != nil {
t.Fatal(err)
}
requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true)
requireIndexEffect(t, fx, plan, change, "restore-claimed", true)
requireIndexEffect(t, fx, plan, change, "restore-finalize-intent", true)
requireIndexEffect(t, fx, plan, change, "restored", false)
first := wire.since(base)
if first.marker != 1 || first.creates != 1 || first.createWithCAS != 0 || first.clean != 0 || first.deletes != 0 {
t.Fatalf("first attempt dispatched %+v, want exactly one exclusive marker create and no clean write", first)
}
held, err := target.observePlannedIndex(fx.ctx, change)
if err != nil || !indexBytesEqual(held, marker) {
t.Fatalf("held index must still carry the marker body: %v", err)
}
// 撤故障后:标记体 + 它自己的世代证明干净写入从未落地,必须继续而不是死路。
target.restoreMarkerFault = nil
resumeBase := wire.counts()
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("clean dispatch refusal must be resumable from its own marker generation, got: %v", err)
}
if resumed := wire.since(resumeBase); resumed.marker != 0 || resumed.creates != 0 || resumed.clean != 1 || resumed.deletes != 0 {
t.Fatalf("resume dispatched %+v, want exactly one clean write and no marker rewrite", resumed)
}
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("post-recovery PG differs from the before-image: %s", diff)
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("post-recovery verification: %v", err)
}
afterIndex := recoveryIndexReadback(t, fx)
if len(afterIndex) != len(beforeIndex) {
t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex))
}
for id, old := range beforeIndex {
got, ok := afterIndex[id]
if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) {
t.Fatalf("index document %s not restored to its before-image", id)
}
if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) {
t.Fatal("restoration reused the applied generation instead of writing a new one")
}
}
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
t.Fatalf("object versions differ from the before-image:\n before=%v\n after=%v", beforeObjects, after)
}
restored, err := target.loadIndexEffect(change, "restored")
if err != nil || restored == nil || restored.Receipt == nil {
t.Fatalf("restored receipt missing: %v", err)
}
live, err := target.observePlannedIndex(fx.ctx, change)
if err != nil || !sameIndexGeneration(live, restored.Receipt.SeqNo, restored.Receipt.PrimaryTerm) || !indexBytesEqual(live, change.Before) {
t.Fatalf("restored receipt does not match the live before-image: %v", err)
}
// 重放同一份预备份:不得再派发任何恢复写入。
replayBase := wire.counts()
next := fx.target(t)
if err := next.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("replayed restoration: %v", err)
}
if replayed := wire.since(replayBase); replayed.creates != 0 || replayed.marker != 0 || replayed.clean != 0 || replayed.deletes != 0 {
t.Fatalf("replay dispatched recovery writes: %+v", replayed)
}
if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) {
t.Fatal("replayed restoration rewrote index generations")
}
}
// 缺口 2b:删除补偿(Before 为空、恢复写入是 DELETE)在派发边界上被拒绝。
// restore-intent 已落盘,重试必须凭"文档仍是 applied 结果、世代仍是 applied 回执"证明
// 删除没有落地并重放 DELETE;不能退化成永久 "requires reconciliation"。
func TestContractKnownIndexRecoveryAfterDeleteCompensationFaultIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectVersionSet(t, fx)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
client := *target.search.client
inner := client.Transport
if inner == nil {
inner = http.DefaultTransport
}
wire := &restorePhaseWire{inner: inner}
client.Transport = wire
target.search.client = &client
var base wireCounts
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
if c.ID != "chunk-src" {
return nil
}
base = wire.counts()
return errors.New("injected delete compensation boundary fault")
}
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
if outcome.State != "rollback_failed" {
t.Fatalf("state = %q (rollback=%q error=%q), want rollback_failed", outcome.State, outcome.Rollback, outcome.Error)
}
plan := outcome.Backup.WriteSet
change := indexChangeOf(t, plan, "chunk-src")
if len(change.Before) != 0 || len(change.After) == 0 {
t.Fatalf("fixture change is not a created document: %+v", change)
}
requireIndexEffect(t, fx, plan, change, "restore-intent", true)
requireIndexEffect(t, fx, plan, change, "restored", false)
if refused := wire.since(base); refused.deletes != 0 || refused.clean != 0 || refused.creates != 0 {
t.Fatalf("boundary fault dispatched %+v; refusal must send nothing", refused)
}
if _, ok := recoveryIndexReadback(t, fx)["chunk-src"]; !ok {
t.Fatal("precondition: the created document must still be present after the refused delete")
}
target.restoreMarkerFault = nil
resumeBase := wire.counts()
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("refused delete compensation must resume from the applied generation, got: %v", err)
}
if resumed := wire.since(resumeBase); resumed.deletes != 1 || resumed.marker != 1 || resumed.clean != 1 {
t.Fatalf("resume dispatched %+v, want one delete compensation + one marker create + one clean write", resumed)
}
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("post-recovery PG differs from the before-image: %s", diff)
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("post-recovery verification: %v", err)
}
afterIndex := recoveryIndexReadback(t, fx)
if len(afterIndex) != len(beforeIndex) {
t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex))
}
for id, old := range beforeIndex {
got, ok := afterIndex[id]
if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) {
t.Fatalf("index document %s not restored to its before-image", id)
}
if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) {
t.Fatal("restoration reused the applied generation instead of writing a new one")
}
}
if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) {
t.Fatalf("object versions differ from the before-image:\n before=%v\n after=%v", beforeObjects, after)
}
}
@@ -11,15 +11,25 @@ import (
const indexRestoreClaimPrefix = "kg-release-transient-restore-claim:"
// restorationProvablyUnsent 判定"这份恢复意图一次都没到达索引"。
// restorationProvablyUnsent 判定"这份恢复意图确实没有到达索引"。
//
// 判据只有世代:观察到的内容必须仍是我们确认过的 applied 结果(删除即缺失,
// 否则字节与 applied 回执一致),且世代与那份回执完全相同。成功写入必然产生新
// 世代,所以世代没变就证明恢复写入没有落地——按证据重做不会覆盖别人的字节。
// 反过来,任何"内容看起来对、世代对不上"的状态都只能 held(B19),不认领。
func (t *ProductionLibraryTarget) restorationProvablyUnsent(observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt) (bool, error) {
// 判据不是"文档缺失":marker 写成功、回执丢失、随后又被删除,与从未写过在索引里
// 完全同形,缺失本身不可归属。可用的证据只有"每成功写一次必然会变的东西":
// - 更新型:观察到的内容必须仍是 applied 结果,且世代与 applied 回执完全相同;
// - 删除型(恢复写入是独占 create):文档仍缺失之外,还必须与派发前冻结的索引
// 写计数器栅栏逐位相同。计数器只在成功写入时推进,没动才证明没写过。
//
// 任何"内容看起来对但世代/计数器对不上"的状态都只能 held(B19),不认领。
func (t *ProductionLibraryTarget) restorationProvablyUnsent(ctx context.Context, observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt, intent *indexEffect) (bool, error) {
if len(c.After) == 0 {
return !observed.Found, nil
if observed.Found || intent == nil || intent.Fence == nil {
return false, nil
}
current, err := t.search.indexWriteFence(ctx, c.Index)
if err != nil {
return false, err
}
return intent.Fence.equal(current), nil
}
if !sameIndexGeneration(observed, applied.SeqNo, applied.PrimaryTerm) {
return false, nil
@@ -76,7 +86,10 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if !start {
return errors.New("legacy unknown index restoration lacks attributable marker intent")
}
if err = t.saveIndexEffect(c, "restore-mark-intent", &applied); err != nil {
if err = t.beginRestoreMarker(ctx, c, applied); err != nil {
return err
}
if markIntent, err = t.loadIndexEffect(c, "restore-mark-intent"); err != nil {
return err
}
}
@@ -84,19 +97,26 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if err != nil {
return err
}
if finalizing != nil {
return errors.New("unknown clean restoration cannot be attributed by appearance; held")
}
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}
if finalizing != nil {
// 干净写入在派发边界上被拒绝时,标记体会原样停在自己那份世代上;落地过一次
// 干净写入必然换掉内容并推进世代。两条同时成立才证明没写过,否则 held。
if finalizing.Receipt == nil || !indexBytesEqual(observed, body) || !sameIndexGeneration(observed, finalizing.Receipt.SeqNo, finalizing.Receipt.PrimaryTerm) {
return errors.New("unknown clean restoration cannot be attributed by appearance; held")
}
return t.finishRestoreCleanWrite(ctx, c, *finalizing.Receipt)
}
if !indexBytesEqual(observed, body) {
// The marker is absent: either the intent never reached the index (a
// deterministic refusal before dispatch) or its write outcome is unknown.
// Only an unchanged owned generation separates the two; an unknown write
// stays held (B19) and is never adopted by appearance.
unsent, err := t.restorationProvablyUnsent(observed, c, applied)
// Only evidence that every successful write would move — the owned
// generation for updates, the frozen index write fence for the exclusive
// create — separates the two; an unknown write stays held (B19) and is
// never adopted by appearance.
unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, markIntent)
if err != nil {
return err
}
@@ -139,17 +159,24 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
if err = t.saveIndexEffect(c, "restore-finalize-intent", claimed.Receipt); err != nil {
return err
}
return t.finishRestoreCleanWrite(ctx, c, *claimed.Receipt)
}
// finishRestoreCleanWrite 派发"标记体 -> 冻结 before-image"的 CAS 收尾,并把回执与
// 实时观察逐位核对。finalize-intent 存在时,只有标记体仍停在它自己的世代上才会
// 走到这里:落地过干净写入必然换掉内容并推进世代。
func (t *ProductionLibraryTarget) finishRestoreCleanWrite(ctx context.Context, c libraryrelease.IndexWrite, marker indexReceipt) error {
if err := t.refuseRestoreWrite(c); err != nil {
return err
}
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm, false)
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, marker.SeqNo, marker.PrimaryTerm, false)
if err != nil {
return err
}
if err = t.saveIndexEffect(c, "restored", &receipt); err != nil {
return err
}
observed, err = t.observePlannedIndex(ctx, c)
observed, err := t.observePlannedIndex(ctx, c)
if err != nil {
return err
}