diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ec240..3a18df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 开发快照(恢复"确定没写进去"的判据改为可证明的栅栏/世代) + +### Fixed + +- 索引恢复不再把"文档缺失"当成"证明从未发送":恢复 marker 写成功、回执丢失、随后又被删除, + 与从未写过在索引里完全同形。删除型恢复写入是**独占 `_create`**,现在只有在第一次派发前 + 冻结的**索引写计数器栅栏**(索引 uuid + `index_total` + `delete_total`)逐位未变时才允许重做; + 栅栏推进、栅栏缺失或读取失败一律 held,不认领(B19 不放宽)。 +- `restore-finalize-intent` 已落盘、干净收尾写入在派发前被拒绝后,重试不再永久 held:标记体 + 仍停在自己那份世代(内容 == 冻结标记体,`_seq_no`/`_primary_term` == 意图回执)即证明干净 + 写入没有落地,按同一 CAS 继续收尾;内容或世代对不上仍然 held。 +- 删除补偿(`Before` 为空、恢复写入是 `DELETE`)在派发前被拒绝后同样可以续作:文档仍是 + applied 结果、世代仍是 applied 回执才重放 `DELETE`;文档已变或世代不符仍然 held。 +- 下一条旧快照里"删除即缺失、判据只有世代"的表述仅适用于更新型恢复;删除型以本条冻结的 + 索引写计数器栅栏为准。 + +### Changed + +- 对象验收证据改用 `ListObjectVersions`(键 + versionId + ETag/字节数/删除标记):`ListObjectsV2` + 只枚举当前可见对象,不能证明历史版本。 + ## 开发快照(索引恢复在"确定没写进去"时可按证据重做) ### Fixed diff --git a/internal/releaseadapter/v071/library_index_effects.go b/internal/releaseadapter/v071/library_index_effects.go index ad8c0a6..a0c14ad 100644 --- a/internal/releaseadapter/v071/library_index_effects.go +++ b/internal/releaseadapter/v071/library_index_effects.go @@ -29,6 +29,65 @@ type indexEffect struct { Plan string `json:"plan"` Change libraryrelease.IndexWrite `json:"change"` Receipt *indexReceipt `json:"receipt,omitempty"` + // Fence 是第一次恢复写入派发前冻结的索引写计数器(仅 restore-mark-intent 使用)。 + // 文档缺失本身不是"证明未发送":marker 写成功、回执丢失、随后被删除,与从未 + // 写过在索引里完全同形。只有计数器也没动,才证明那次写入确实没有落地。 + Fence *indexWriteFence `json:"fence,omitempty"` +} + +type indexWriteFence struct { + IndexUUID string `json:"index_uuid"` + Indexed int64 `json:"indexed"` + Deleted int64 `json:"deleted"` +} + +func (f indexWriteFence) equal(other indexWriteFence) bool { + return f.IndexUUID != "" && f.IndexUUID == other.IndexUUID && f.Indexed == other.Indexed && f.Deleted == other.Deleted +} + +// indexWriteFence 读取索引主分片的写计数器。任何成功的 index/create/update/delete +// 都会推进它们,所以"与派发前冻结的值完全相同"能证明这份恢复写入没有落地; +// 计数器推进(或读取失败/UUID 不符)一律不能证明,调用方必须 held。 +func (exporter *SearchExporter) indexWriteFence(ctx context.Context, index string) (indexWriteFence, error) { + var response struct { + Indices map[string]struct { + UUID string `json:"uuid"` + } `json:"indices"` + All struct { + Primaries struct { + Indexing struct { + IndexTotal *int64 `json:"index_total"` + DeleteTotal *int64 `json:"delete_total"` + } `json:"indexing"` + } `json:"primaries"` + } `json:"_all"` + } + if err := exporter.requestJSON(ctx, http.MethodGet, "/"+url.PathEscape(index)+"/_stats?level=indices", nil, &response); err != nil { + return indexWriteFence{}, err + } + meta, ok := response.Indices[index] + if !ok || meta.UUID == "" || response.All.Primaries.Indexing.IndexTotal == nil || response.All.Primaries.Indexing.DeleteTotal == nil { + return indexWriteFence{}, errors.New("index write fence missing") + } + return indexWriteFence{ + IndexUUID: meta.UUID, + Indexed: *response.All.Primaries.Indexing.IndexTotal, + Deleted: *response.All.Primaries.Indexing.DeleteTotal, + }, nil +} + +// beginRestoreMarker 把"索引写计数器栅栏"和恢复标记意图一起原子落盘:先读栅栏, +// 再写不可变证据,之后才允许派发第一次恢复写入。 +func (t *ProductionLibraryTarget) beginRestoreMarker(ctx context.Context, c libraryrelease.IndexWrite, applied indexReceipt) error { + fence, err := t.search.indexWriteFence(ctx, c.Index) + if err != nil { + return err + } + raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &applied, Fence: &fence}) + if err != nil { + return err + } + return persistImmutableJSON(t.indexEffectPath(c, "restore-mark-intent"), raw) } func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, phase string) string { @@ -36,7 +95,7 @@ func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, p return filepath.Join(t.config.ArchiveRoot, "effects", t.activePlan.Digest, "index", hex.EncodeToString(sum[:]), phase+".json") } func (t *ProductionLibraryTarget) saveIndexEffect(c libraryrelease.IndexWrite, phase string, r *indexReceipt) error { - raw, err := json.Marshal(indexEffect{t.activePlan.Digest, c, r}) + raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: r}) if err != nil { return err } @@ -260,7 +319,36 @@ func (t *ProductionLibraryTarget) restorePlannedIndex(ctx context.Context) error } continue } - return errors.New("unknown index compensation requires reconciliation") + // 删除型补偿:删除落地必然移除文档并推进世代;文档仍是 applied 结果、 + // 世代仍是 applied 回执,才证明那次 DELETE 没有落地,可按同一 CAS 重放。 + expected := c.After + if applied.Receipt.Marked { + expected, err = t.indexClaimBody(c) + if err != nil { + return err + } + } + if !indexBytesEqual(g, expected) || !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm) { + return errors.New("unknown index compensation requires reconciliation") + } + if err := t.refuseRestoreWrite(c); err != nil { + return err + } + receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, false) + if err != nil { + return err + } + if err = t.saveIndexEffect(c, "restored", &receipt); err != nil { + return err + } + g, err = t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.Before) { + return errors.New("index compensation readback failed") + } + continue } expected := c.After if applied.Receipt.Marked { diff --git a/internal/releaseadapter/v071/library_index_known_recovery_test.go b/internal/releaseadapter/v071/library_index_known_recovery_test.go index 3bf9fda..7d69801 100644 --- a/internal/releaseadapter/v071/library_index_known_recovery_test.go +++ b/internal/releaseadapter/v071/library_index_known_recovery_test.go @@ -10,13 +10,9 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" "testing" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "knowledge-governance/internal/libraryrelease" ) @@ -44,28 +40,6 @@ func (w *restoreMarkerWire) RoundTrip(r *http.Request) (*http.Response, error) { return w.inner.RoundTrip(r) } -// objectKeySet 是对象存储的精确集合证据:键 -> 字节长度/ETag。 -// 比"某个对象在不在"强:多一个残留键、少一个原有键、或原有对象被改写都会红。 -func objectKeySet(t *testing.T, fx *writeSetFixture) map[string]string { - t.Helper() - out := map[string]string{} - var token *string - for { - list, err := fx.objects.s3.client.ListObjectsV2(fx.ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(fx.bucket), ContinuationToken: token}) - if err != nil { - t.Fatalf("list objects: %v", err) - } - for _, object := range list.Contents { - out[aws.ToString(object.Key)] = aws.ToString(object.ETag) + "/" + strconv.FormatInt(aws.ToInt64(object.Size), 10) - } - if !aws.ToBool(list.IsTruncated) { - return out - } - token = list.NextContinuationToken - } -} - func indexChangeOf(t *testing.T, plan *libraryrelease.WriteSet, id string) libraryrelease.IndexWrite { t.Helper() if plan == nil { @@ -113,7 +87,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing. seedTargetData(t, fx) beforePG := universeDump(t, fx.pool) beforeIndex := recoveryIndexReadback(t, fx) - beforeObjects := objectKeySet(t, fx) + beforeObjects := objectVersionSet(t, fx) snapshot := sealSourceSnapshot(t, fx, nil) target := fx.target(t) @@ -216,7 +190,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing. } // 真实对象存储:精确键集合 + 每个对象字节身份回到预备份时刻。 - if afterObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) { + if afterObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) { t.Fatalf("object set differs from the before-image:\n before=%v\n after=%v", beforeObjects, afterObjects) } @@ -242,7 +216,7 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing. if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) { t.Fatal("replayed restoration rewrote index generations") } - if replayObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) { + if replayObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) { t.Fatal("replayed restoration changed the object set") } } diff --git a/internal/releaseadapter/v071/library_index_recovery_gaps_test.go b/internal/releaseadapter/v071/library_index_recovery_gaps_test.go new file mode 100644 index 0000000..c5b2766 --- /dev/null +++ b/internal/releaseadapter/v071/library_index_recovery_gaps_test.go @@ -0,0 +1,462 @@ +package v071 + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "os" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "knowledge-governance/internal/libraryrelease" +) + +// restorePhaseWire 只旁路计数,不改写请求/响应: +// - creates / createWithCAS:真正派发的 `_create`(独占 create)次数,以及其中带 +// if_seq_no 的次数。独占 create 是恢复"删除型"变更时唯一允许的重放操作: +// 目标已存在则 409,永远不覆盖别人的字节。 +// - clean:标记体之外的 PUT(干净收尾写入)。 +// - deletes:DELETE(删除补偿)。 +// - marker:带恢复标记前缀的 PUT。 +type restorePhaseWire struct { + inner http.RoundTripper + creates int + createWithCAS int + clean int + deletes int + marker int +} + +func (w *restorePhaseWire) RoundTrip(r *http.Request) (*http.Response, error) { + var raw []byte + if r.Body != nil { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + r.Body = io.NopCloser(bytes.NewReader(body)) + raw = body + } + create := strings.Contains(r.URL.Path, "/_create/") + doc := create || strings.Contains(r.URL.Path, "/_doc/") + switch { + case !doc: + // `_pit` 之类的控制面请求不是文档写入,不能算进派发次数。 + case r.Method == http.MethodDelete: + w.deletes++ + case r.Method == http.MethodPut && create: + w.creates++ + if r.URL.Query().Get("if_seq_no") != "" || r.URL.Query().Get("if_primary_term") != "" { + w.createWithCAS++ + } + if bytes.Contains(raw, []byte(indexRestoreClaimPrefix)) { + w.marker++ + } + case r.Method == http.MethodPut && bytes.Contains(raw, []byte(indexRestoreClaimPrefix)): + w.marker++ + case r.Method == http.MethodPut: + w.clean++ + } + return w.inner.RoundTrip(r) +} + +type wireCounts struct { + creates, createWithCAS, marker, clean, deletes int +} + +func (w *restorePhaseWire) counts() wireCounts { + return wireCounts{w.creates, w.createWithCAS, w.marker, w.clean, w.deletes} +} + +// since 返回基线之后的增量:Run 的 apply 阶段也会写索引,绝对计数不是恢复阶段的证据。 +func (w *restorePhaseWire) since(base wireCounts) wireCounts { + now := w.counts() + return wireCounts{ + creates: now.creates - base.creates, + createWithCAS: now.createWithCAS - base.createWithCAS, + marker: now.marker - base.marker, + clean: now.clean - base.clean, + deletes: now.deletes - base.deletes, + } +} + +// objectVersionSet 是对象存储的版本级精确证据:键 + versionId -> ETag/字节数/是否当前版本。 +// ListObjectsV2 只枚举"当前可见对象",不能证明历史版本;这里显式枚举版本与删除标记。 +func objectVersionSet(t *testing.T, fx *writeSetFixture) map[string]string { + t.Helper() + out := map[string]string{} + pages := s3.NewListObjectVersionsPaginator(fx.objects.s3.client, &s3.ListObjectVersionsInput{ + Bucket: aws.String(fx.bucket), + }) + for pages.HasMorePages() { + page, err := pages.NextPage(fx.ctx) + if err != nil { + t.Fatalf("list object versions: %v", err) + } + for _, version := range page.Versions { + out[aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)] = "v/" + aws.ToString(version.ETag) + "/" + strconv.FormatInt(aws.ToInt64(version.Size), 10) + } + for _, marker := range page.DeleteMarkers { + out[aws.ToString(marker.Key)+"@"+aws.ToString(marker.VersionId)] = "delete-marker/" + strconv.FormatBool(aws.ToBool(marker.IsLatest)) + } + } + return out +} + +// indexCounters 是测试侧独立读取的索引写计数器(不经过生产代码的类型), +// 用来核对落盘栅栏是否真的冻结了派发之前的值。 +type indexCounters struct { + IndexUUID string + Indexed int64 + Deleted int64 +} + +func readIndexCounters(t *testing.T, fx *writeSetFixture, index string) indexCounters { + t.Helper() + response, err := fx.client.Get(fx.searchBase + "/" + index + "/_stats?level=indices") + if err != nil { + t.Fatalf("read index counters: %v", err) + } + defer response.Body.Close() + var body struct { + Indices map[string]struct { + UUID string `json:"uuid"` + } `json:"indices"` + All struct { + Primaries struct { + Indexing struct { + IndexTotal int64 `json:"index_total"` + DeleteTotal int64 `json:"delete_total"` + } `json:"indexing"` + } `json:"primaries"` + } `json:"_all"` + } + if err := json.NewDecoder(response.Body).Decode(&body); err != nil { + t.Fatalf("decode index counters: %v", err) + } + meta, ok := body.Indices[index] + if !ok || meta.UUID == "" { + t.Fatalf("index %s has no uuid in stats", index) + } + return indexCounters{IndexUUID: meta.UUID, Indexed: body.All.Primaries.Indexing.IndexTotal, Deleted: body.All.Primaries.Indexing.DeleteTotal} +} + +// rawEffectKeys 读取不可变证据文件的顶层字段名:用来证明"判据所需证据真的落盘了", +// 而不是只看某个 Go 字段(字段不存在时测试仍然要红)。 +func rawEffectKeys(t *testing.T, fx *writeSetFixture, plan *libraryrelease.WriteSet, change libraryrelease.IndexWrite, phase string) map[string]json.RawMessage { + t.Helper() + raw, err := os.ReadFile(indexEffectFile(fx, plan, change, phase)) + if err != nil { + t.Fatalf("read %s evidence: %v", phase, err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatalf("decode %s evidence: %v", phase, err) + } + return fields +} + +// 反例:恢复 marker 已真实写入、回执丢失,随后 marker 被删除 —— 观测到的"缺失" +// 与"从未发送"在索引里完全同形。缺失本身不是"证明未发送",因此必须 held, +// 不能凭 !Found 直接重建(那等于把 CAS create 的不覆盖性质冒充成历史证明)。 +func TestContractRestoreMarkerWrittenThenDeletedIsNotProvablyUnsentIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshot := sealSourceSnapshot(t, fx, nil) + + target := fx.target(t) + backups := fx.backups(t, target) + client := *target.search.client + inner := client.Transport + if inner == nil { + inner = http.DefaultTransport + } + wire := &restorePhaseWire{inner: inner} + client.Transport = wire + target.search.client = &client + + target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error { + if c.ID != "chunk-target" { + return nil + } + return errors.New("injected restore boundary fault: dispatch refused before any index write") + } + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil) + if outcome.State != "rollback_failed" { + t.Fatalf("state = %q, want rollback_failed (held)", outcome.State) + } + plan := outcome.Backup.WriteSet + change := indexChangeOf(t, plan, "chunk-target") + marker, err := target.restoreIndexClaimBody(change) + if err != nil { + t.Fatal(err) + } + requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true) + + // marker 写入真实成功(服务器已回执),但调用方没收到:不保存 restore-claimed。 + written, err := target.mutatePlannedIndex(fx.ctx, change, marker, nil, nil, true) + if err != nil { + t.Fatalf("real marker create: %v", err) + } + // 随后 marker 被删除:现在观测为"缺失",与从未写入不可区分。 + if _, err := target.mutatePlannedIndex(fx.ctx, change, nil, written.SeqNo, written.PrimaryTerm, false); err != nil { + t.Fatalf("delete the marker after its lost response: %v", err) + } + observed, err := target.observePlannedIndex(fx.ctx, change) + if err != nil || observed.Found { + t.Fatalf("precondition: marker must be absent, observed=%+v err=%v", observed, err) + } + requireIndexEffect(t, fx, plan, change, "restore-claimed", false) + requireIndexEffect(t, fx, plan, change, "restored", false) + heldPG := universeDump(t, fx.pool) + heldIndex := recoveryIndexReadback(t, fx) + heldObjects := objectVersionSet(t, fx) + + // 撤掉故障后只允许"可证明未发送"的重做;这里历史已经发生,必须 held。 + target.restoreMarkerFault = nil + base := wire.counts() + if err := target.Restore(fx.ctx, outcome.Backup); err == nil { + t.Fatal("absence after a lost marker write is indistinguishable from never-sent; restoration must be held, not rebuilt") + } else if !strings.Contains(err.Error(), "held") { + t.Fatalf("held error must say held, got: %v", err) + } + + // 判据证据必须真的落盘:落盘前冻结的索引写计数器栅栏。 + fence := rawEffectKeys(t, fx, plan, change, "restore-mark-intent")["fence"] + if len(fence) == 0 || string(fence) == "null" { + t.Fatalf("restore-mark-intent must freeze the index write fence before dispatch, got %s", fence) + } + var frozen struct { + IndexUUID string `json:"index_uuid"` + Indexed int64 `json:"indexed"` + Deleted int64 `json:"deleted"` + } + if err := json.Unmarshal(fence, &frozen); err != nil || frozen.IndexUUID == "" { + t.Fatalf("frozen fence invalid: %s (%v)", fence, err) + } + current := readIndexCounters(t, fx, change.Index) + if frozen.IndexUUID == current.IndexUUID && frozen.Indexed == current.Indexed && frozen.Deleted == current.Deleted { + t.Fatalf("fence did not advance although the marker was created and deleted: %+v", current) + } + + // held 不许偷偷写入,也不许改动任何真实存储。 + if sent := wire.since(base); sent.creates != 0 || sent.marker != 0 || sent.clean != 0 || sent.deletes != 0 { + t.Fatalf("held restoration dispatched writes: %+v", sent) + } + if diff := diffUniverse(heldPG, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("held restoration changed PG: %s", diff) + } + if afterIndex := recoveryIndexReadback(t, fx); !reflect.DeepEqual(heldIndex, afterIndex) { + t.Fatal("held restoration changed the index") + } + if after := objectVersionSet(t, fx); !reflect.DeepEqual(heldObjects, after) { + t.Fatalf("held restoration changed object versions:\n before=%v\n after=%v", heldObjects, after) + } +} + +// 缺口 2a:restore-finalize-intent 已落盘、干净收尾写入在派发边界上被拒绝。 +// 重试必须能凭"标记体仍停在自己那份世代上"证明干净写入没有落地,并继续收尾; +// 不能因为 finalize-intent 存在就永久 held(那是同一状态的死路)。 +func TestContractKnownIndexRecoveryAfterCleanDispatchFaultIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + beforePG := universeDump(t, fx.pool) + beforeIndex := recoveryIndexReadback(t, fx) + beforeObjects := objectVersionSet(t, fx) + snapshot := sealSourceSnapshot(t, fx, nil) + + target := fx.target(t) + backups := fx.backups(t, target) + client := *target.search.client + inner := client.Transport + if inner == nil { + inner = http.DefaultTransport + } + wire := &restorePhaseWire{inner: inner} + client.Transport = wire + target.search.client = &client + + refusals := 0 + var base wireCounts + target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error { + if c.ID != "chunk-target" { + return nil + } + refusals++ + if refusals == 1 { + // 标记写入派发前的基线:之后的增量只属于恢复阶段。 + base = wire.counts() + return nil + } + if refusals == 2 { + return errors.New("injected clean dispatch fault: refused before any clean write") + } + return nil + } + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil) + if outcome.State != "rollback_failed" { + t.Fatalf("state = %q (rollback=%q error=%q), want rollback_failed", outcome.State, outcome.Rollback, outcome.Error) + } + if refusals != 2 { + t.Fatalf("refusals = %d, want marker allowed + clean refused = 2", refusals) + } + plan := outcome.Backup.WriteSet + change := indexChangeOf(t, plan, "chunk-target") + marker, err := target.restoreIndexClaimBody(change) + if err != nil { + t.Fatal(err) + } + requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true) + requireIndexEffect(t, fx, plan, change, "restore-claimed", true) + requireIndexEffect(t, fx, plan, change, "restore-finalize-intent", true) + requireIndexEffect(t, fx, plan, change, "restored", false) + first := wire.since(base) + if first.marker != 1 || first.creates != 1 || first.createWithCAS != 0 || first.clean != 0 || first.deletes != 0 { + t.Fatalf("first attempt dispatched %+v, want exactly one exclusive marker create and no clean write", first) + } + held, err := target.observePlannedIndex(fx.ctx, change) + if err != nil || !indexBytesEqual(held, marker) { + t.Fatalf("held index must still carry the marker body: %v", err) + } + + // 撤故障后:标记体 + 它自己的世代证明干净写入从未落地,必须继续而不是死路。 + target.restoreMarkerFault = nil + resumeBase := wire.counts() + if err := target.Restore(fx.ctx, outcome.Backup); err != nil { + t.Fatalf("clean dispatch refusal must be resumable from its own marker generation, got: %v", err) + } + if resumed := wire.since(resumeBase); resumed.marker != 0 || resumed.creates != 0 || resumed.clean != 1 || resumed.deletes != 0 { + t.Fatalf("resume dispatched %+v, want exactly one clean write and no marker rewrite", resumed) + } + + if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("post-recovery PG differs from the before-image: %s", diff) + } + if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { + t.Fatalf("post-recovery verification: %v", err) + } + afterIndex := recoveryIndexReadback(t, fx) + if len(afterIndex) != len(beforeIndex) { + t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex)) + } + for id, old := range beforeIndex { + got, ok := afterIndex[id] + if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) { + t.Fatalf("index document %s not restored to its before-image", id) + } + if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) { + t.Fatal("restoration reused the applied generation instead of writing a new one") + } + } + if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) { + t.Fatalf("object versions differ from the before-image:\n before=%v\n after=%v", beforeObjects, after) + } + restored, err := target.loadIndexEffect(change, "restored") + if err != nil || restored == nil || restored.Receipt == nil { + t.Fatalf("restored receipt missing: %v", err) + } + live, err := target.observePlannedIndex(fx.ctx, change) + if err != nil || !sameIndexGeneration(live, restored.Receipt.SeqNo, restored.Receipt.PrimaryTerm) || !indexBytesEqual(live, change.Before) { + t.Fatalf("restored receipt does not match the live before-image: %v", err) + } + + // 重放同一份预备份:不得再派发任何恢复写入。 + replayBase := wire.counts() + next := fx.target(t) + if err := next.Restore(fx.ctx, outcome.Backup); err != nil { + t.Fatalf("replayed restoration: %v", err) + } + if replayed := wire.since(replayBase); replayed.creates != 0 || replayed.marker != 0 || replayed.clean != 0 || replayed.deletes != 0 { + t.Fatalf("replay dispatched recovery writes: %+v", replayed) + } + if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) { + t.Fatal("replayed restoration rewrote index generations") + } +} + +// 缺口 2b:删除补偿(Before 为空、恢复写入是 DELETE)在派发边界上被拒绝。 +// restore-intent 已落盘,重试必须凭"文档仍是 applied 结果、世代仍是 applied 回执"证明 +// 删除没有落地并重放 DELETE;不能退化成永久 "requires reconciliation"。 +func TestContractKnownIndexRecoveryAfterDeleteCompensationFaultIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + beforePG := universeDump(t, fx.pool) + beforeIndex := recoveryIndexReadback(t, fx) + beforeObjects := objectVersionSet(t, fx) + snapshot := sealSourceSnapshot(t, fx, nil) + + target := fx.target(t) + backups := fx.backups(t, target) + client := *target.search.client + inner := client.Transport + if inner == nil { + inner = http.DefaultTransport + } + wire := &restorePhaseWire{inner: inner} + client.Transport = wire + target.search.client = &client + + var base wireCounts + target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error { + if c.ID != "chunk-src" { + return nil + } + base = wire.counts() + return errors.New("injected delete compensation boundary fault") + } + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil) + if outcome.State != "rollback_failed" { + t.Fatalf("state = %q (rollback=%q error=%q), want rollback_failed", outcome.State, outcome.Rollback, outcome.Error) + } + plan := outcome.Backup.WriteSet + change := indexChangeOf(t, plan, "chunk-src") + if len(change.Before) != 0 || len(change.After) == 0 { + t.Fatalf("fixture change is not a created document: %+v", change) + } + requireIndexEffect(t, fx, plan, change, "restore-intent", true) + requireIndexEffect(t, fx, plan, change, "restored", false) + if refused := wire.since(base); refused.deletes != 0 || refused.clean != 0 || refused.creates != 0 { + t.Fatalf("boundary fault dispatched %+v; refusal must send nothing", refused) + } + if _, ok := recoveryIndexReadback(t, fx)["chunk-src"]; !ok { + t.Fatal("precondition: the created document must still be present after the refused delete") + } + + target.restoreMarkerFault = nil + resumeBase := wire.counts() + if err := target.Restore(fx.ctx, outcome.Backup); err != nil { + t.Fatalf("refused delete compensation must resume from the applied generation, got: %v", err) + } + if resumed := wire.since(resumeBase); resumed.deletes != 1 || resumed.marker != 1 || resumed.clean != 1 { + t.Fatalf("resume dispatched %+v, want one delete compensation + one marker create + one clean write", resumed) + } + if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("post-recovery PG differs from the before-image: %s", diff) + } + if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { + t.Fatalf("post-recovery verification: %v", err) + } + afterIndex := recoveryIndexReadback(t, fx) + if len(afterIndex) != len(beforeIndex) { + t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex)) + } + for id, old := range beforeIndex { + got, ok := afterIndex[id] + if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) { + t.Fatalf("index document %s not restored to its before-image", id) + } + if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) { + t.Fatal("restoration reused the applied generation instead of writing a new one") + } + } + if after := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, after) { + t.Fatalf("object versions differ from the before-image:\n before=%v\n after=%v", beforeObjects, after) + } +} diff --git a/internal/releaseadapter/v071/library_index_restore_claim.go b/internal/releaseadapter/v071/library_index_restore_claim.go index 0ef5390..268b148 100644 --- a/internal/releaseadapter/v071/library_index_restore_claim.go +++ b/internal/releaseadapter/v071/library_index_restore_claim.go @@ -11,15 +11,25 @@ import ( const indexRestoreClaimPrefix = "kg-release-transient-restore-claim:" -// restorationProvablyUnsent 判定"这份恢复意图一次都没到达索引"。 +// restorationProvablyUnsent 判定"这份恢复意图确实没有到达索引"。 // -// 判据只有世代:观察到的内容必须仍是我们确认过的 applied 结果(删除即缺失, -// 否则字节与 applied 回执一致),且世代与那份回执完全相同。成功写入必然产生新 -// 世代,所以世代没变就证明恢复写入没有落地——按证据重做不会覆盖别人的字节。 -// 反过来,任何"内容看起来对、世代对不上"的状态都只能 held(B19),不认领。 -func (t *ProductionLibraryTarget) restorationProvablyUnsent(observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt) (bool, error) { +// 判据不是"文档缺失":marker 写成功、回执丢失、随后又被删除,与从未写过在索引里 +// 完全同形,缺失本身不可归属。可用的证据只有"每成功写一次必然会变的东西": +// - 更新型:观察到的内容必须仍是 applied 结果,且世代与 applied 回执完全相同; +// - 删除型(恢复写入是独占 create):文档仍缺失之外,还必须与派发前冻结的索引 +// 写计数器栅栏逐位相同。计数器只在成功写入时推进,没动才证明没写过。 +// +// 任何"内容看起来对但世代/计数器对不上"的状态都只能 held(B19),不认领。 +func (t *ProductionLibraryTarget) restorationProvablyUnsent(ctx context.Context, observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt, intent *indexEffect) (bool, error) { if len(c.After) == 0 { - return !observed.Found, nil + if observed.Found || intent == nil || intent.Fence == nil { + return false, nil + } + current, err := t.search.indexWriteFence(ctx, c.Index) + if err != nil { + return false, err + } + return intent.Fence.equal(current), nil } if !sameIndexGeneration(observed, applied.SeqNo, applied.PrimaryTerm) { return false, nil @@ -76,7 +86,10 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if !start { return errors.New("legacy unknown index restoration lacks attributable marker intent") } - if err = t.saveIndexEffect(c, "restore-mark-intent", &applied); err != nil { + if err = t.beginRestoreMarker(ctx, c, applied); err != nil { + return err + } + if markIntent, err = t.loadIndexEffect(c, "restore-mark-intent"); err != nil { return err } } @@ -84,19 +97,26 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if err != nil { return err } - if finalizing != nil { - return errors.New("unknown clean restoration cannot be attributed by appearance; held") - } observed, err := t.observePlannedIndex(ctx, c) if err != nil { return err } + if finalizing != nil { + // 干净写入在派发边界上被拒绝时,标记体会原样停在自己那份世代上;落地过一次 + // 干净写入必然换掉内容并推进世代。两条同时成立才证明没写过,否则 held。 + if finalizing.Receipt == nil || !indexBytesEqual(observed, body) || !sameIndexGeneration(observed, finalizing.Receipt.SeqNo, finalizing.Receipt.PrimaryTerm) { + return errors.New("unknown clean restoration cannot be attributed by appearance; held") + } + return t.finishRestoreCleanWrite(ctx, c, *finalizing.Receipt) + } if !indexBytesEqual(observed, body) { // The marker is absent: either the intent never reached the index (a // deterministic refusal before dispatch) or its write outcome is unknown. - // Only an unchanged owned generation separates the two; an unknown write - // stays held (B19) and is never adopted by appearance. - unsent, err := t.restorationProvablyUnsent(observed, c, applied) + // Only evidence that every successful write would move — the owned + // generation for updates, the frozen index write fence for the exclusive + // create — separates the two; an unknown write stays held (B19) and is + // never adopted by appearance. + unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, markIntent) if err != nil { return err } @@ -139,17 +159,24 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if err = t.saveIndexEffect(c, "restore-finalize-intent", claimed.Receipt); err != nil { return err } + return t.finishRestoreCleanWrite(ctx, c, *claimed.Receipt) +} + +// finishRestoreCleanWrite 派发"标记体 -> 冻结 before-image"的 CAS 收尾,并把回执与 +// 实时观察逐位核对。finalize-intent 存在时,只有标记体仍停在它自己的世代上才会 +// 走到这里:落地过干净写入必然换掉内容并推进世代。 +func (t *ProductionLibraryTarget) finishRestoreCleanWrite(ctx context.Context, c libraryrelease.IndexWrite, marker indexReceipt) error { if err := t.refuseRestoreWrite(c); err != nil { return err } - receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm, false) + receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, marker.SeqNo, marker.PrimaryTerm, false) if err != nil { return err } if err = t.saveIndexEffect(c, "restored", &receipt); err != nil { return err } - observed, err = t.observePlannedIndex(ctx, c) + observed, err := t.observePlannedIndex(ctx, c) if err != nil { return err }