From 066dc45a2563e924db49c98327f129c034c8667b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 2 Nov 2026 01:19:00 +0800 Subject: [PATCH] =?UTF-8?q?fix(release):=20=E5=88=A0=E9=99=A4=E5=9E=8B?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E6=81=A2=E5=A4=8D=E4=B8=8D=E5=86=8D=E7=94=A8?= =?UTF-8?q?=20ES=20=E7=BB=9F=E8=AE=A1=E8=AE=A1=E6=95=B0=E6=94=BE=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ES `_stats` 写计数是分片生命周期统计(官方文档:分片迁到另一节点时该分片统计清零), 索引 uuid 不变不绑定统计生命周期,清零后重新累计到同一数值(ABA)无法排除。 撤销 r36 的"写计数器栅栏"授权:既不落盘也不读取,统计仅作诊断。 - 删除型恢复(独占 `_create`)只有在本次尝试开始时不存在任何恢复意图、且文档确实缺失时 才可重做;既存 restore-mark-intent 配缺失文档一律 held、零写入。 - 保留可证明路径:marker 仍在且世代一致的 clean 收尾重试;文档仍是 applied 结果、 世代一致的 DELETE 补偿重试。 - 新增 ABA 反例(传输层 stats 故障模拟、真实 ES 文档写删、未重启共享 ES); B19 / clean / delete 安全正例全回归。 验证(真实 PG/ES/MinIO 隔离夹具): - ABA 反例:旧代码 RED(restoration must be held, not rebuilt),新代码 PASS - 索引/恢复回归 13 项 PASS - 回滚回归 2 项 PASS - make verify 16 项 success(identity 仅 CI 可核) 未部署、未写生产/云、未重启共享 ES。 --- CHANGELOG.md | 18 ++ docs/library-acceptance-matrix.md | 1 + .../v071/library_index_effects.go | 57 +---- .../v071/library_index_known_recovery_test.go | 97 +++----- .../v071/library_index_recovery_gaps_test.go | 208 ++++++++++++++++-- .../v071/library_index_restore_claim.go | 38 ++-- 6 files changed, 262 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a18df9..ace3a4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 开发快照(删除型索引恢复不再把 ES 统计计数器当证据) + +### Fixed + +- 撤销上一快照的"写计数器栅栏"授权:ES `_stats` 的写计数是**分片生命周期**统计,官方 + 文档明确写"分片迁到另一节点时该分片的统计会被清零";索引 uuid 不变并不绑定统计的 + 生命周期,清零后重新累计到与冻结值相同(ABA)无法排除。统计相等不是不可变证据, + 现在既不落盘也不读取,恢复路径里没有"凭计数放行写入"的分支。 +- 删除型索引恢复(恢复写入是独占 `_create`)只有在**本次尝试开始时不存在任何恢复意图** + 且文档确实缺失时才允许重做("先落意图、后派发"的不变量说明本作业没派发过)。既存的 + `restore-mark-intent` 配上缺失文档没有持久、不可重用的"没写进去"证据,一律 held、 + 零写入。该自动重试缺口已按现状记入验收矩阵,不宣称这是用户接受的能力收窄。 +- 保留可证明的路径:`restore-finalize-intent` 已落盘、真实 marker 仍停在自己那份世代上时 + 继续干净收尾;删除补偿(`Before` 为空)在文档仍是 applied 结果、世代仍是 applied + 回执时重放 `DELETE`。 +- 验收用例新增 ABA 反例:stats 一直返回同一 uuid 同一计数(传输层故障模拟),而真实 ES + 里 marker 已写入又被删除;恢复必须 held。没有重启共享 ES,也没有伪造文档观测。 + ## 开发快照(恢复"确定没写进去"的判据改为可证明的栅栏/世代) ### Fixed diff --git a/docs/library-acceptance-matrix.md b/docs/library-acceptance-matrix.md index af585b5..faac8d3 100644 --- a/docs/library-acceptance-matrix.md +++ b/docs/library-acceptance-matrix.md @@ -81,6 +81,7 @@ A06(内容、问答、引用图片效果、性能报告)已由用户取消 | ID | 缺口 | 现状与影响 | | --- | --- | --- | +| R02-b | 删除型索引恢复的自动重试 | 应用期删掉了目标文档、恢复写入是独占 `_create`。持久意图已落盘而文档缺失时,服务端没有可持久、不可重用的"这份写入没落地"证据:缺失与"从未写过"同形;ES `_stats` 写计数是分片生命周期统计(官方文档:分片迁到另一节点时该分片统计会被清零),索引 uuid 不变不绑定统计生命周期,清零后重新累计到同一数值(ABA)不能排除。**现状 fail-closed**:既存 `restore-mark-intent` 一律 held(`rollback_failed`、零写入、零误归属),不自动重试,需人工按预备份核验后重新规划;本次尝试开始前不存在任何恢复意图时仍可正常重建。这是**开发缺口**(r37 反例:同 uuid 同计数 + 真实写删,恢复仍须 held),不是用户接受的能力收窄;是否引入可校验的持久 not-dispatched 状态与派发原子协议,由后续设计决定。保留的可证明路径:真实 marker 仍在且世代一致的 clean 收尾重试、文档仍是 applied 结果且世代一致的 `DELETE` 补偿重试 | | R03 | 存快/恢复快对比 | A06 性能报告已由用户取消,不作为门槛。**已记实测耗时**(小测试库,不是压测):封存 15.8s(含停写~5s)、同步 13.6s(预备份 13.4s 占主要)、恢复 13.8s;证据 `13-r03-durations.json`。没有做高负载压测,也不写没有测量的数字 | | 死信解除·真实重放演练 | 没有可解除的真实死信,也没有源侧系统管理员凭据 | 契约与执行侧已实现(见下节新增行):上游保留原行、治理侧审计+指纹绑定、未结清/变化/新死信仍阻断。**本轮没有真实死信**(`dead_letters=0`),无法演练"真实重放成功→解除→放行"整条链路;上游只有 `系统管理 runtime run_now`(需要源侧管理员权限)与整队列 purge(销毁证据,不采用)。治理平台故意不持有源侧管理员凭据(放大授权),因此这一步**阻塞在真实死信 + 重放执行方**,不伪造成功 | diff --git a/internal/releaseadapter/v071/library_index_effects.go b/internal/releaseadapter/v071/library_index_effects.go index a0c14ad..dba1ac4 100644 --- a/internal/releaseadapter/v071/library_index_effects.go +++ b/internal/releaseadapter/v071/library_index_effects.go @@ -29,61 +29,14 @@ type indexEffect struct { Plan string `json:"plan"` Change libraryrelease.IndexWrite `json:"change"` Receipt *indexReceipt `json:"receipt,omitempty"` - // Fence 是第一次恢复写入派发前冻结的索引写计数器(仅 restore-mark-intent 使用)。 - // 文档缺失本身不是"证明未发送":marker 写成功、回执丢失、随后被删除,与从未 - // 写过在索引里完全同形。只有计数器也没动,才证明那次写入确实没有落地。 - Fence *indexWriteFence `json:"fence,omitempty"` } -type indexWriteFence struct { - IndexUUID string `json:"index_uuid"` - Indexed int64 `json:"indexed"` - Deleted int64 `json:"deleted"` -} - -func (f indexWriteFence) equal(other indexWriteFence) bool { - return f.IndexUUID != "" && f.IndexUUID == other.IndexUUID && f.Indexed == other.Indexed && f.Deleted == other.Deleted -} - -// indexWriteFence 读取索引主分片的写计数器。任何成功的 index/create/update/delete -// 都会推进它们,所以"与派发前冻结的值完全相同"能证明这份恢复写入没有落地; -// 计数器推进(或读取失败/UUID 不符)一律不能证明,调用方必须 held。 -func (exporter *SearchExporter) indexWriteFence(ctx context.Context, index string) (indexWriteFence, error) { - var response struct { - Indices map[string]struct { - UUID string `json:"uuid"` - } `json:"indices"` - All struct { - Primaries struct { - Indexing struct { - IndexTotal *int64 `json:"index_total"` - DeleteTotal *int64 `json:"delete_total"` - } `json:"indexing"` - } `json:"primaries"` - } `json:"_all"` - } - if err := exporter.requestJSON(ctx, http.MethodGet, "/"+url.PathEscape(index)+"/_stats?level=indices", nil, &response); err != nil { - return indexWriteFence{}, err - } - meta, ok := response.Indices[index] - if !ok || meta.UUID == "" || response.All.Primaries.Indexing.IndexTotal == nil || response.All.Primaries.Indexing.DeleteTotal == nil { - return indexWriteFence{}, errors.New("index write fence missing") - } - return indexWriteFence{ - IndexUUID: meta.UUID, - Indexed: *response.All.Primaries.Indexing.IndexTotal, - Deleted: *response.All.Primaries.Indexing.DeleteTotal, - }, nil -} - -// beginRestoreMarker 把"索引写计数器栅栏"和恢复标记意图一起原子落盘:先读栅栏, -// 再写不可变证据,之后才允许派发第一次恢复写入。 +// beginRestoreMarker 在第一次恢复写入派发之前,把恢复标记意图原子落盘。 +// 证据只描述"我们要写什么",不记录任何用来证明"没写过"的计数: +// ES 的写计数器是分片生命周期内的统计量(分片迁移/节点替换会清零), +// 不是不可变事务日志,不能作为放行写入的授权依据。 func (t *ProductionLibraryTarget) beginRestoreMarker(ctx context.Context, c libraryrelease.IndexWrite, applied indexReceipt) error { - fence, err := t.search.indexWriteFence(ctx, c.Index) - if err != nil { - return err - } - raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &applied, Fence: &fence}) + raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &applied}) if err != nil { return err } diff --git a/internal/releaseadapter/v071/library_index_known_recovery_test.go b/internal/releaseadapter/v071/library_index_known_recovery_test.go index 7d69801..30fcdba 100644 --- a/internal/releaseadapter/v071/library_index_known_recovery_test.go +++ b/internal/releaseadapter/v071/library_index_known_recovery_test.go @@ -73,21 +73,19 @@ func requireIndexEffect(t *testing.T, fx *writeSetFixture, plan *libraryrelease. return path } -// 目标:已知索引效果(真实回执已落盘)在 rollback 的恢复步骤上"确定失败"之后, -// 撤掉故障必须能把同一 job/预备份的原 before-image 恢复完成,且 PG / 对象 / 索引 -// 三个真实存储的集合精确回到预备份时刻。 +// r37:删除型恢复(Before 非空、After 为空,恢复写入是独占 `_create`)在持久意图 +// 落盘之后、派发之前"确定失败"。撤掉故障后**不能**自动重做:文档缺失与"从未写过" +// 在索引里完全同形,服务端没有可持久、不可重用的"没写进去"证据。测试侧的派发次数 +// 是内存观测,不是可复核的持久证据;ES `_stats` 写计数器是分片生命周期统计(分片 +// 迁移/节点替换会清零,见官方 `_stats` 文档),索引 uuid 不变不绑定统计生命周期, +// 清零后重新累计到同一数值(ABA)无法排除。因此本状态必须 held(B19 不放宽)。 // // 故障边界(target.restoreMarkerFault):在恢复标记的持久意图落盘之后、网络派发之前 -// 返回错误。这是**控制流在边界上拒绝**,不是断线,也不是响应丢失:真实 ES 一个字节都 -// 没收到(用例用派发次数 0 证明)。因此重试必须凭"内容仍等于 applied 结果 + 世代仍等于 -// 自己那份回执"证明第一次没写进去,然后安全重做;而不是把它当成 B19 的不可归属 -// unknown 永久 held(那种情况仍必须 held,本用例不放宽)。 -func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing.T) { +// 返回错误——真实 ES 一个字节都没收到(用例用派发次数 0 证明),但"证明"只存在于 +// 测试进程里,不能变成生产可用的授权。 +func TestContractRestoreMarkerAfterBoundaryFaultIsHeldIsolated(t *testing.T) { fx := openWriteSetFixture(t) seedTargetData(t, fx) - beforePG := universeDump(t, fx.pool) - beforeIndex := recoveryIndexReadback(t, fx) - beforeObjects := objectVersionSet(t, fx) snapshot := sealSourceSnapshot(t, fx, nil) target := fx.target(t) @@ -143,13 +141,16 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing. } requireIndexEffect(t, fx, plan, change, "apply-intent", true) requireIndexEffect(t, fx, plan, change, "applied", true) - // 恢复意图已落盘,但恢复标记从未派发:这正是"可归属、且证明没写过"的状态。 + // 恢复意图已落盘,但恢复标记在测试进程里从未派发。这条"没派发"只存在于 + // 内存计数里,不是生产可复核的持久证据。 requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true) requireIndexEffect(t, fx, plan, change, "restore-claimed", false) requireIndexEffect(t, fx, plan, change, "restored", false) // held 期间业务存储不许被"顺手恢复一半再报成功":索引仍是应用后的样子。 heldIndex := recoveryIndexReadback(t, fx) + heldPG := universeDump(t, fx.pool) + heldObjects := objectVersionSet(t, fx) if _, ok := heldIndex["chunk-src"]; ok { t.Fatal("index still carries the applied chunk-src after a failed index restoration") } @@ -157,66 +158,26 @@ func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing. t.Fatal("chunk-target came back without the restoration marker ever being dispatched") } - // 撤掉故障:同一 job / 同一预备份 / 同一 before-image 重新执行恢复。 + // 撤掉故障。删除型恢复写入是独占 `_create`,文档缺失与"从未写过"同形;服务端 + // 没有可持久、不可重用的未发送证据(ES 统计随分片迁移清零,不能当证据), + // 所以必须 held,不能凭测试侧"派发 0 次"放行写入。 target.restoreMarkerFault = nil - if err := target.Restore(fx.ctx, outcome.Backup); err != nil { - t.Fatalf("recovery after clearing the fault must settle the durable intent, got: %v", err) + beforeDispatch := wire.dispatch + if err := target.Restore(fx.ctx, outcome.Backup); err == nil { + t.Fatal("delete-type restoration without a durable not-dispatched proof must stay held") + } else if !strings.Contains(err.Error(), "held") { + t.Fatalf("held error must say held, got: %v", err) } - if wire.dispatch != 1 { - t.Fatalf("recovery dispatched %d restore marker writes, want exactly 1", wire.dispatch) + if wire.dispatch != beforeDispatch { + t.Fatalf("held restoration dispatched %d restore marker writes", wire.dispatch-beforeDispatch) } - - // 真实 PG:整实例逐行字节回到预备份时刻。 - if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" { - t.Fatalf("post-recovery PG differs from the before-image: %s", diff) + if diff := diffUniverse(heldPG, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("held restoration changed PG: %s", diff) } - if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { - t.Fatalf("post-recovery verification: %v", err) + if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(heldIndex, after) { + t.Fatal("held restoration changed the index") } - - // 真实索引:ID 集合精确一致、字节一致,恢复产生新世代(不是旧世代冒充)。 - afterIndex := recoveryIndexReadback(t, fx) - if len(afterIndex) != len(beforeIndex) { - t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex)) - } - for id, old := range beforeIndex { - got, ok := afterIndex[id] - if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) { - t.Fatalf("index document %s not restored to its before-image", id) - } - if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) { - t.Fatal("restoration reused the applied generation instead of writing a new one") - } - } - - // 真实对象存储:精确键集合 + 每个对象字节身份回到预备份时刻。 - if afterObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) { - t.Fatalf("object set differs from the before-image:\n before=%v\n after=%v", beforeObjects, afterObjects) - } - - // 恢复回执与观察到的世代逐位一致:证据不是"跑完了",而是可复核的服务器回执。 - restored, err := target.loadIndexEffect(change, "restored") - if err != nil || restored == nil || restored.Receipt == nil { - t.Fatalf("restored receipt missing: %v", err) - } - requireIndexEffect(t, fx, plan, change, "restore-claimed", true) - observed, err := target.observePlannedIndex(fx.ctx, change) - if err != nil || !sameIndexGeneration(observed, restored.Receipt.SeqNo, restored.Receipt.PrimaryTerm) { - t.Fatalf("restored receipt does not match the live generation: %v", err) - } - if !indexBytesEqual(observed, change.Before) { - t.Fatal("restored generation does not carry the frozen before-image bytes") - } - - // 新目标重放同一份预备份:不得重复派发恢复写入,也不得改动任何集合。 - next := fx.target(t) - if err := next.Restore(fx.ctx, outcome.Backup); err != nil { - t.Fatalf("replayed restoration: %v", err) - } - if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) { - t.Fatal("replayed restoration rewrote index generations") - } - if replayObjects := objectVersionSet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) { - t.Fatal("replayed restoration changed the object set") + if after := objectVersionSet(t, fx); !reflect.DeepEqual(heldObjects, after) { + t.Fatalf("held restoration changed object versions:\n before=%v\n after=%v", heldObjects, after) } } diff --git a/internal/releaseadapter/v071/library_index_recovery_gaps_test.go b/internal/releaseadapter/v071/library_index_recovery_gaps_test.go index c5b2766..8988f26 100644 --- a/internal/releaseadapter/v071/library_index_recovery_gaps_test.go +++ b/internal/releaseadapter/v071/library_index_recovery_gaps_test.go @@ -110,8 +110,8 @@ func objectVersionSet(t *testing.T, fx *writeSetFixture) map[string]string { return out } -// indexCounters 是测试侧独立读取的索引写计数器(不经过生产代码的类型), -// 用来核对落盘栅栏是否真的冻结了派发之前的值。 +// indexCounters 是测试侧独立读取的索引写计数器(不经过生产代码的类型)。 +// r37 起它只作**诊断**:证明真实写入确实推进了统计,而统计本身不是授权证据。 type indexCounters struct { IndexUUID string Indexed int64 @@ -228,22 +228,10 @@ func TestContractRestoreMarkerWrittenThenDeletedIsNotProvablyUnsentIsolated(t *t t.Fatalf("held error must say held, got: %v", err) } - // 判据证据必须真的落盘:落盘前冻结的索引写计数器栅栏。 - fence := rawEffectKeys(t, fx, plan, change, "restore-mark-intent")["fence"] - if len(fence) == 0 || string(fence) == "null" { - t.Fatalf("restore-mark-intent must freeze the index write fence before dispatch, got %s", fence) - } - var frozen struct { - IndexUUID string `json:"index_uuid"` - Indexed int64 `json:"indexed"` - Deleted int64 `json:"deleted"` - } - if err := json.Unmarshal(fence, &frozen); err != nil || frozen.IndexUUID == "" { - t.Fatalf("frozen fence invalid: %s (%v)", fence, err) - } - current := readIndexCounters(t, fx, change.Index) - if frozen.IndexUUID == current.IndexUUID && frozen.Indexed == current.Indexed && frozen.Deleted == current.Deleted { - t.Fatalf("fence did not advance although the marker was created and deleted: %+v", current) + // r37:ES 写计数器不是不可变证据(分片迁移/节点替换会清零,索引 uuid 不变不 + // 绑定统计生命周期),因此删除型恢复既不再落盘计数器栅栏,也不得据它放行写入。 + if _, ok := rawEffectKeys(t, fx, plan, change, "restore-mark-intent")["fence"]; ok { + t.Fatal("restore-mark-intent must not record a counter fence as authorization evidence") } // held 不许偷偷写入,也不许改动任何真实存储。 @@ -261,6 +249,190 @@ func TestContractRestoreMarkerWrittenThenDeletedIsNotProvablyUnsentIsolated(t *t } } +// resetStatsTransport 模拟 `_stats` 的分片迁移/统计清零后重新累计(ABA): +// +// 第一次 stats 响应原样转发并记住;此后所有 stats 请求都返回**这一份旧响应**。 +// +// 于是索引 uuid 与 (indexed, deleted) 与冻结时逐位相同,而真实索引里文档已经被 +// 写过又删掉。这是**传输层的 stats 故障模拟**:文档写入走真实 ES,没有重启共享 ES, +// 也没有伪造文档观测结果。 +type resetStatsTransport struct { + inner http.RoundTripper + frozen []byte + restores int + replayed int +} + +func (w *resetStatsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/_stats") { + w.restores++ + if w.frozen == nil { + response, err := w.inner.RoundTrip(r) + if err != nil { + return nil, err + } + body, err := io.ReadAll(response.Body) + response.Body.Close() + if err != nil { + return nil, err + } + w.frozen = body + response.Body = io.NopCloser(bytes.NewReader(body)) + return response, nil + } + w.replayed++ + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(w.frozen)), + Request: r, + }, nil + } + return w.inner.RoundTrip(r) +} + +func statsCountersOf(t *testing.T, body []byte, index string) indexCounters { + t.Helper() + var decoded struct { + Indices map[string]struct { + UUID string `json:"uuid"` + } `json:"indices"` + All struct { + Primaries struct { + Indexing struct { + IndexTotal int64 `json:"index_total"` + DeleteTotal int64 `json:"delete_total"` + } `json:"indexing"` + } `json:"primaries"` + } `json:"_all"` + } + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("decode frozen stats: %v (%s)", err, body) + } + meta, ok := decoded.Indices[index] + if !ok || meta.UUID == "" { + t.Fatalf("frozen stats has no uuid for %s: %s", index, body) + } + return indexCounters{IndexUUID: meta.UUID, Indexed: decoded.All.Primaries.Indexing.IndexTotal, Deleted: decoded.All.Primaries.Indexing.DeleteTotal} +} + +// r37 反例:分片迁移/节点替换会清掉分片级统计(官方 `_stats` 文档明确写了这一点), +// 索引 uuid 不变并不绑定统计生命周期;清零后再累计到与冻结值相同的情况(ABA)无法排除。 +// 因此"统计逐位相同"不是不可变证据:marker 真的写过又被删掉时,观测仍是缺失,必须 held。 +// +// 现场:先让恢复在派发前确定失败(restore-mark-intent 落盘),再**真实**写入 marker、 +// **真实**删除 marker;stats 端点则一直返回测试侧预先取得的那份响应(同 uuid 同计数), +// 模拟分片迁移清零后又累计回同一数值。旧实现据此判定"未发送"并重建; +// 本用例要求 held、零写入,且删除型恢复不得再读取统计。 +func TestContractRestoreMarkerAfterStatsResetABAIsHeldIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshot := sealSourceSnapshot(t, fx, nil) + + target := fx.target(t) + backups := fx.backups(t, target) + client := *target.search.client + inner := client.Transport + if inner == nil { + inner = http.DefaultTransport + } + wire := &restorePhaseWire{inner: inner} + // 测试侧自己先取一份统计快照作为"冻结值",之后 stats 端点一直返回它; + // 快照不依赖生产代码是否读取统计,避免用例被"实现先读一次"绑死。 + statsResponse, err := fx.client.Get(fx.searchBase + "/" + fx.index + "/_stats?level=indices") + if err != nil { + t.Fatalf("read stats snapshot: %v", err) + } + statsBody, err := io.ReadAll(statsResponse.Body) + statsResponse.Body.Close() + if err != nil { + t.Fatalf("read stats snapshot body: %v", err) + } + stats := &resetStatsTransport{inner: wire, frozen: statsBody} + client.Transport = stats + target.search.client = &client + + target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error { + if c.ID != "chunk-target" { + return nil + } + return errors.New("injected restore boundary fault: dispatch refused before any index write") + } + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil) + if outcome.State != "rollback_failed" { + t.Fatalf("state = %q, want rollback_failed (held)", outcome.State) + } + plan := outcome.Backup.WriteSet + change := indexChangeOf(t, plan, "chunk-target") + marker, err := target.restoreIndexClaimBody(change) + if err != nil { + t.Fatal(err) + } + requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true) + requireIndexEffect(t, fx, plan, change, "restore-claimed", false) + requireIndexEffect(t, fx, plan, change, "restored", false) + + // 恢复意图落盘(真实派发 0 次),随后 marker 真实写入、真实删除。 + written, err := target.mutatePlannedIndex(fx.ctx, change, marker, nil, nil, true) + if err != nil { + t.Fatalf("real marker create: %v", err) + } + if _, err := target.mutatePlannedIndex(fx.ctx, change, nil, written.SeqNo, written.PrimaryTerm, false); err != nil { + t.Fatalf("delete the marker after its lost response: %v", err) + } + observed, err := target.observePlannedIndex(fx.ctx, change) + if err != nil || observed.Found { + t.Fatalf("precondition: marker must be absent, observed=%+v err=%v", observed, err) + } + + // 预先取得的统计必须真的与现场不符:真实写入+删除推进了计数器, + // 而 stats 端点返回的仍是派发前的同 uuid 同计数(模拟清零后 ABA)。 + frozen := statsCountersOf(t, stats.frozen, change.Index) + live := readIndexCounters(t, fx, change.Index) + if frozen.IndexUUID != live.IndexUUID { + t.Fatalf("fixture index uuid changed: frozen=%s live=%s", frozen.IndexUUID, live.IndexUUID) + } + if frozen.Indexed == live.Indexed && frozen.Deleted == live.Deleted { + t.Fatalf("real marker write+delete must advance the live counters, live=%+v", live) + } + + heldPG := universeDump(t, fx.pool) + heldIndex := recoveryIndexReadback(t, fx) + heldObjects := objectVersionSet(t, fx) + + // 撤掉故障:stats 仍然"逐位相同"。统计相等不构成不可变证据,恢复必须 held。 + target.restoreMarkerFault = nil + base := wire.counts() + err = target.Restore(fx.ctx, outcome.Backup) + if err == nil { + t.Fatal("stats counters that happen to match after a reset are not immutable evidence; restoration must be held, not rebuilt") + } + if !strings.Contains(err.Error(), "held") { + t.Fatalf("held error must say held, got: %v", err) + } + if sent := wire.since(base); sent.creates != 0 || sent.marker != 0 || sent.clean != 0 || sent.deletes != 0 { + t.Fatalf("held restoration dispatched writes: %+v", sent) + } + if diff := diffUniverse(heldPG, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("held restoration changed PG: %s", diff) + } + if after := recoveryIndexReadback(t, fx); !reflect.DeepEqual(heldIndex, after) { + t.Fatal("held restoration changed the index") + } + if after := objectVersionSet(t, fx); !reflect.DeepEqual(heldObjects, after) { + t.Fatalf("held restoration changed object versions:\n before=%v\n after=%v", heldObjects, after) + } + + // 契约:删除型恢复不得再读取 / 依赖 ES 写计数器放行写入。 + if stats.restores != 0 { + t.Fatalf("restoration consulted %d stats snapshots; ES counters are diagnostic only", stats.restores) + } + if _, ok := rawEffectKeys(t, fx, plan, change, "restore-mark-intent")["fence"]; ok { + t.Fatal("restore-mark-intent still records a counter fence as authorization evidence") + } +} + // 缺口 2a:restore-finalize-intent 已落盘、干净收尾写入在派发边界上被拒绝。 // 重试必须能凭"标记体仍停在自己那份世代上"证明干净写入没有落地,并继续收尾; // 不能因为 finalize-intent 存在就永久 held(那是同一状态的死路)。 diff --git a/internal/releaseadapter/v071/library_index_restore_claim.go b/internal/releaseadapter/v071/library_index_restore_claim.go index 268b148..f098952 100644 --- a/internal/releaseadapter/v071/library_index_restore_claim.go +++ b/internal/releaseadapter/v071/library_index_restore_claim.go @@ -13,23 +13,19 @@ const indexRestoreClaimPrefix = "kg-release-transient-restore-claim:" // restorationProvablyUnsent 判定"这份恢复意图确实没有到达索引"。 // -// 判据不是"文档缺失":marker 写成功、回执丢失、随后又被删除,与从未写过在索引里 -// 完全同形,缺失本身不可归属。可用的证据只有"每成功写一次必然会变的东西": +// 判据只能是"每成功写一次必然会在服务端留下的东西": // - 更新型:观察到的内容必须仍是 applied 结果,且世代与 applied 回执完全相同; -// - 删除型(恢复写入是独占 create):文档仍缺失之外,还必须与派发前冻结的索引 -// 写计数器栅栏逐位相同。计数器只在成功写入时推进,没动才证明没写过。 +// - 删除型(恢复写入是独占 create):服务端没有持久、不可重用的证据。文档缺失与 +// "从未写过"在索引里完全同形;ES 的写计数器只是分片生命周期内的统计量(官方 +// `_stats` 文档:分片迁到别的节点时该分片的统计会被清零),索引 uuid 不变并不 +// 绑定统计生命周期,清零后重新累计到同一数值(ABA)无法排除。删除型只有当 +// **本次尝试开始时不存在任何恢复意图**(freshIntent:先落意图、后派发的不变量 +// 说明本作业从未派发过)且文档确实缺失时才能重做;既存意图一律 held(B19)。 // -// 任何"内容看起来对但世代/计数器对不上"的状态都只能 held(B19),不认领。 -func (t *ProductionLibraryTarget) restorationProvablyUnsent(ctx context.Context, observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt, intent *indexEffect) (bool, error) { +// 任何"内容看起来对但世代对不上"的状态都只能 held,不认领。 +func (t *ProductionLibraryTarget) restorationProvablyUnsent(ctx context.Context, observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt, freshIntent bool) (bool, error) { if len(c.After) == 0 { - if observed.Found || intent == nil || intent.Fence == nil { - return false, nil - } - current, err := t.search.indexWriteFence(ctx, c.Index) - if err != nil { - return false, err - } - return intent.Fence.equal(current), nil + return freshIntent && !observed.Found, nil } if !sameIndexGeneration(observed, applied.SeqNo, applied.PrimaryTerm) { return false, nil @@ -78,6 +74,9 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if err != nil { return err } + // 本次尝试开始时没有任何恢复意图,才可能证明"本作业从未派发过": + // 生产路径先落意图再派发,意图不可覆盖。既存意图的派发历史无法由缺失证明。 + freshIntent := markIntent == nil claimed, err := t.loadIndexEffect(c, "restore-claimed") if err != nil { return err @@ -112,11 +111,12 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, if !indexBytesEqual(observed, body) { // The marker is absent: either the intent never reached the index (a // deterministic refusal before dispatch) or its write outcome is unknown. - // Only evidence that every successful write would move — the owned - // generation for updates, the frozen index write fence for the exclusive - // create — separates the two; an unknown write stays held (B19) and is - // never adopted by appearance. - unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, markIntent) + // Only a server-side generation that every successful write would move — + // available for updates — separates the two. The exclusive-create path has + // no durable, non-reusable evidence (ES counters are shard-lifetime + // statistics and are cleared on shard relocation), so it stays held (B19) + // and is never adopted by appearance or by counter equality. + unsent, err := t.restorationProvablyUnsent(ctx, observed, c, applied, freshIntent) if err != nil { return err }