From 68f03bb30e3bd1b2d2bf812583904691b6a85e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 2 Nov 2026 04:37:16 +0800 Subject: [PATCH] =?UTF-8?q?test(ds03):=20=E5=AF=B9=E8=B1=A1=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E4=BC=A0=E8=BE=93=E5=A4=B1=E8=B4=A5=E5=90=8E=E5=90=8C?= =?UTF-8?q?=E7=AA=97=E5=8F=A3=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 对象分支在对象网关前拒绝一次回滚删除请求(派发前 503、转发 0、无 intent), 让自动回滚真实失败并进入 held;撤障后用同一 recover API/窗口/备份恢复, 并读回十表整行、ES 完整 _source、对象全版本与 delete markers。 同时补四类进程 executable/hash 身份与对象清单分页;旧的 ES 分支保持不变。 --- CHANGELOG.md | 26 ++ .../formal_maintenance_recovery_r38_test.go | 420 +++++++++++++++++- 2 files changed, 423 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 958d15f..0093140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## 开发快照(DS03 对象恢复传输失败 → held → 同窗口恢复) + +### Added + +- 新增对象恢复失败分支的真实隔离用例 `TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered`: + 在对象网关前拒绝一次"回滚删除本轮创建对象"的请求(派发前 503,转发计数 0,不产生网关 + intent/receipt),让自动回滚真实失败并进入窗口 held + scope/instance `needs_restore`; + 撤障后仍用同一 `POST /api/v1/library/maintenance/recover`、同一 window/job/backup/plan/mapping + 完成恢复,并逐项读回 PG 十表整行、ES 完整 `_source`、对象全部分页版本/delete markers 与字节摘要。 +- 新增四类进程身份证据:api / worker / object-gateway / 测试进程分别绑定最终源码树、binary + sha256、`/proc//exe` 实际 executable 与 commit;测试进程自证 executable 摘要,不匹配即失败。 +- 新增对象网关拒绝现场:被拒请求的 operation/key/action/plan/epoch、目标 receiver、转发计数、 + 网关侧 intent/receipt 行数、引用到的 durable create receipt,以及对象恢复的 `delete-intent` 效果。 + +### Changed + +- r38 对象分支不再把旧的 ES clean-dispatch 503 改名成对象故障:ES 回滚写入必须真正完成, + 失败必须来自对象恢复边界;未走正式发布 API 的入口在证据中明确标注"发布 API 入口未证"。 +- 对象清单改为分页枚举全部 versions 与 delete markers,版本指纹包含 ETag 与字节数。 + +### Fixed + +- 修正隔离运行器工具(证据根,不改变产品版本):对象删除前先核对容器 name/Id/label,之后 + 破坏性操作只用不可变 Id;清理异常(含 PG 读回 RuntimeError)统一落 `cleanup-incomplete` + 失败终态与清理结果文件并非零退出,不再持久 `passed`。 + ## 开发快照(DS02 oracle 独立性、映射语义与运行器失败关闭) ### Added diff --git a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go index 8e03e25..a051c84 100644 --- a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go +++ b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go @@ -19,6 +19,7 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" @@ -203,6 +204,116 @@ func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) { return out, p.refused } +// r38ObjectRefusalProxy 是对象网关前的传输层拒绝:被选中的恢复型对象操作在派发前 +// 直接 503,网关一次都没收到;其余请求原样转发并记录现场。拒绝发生在 reserve 之前, +// 因此不会产生"已落盘但结果未知"的 intent,撤障后同一 operation 仍可安全重试。 +type r38ObjectRefusalProxy struct { + url string + upstream string + root string + mu sync.Mutex + armed bool + events []map[string]any + refused int + forwarded int +} + +func newR38ObjectRefusalProxy(t *testing.T, root, upstream string) *r38ObjectRefusalProxy { + t.Helper() + p := &r38ObjectRefusalProxy{root: root, upstream: upstream} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, e := io.ReadAll(r.Body) + if e != nil { + http.Error(w, "read failed", 500) + return + } + var envelope struct { + Authorization map[string]any `json:"authorization"` + Request map[string]any `json:"request"` + } + _ = json.Unmarshal(raw, &envelope) + p.mu.Lock() + armed := p.armed + p.mu.Unlock() + action, _ := envelope.Request["action"].(string) + selected := armed && r.Method == http.MethodPost && r.URL.Path == "/v1/execute" && action == "delete" + if selected { + p.mu.Lock() + p.refused++ + p.events = append(p.events, map[string]any{ + "method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request, + "authorization": envelope.Authorization, "body": string(raw), + "receiver": p.upstream, "gateway_scope": r.Header.Get("X-KG-Gateway-Scope"), + "dispatched": false, "forwarded": false, "refused": true, + "at": time.Now().UTC().Format(time.RFC3339Nano)}) + p.flush() + p.mu.Unlock() + http.Error(w, "r38 injected object restore refusal before dispatch", http.StatusServiceUnavailable) + return + } + request, e := http.NewRequestWithContext(r.Context(), r.Method, p.upstream+r.URL.RequestURI(), bytes.NewReader(raw)) + if e != nil { + http.Error(w, "request failed", 500) + return + } + request.Header = r.Header.Clone() + request.Header.Del("Accept-Encoding") + response, e := http.DefaultClient.Do(request) + if e != nil { + http.Error(w, "upstream failed", 502) + return + } + body, e := io.ReadAll(response.Body) + response.Body.Close() + if e != nil { + http.Error(w, "upstream body failed", 502) + return + } + if r.Method == http.MethodPost { + p.mu.Lock() + p.events = append(p.events, map[string]any{ + "method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request, + "status": response.StatusCode, "receiver": p.upstream, + "dispatched": true, "forwarded": true, "refused": false, + "at": time.Now().UTC().Format(time.RFC3339Nano)}) + if !selected { + p.forwarded++ + } + p.mu.Unlock() + } + for k, values := range response.Header { + for _, v := range values { + w.Header().Add(k, v) + } + } + w.WriteHeader(response.StatusCode) + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + p.url = server.URL + p.flush() + return p +} + +func (p *r38ObjectRefusalProxy) flush() { + encoded, _ := json.MarshalIndent(p.events, "", " ") + _ = os.WriteFile(filepath.Join(p.root, "object-wire.json"), encoded, 0600) +} + +func (p *r38ObjectRefusalProxy) setArmed(armed bool) { + p.mu.Lock() + defer p.mu.Unlock() + p.armed = armed + p.flush() +} + +func (p *r38ObjectRefusalProxy) snapshot() ([]map[string]any, int, int) { + p.mu.Lock() + defer p.mu.Unlock() + out := append([]map[string]any(nil), p.events...) + return out, p.refused, p.forwarded +} + type r38API struct { base string client *http.Client @@ -238,6 +349,19 @@ func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[str return response.StatusCode, out } +// TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered 是本轮对象恢复分支的 +// 显式入口:隔离部署里必须由 runner 传入 KG_R38_SCENARIO=object-restore; +// 在隔离部署中未设置直接失败,不把旧 ES 分支的运行冒名成对象分支。 +func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) { + if os.Getenv("KG_R38_RUN") == "" { + t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)") + } + if os.Getenv("KG_R38_SCENARIO") != "object-restore" { + t.Fatalf("object recovery branch requires KG_R38_SCENARIO=object-restore, got %q", os.Getenv("KG_R38_SCENARIO")) + } + TestFormalMaintenanceRecoveryHeldThenRecovered(t) +} + func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) { root := os.Getenv("KG_R38_RUN") if root == "" { @@ -309,6 +433,28 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN) project := os.Getenv("KG_R38_PROJECT") proxy := newR38RefusalProxy(t, root, fx.searchBase) + scenario := os.Getenv("KG_R38_SCENARIO") + objectBranch := scenario == "object-restore" + if objectBranch { + // 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行, + // 否则失败会被误报成旧的 ES clean-dispatch 分支。 + proxy.setRefuseClean(false) + } + objectProxy := newR38ObjectRefusalProxy(t, root, "http://"+os.Getenv("KG_R38_GATEWAY_ADDR")) + save("00-scenario.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch, + "publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"}) + // 四类进程身份:测试进程自证 executable hash 与 runner 期望一致;产品三类在 start() + // 里按 /proc//exe 读回真实 executable 并逐字节比对构建产物。 + testExecutable, e := os.Executable() + must(e) + testHash := sha256.Sum256(func() []byte { raw, _ := os.ReadFile(testExecutable); return raw }()) + testIdentity := map[string]any{"label": "test", "pid": os.Getpid(), "executable": testExecutable, + "executable_sha256": hex.EncodeToString(testHash[:]), "expected_sha256": os.Getenv("KG_R38_TEST_BINARY_SHA"), + "commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")} + if testIdentity["expected_sha256"] == "" || testIdentity["executable_sha256"] != testIdentity["expected_sha256"] { + t.Fatalf("test process identity mismatch: %v", testIdentity) + } + save("00-test-process-identity.json", testIdentity) env := map[string]string{ "KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor", @@ -339,7 +485,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v must(e) dataToken := objectgateway.Digest([]byte(root + "data")) controlToken := objectgateway.Digest([]byte(root + "control")) - env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR") + env["LIBRARY_OBJECT_GATEWAY_URL"] = objectProxy.url env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken // worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。 @@ -395,6 +541,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600)) processes := []map[string]any{} + expectedBinaries := map[string]string{} + must(json.Unmarshal([]byte(os.Getenv("KG_R38_BINARY_SHA")), &expectedBinaries)) start := func(binary, label string, values map[string]string) *exec.Cmd { t.Helper() cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary)) @@ -407,7 +555,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v cmd.Stdout = f cmd.Stderr = f must(cmd.Start()) - processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary}) + executable, e := os.Readlink(fmt.Sprintf("/proc/%d/exe", cmd.Process.Pid)) + must(e) + raw, e := os.ReadFile(executable) + must(e) + sum := sha256.Sum256(raw) + digest := hex.EncodeToString(sum[:]) + expected := expectedBinaries[binary] + record := map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary, + "executable": executable, "executable_sha256": digest, "expected_sha256": expected, + "match": expected != "" && digest == expected, + "commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")} + if !record["match"].(bool) { + t.Fatalf("%s process identity mismatch: %v", label, record) + } + processes = append(processes, record) save("06-processes.json", processes) t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() }) return cmd @@ -535,9 +697,13 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v for i := 0; i < 900; i++ { if _, e := os.Stat(hitPath); e == nil { proxy.armVerifyCorruption() + if objectBranch { + objectProxy.setArmed(true) + } must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600)) save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true, - "verify_corruption_armed": true, "refuse_clean_dispatch": true}) + "verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch, + "refuse_object_restore_dispatch": objectBranch, "scenario": scenario}) released = true break } @@ -561,7 +727,15 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error) } failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery - if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") { + if objectBranch { + // 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。 + if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") { + t.Fatalf("publish failure does not name the refused object restore: %q", failureText) + } + if strings.Contains(failureText, "index effect outcome unknown") { + t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText) + } + } else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") { t.Fatalf("publish failure does not name the refused rollback write: %q", failureText) } if string(libraryAfterPublish.Stage) != "rollback" { @@ -605,13 +779,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v // 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。 effects := r38IndexEffects(t, archiveRoot, "chunk-target") save("12-index-effects-held.json", effects) - for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} { - if _, ok := effects[phase]; !ok { - t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects)) + if objectBranch { + // 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。 + if _, ok := effects["restored"]; !ok { + t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects)) + } + } else { + for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} { + if _, ok := effects[phase]; !ok { + t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects)) + } + } + if _, ok := effects["restored"]; ok { + t.Fatal("held boundary must not carry a restored receipt") } - } - if _, ok := effects["restored"]; ok { - t.Fatal("held boundary must not carry a restored receipt") } heldIndex := recoveryIndexReadback(t, fx) save("13-index-held.json", heldIndex) @@ -621,12 +802,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v } heldSource := heldDoc["_source"].(map[string]any) heldContent, _ := heldSource["content"].(string) - if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) { + if objectBranch { + // 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。 + if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) { + t.Fatalf("object branch left the ES restore marker in place: %q", heldContent) + } + oldSource := beforeIndex["chunk-target"]["_source"] + if !reflect.DeepEqual(oldSource, heldSource) { + t.Fatalf("object branch index was not restored to its before-image") + } + } else if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) { t.Fatalf("held index must still carry the restore marker, got %q", heldContent) } events, refused := proxy.snapshot() save("14-wire-held.json", map[string]any{"events": events, "refused": refused}) - if refused != 1 { + if objectBranch { + if refused != 0 { + t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused) + } + } else if refused != 1 { t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused) } forwarded := 0 @@ -638,9 +832,77 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v forwarded++ } } - if forwarded != 0 { + if !objectBranch && forwarded != 0 { t.Fatalf("refused clean write was forwarded %d times", forwarded) } + if objectBranch { + objectEvents, objectRefused, objectForwarded := objectProxy.snapshot() + save("14-object-wire-held.json", map[string]any{"events": objectEvents, "refused": objectRefused, + "forwarded_mutations": objectForwarded}) + if objectRefused != 1 { + t.Fatalf("object restore dispatch refusals = %d, want exactly 1", objectRefused) + } + var refusedRequest map[string]any + refusedForwarded := 0 + for _, event := range objectEvents { + request, _ := event["request"].(map[string]any) + if event["refused"] == true { + refusedRequest = request + } + if request != nil && request["action"] == "delete" && event["dispatched"] == true { + refusedForwarded++ + } + } + if refusedRequest == nil || refusedRequest["action"] != "delete" || refusedRequest["key"] == "" { + t.Fatalf("refused object request not captured: %v", refusedRequest) + } + if refusedForwarded != 0 { + t.Fatalf("refused object delete was forwarded %d times", refusedForwarded) + } + operation, _ := refusedRequest["operation"].(string) + createdBy, _ := refusedRequest["created_by"].(string) + key, _ := refusedRequest["key"].(string) + var refusedIntent, refusedReceipt, createReceipt int + must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE operation=$1", operation).Scan(&refusedIntent)) + must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&refusedReceipt)) + must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createReceipt)) + if refusedIntent != 0 || refusedReceipt != 0 { + t.Fatalf("pre-dispatch refusal must leave no gateway intent/receipt, got intent=%d receipt=%d", refusedIntent, refusedReceipt) + } + if createReceipt != 1 { + t.Fatalf("refused delete does not reference a durable create receipt (created_by=%s rows=%d)", createdBy, createReceipt) + } + var createState string + var createRaw []byte + must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createRaw)) + var createView struct { + State string `json:"state"` + } + must(json.Unmarshal(createRaw, &createView)) + createState = createView.State + if createState != "committed" { + t.Fatalf("referenced create receipt state = %q", createState) + } + deleteIntent := r38ObjectEffectPhases(t, archiveRoot, key) + if _, ok := deleteIntent["delete-intent"]; !ok { + t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent)) + } + heldObjects := objectInventoryR38(t, ctx, fx) + save("13a-objects-held.json", heldObjects) + heldKeys := map[string]bool{} + for _, version := range heldObjects.Versions { + heldKeys[aws.ToString(version.Key)] = true + } + if !heldKeys[key] { + t.Fatalf("created object %s disappeared despite the rollback refusal", key) + } + save("14a-object-refusal.json", map[string]any{"refused_request": refusedRequest, + "receiver": objectProxy.upstream, "forwarded_delete_count": refusedForwarded, + "gateway_intent_rows": refusedIntent, "gateway_receipt_rows": refusedReceipt, + "referenced_create_receipt": map[string]any{"operation": createdBy, "state": createState}, + "durable_effect_phases": effectPhases(deleteIntent), "created_key": key, + "held_object_present": heldKeys[key]}) + } // HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。 code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil) @@ -684,6 +946,12 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v // ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ---------- proxy.setRefuseClean(false) + if objectBranch { + objectProxy.setArmed(false) + } + save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal", + "backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch, + "publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"}) code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", map[string]any{"scope": "internal", "window_job_id": job.ID}) if code != http.StatusAccepted { @@ -759,6 +1027,47 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v", objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects)) } + if !reflect.DeepEqual(objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects)) { + t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v", + objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects)) + } + if objectBranch { + // 恢复后的对象写入必须来自真实网关:被拒的 delete operation 现在有 removed 回执, + // 版本与计划都对应同一冻结身份,而不是测试手工造的回执。 + objectEvents, _, _ := objectProxy.snapshot() + var operation string + for _, event := range objectEvents { + request, _ := event["request"].(map[string]any) + if event["refused"] == true && request != nil { + operation, _ = request["operation"].(string) + } + } + if operation == "" { + t.Fatal("object branch lost the refused operation identity") + } + var removedRaw []byte + must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw)) + var removed struct { + Request objectgateway.Request `json:"request"` + State string `json:"state"` + Version objectgateway.Version `json:"version"` + } + must(json.Unmarshal(removedRaw, &removed)) + if removed.State != "removed" { + t.Fatalf("recovered object delete receipt state = %q", removed.State) + } + save("23a-object-recovered-receipt.json", map[string]any{"operation": operation, + "state": removed.State, "request": removed.Request, "version": removed.Version}) + found := false + for _, inventory := range beforeObjects.Versions { + if aws.ToString(inventory.Key) == removed.Request.Key && aws.ToString(inventory.VersionId) == removed.Version.ID { + found = true + } + } + if found { + t.Fatal("removed object version still present in the before-image") + } + } afterIndex := recoveryIndexReadback(t, fx) save("24-index-after.json", afterIndex) if len(afterIndex) != len(beforeIndex) { @@ -857,16 +1166,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) { t.Fatal("replay rewrote index generations") } + if !reflect.DeepEqual(objectVersionIDs(afterObjects), objectVersionIDs(objectInventoryR38(t, ctx, fx))) || + !reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(objectInventoryR38(t, ctx, fx))) { + t.Fatal("replay rewrote object versions or delete markers") + } must(worker.Process.Signal(os.Interrupt)) _ = worker.Wait() - save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID, + objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot() + save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch, + "window_job": job.ID, "publish_queue": queueID, "recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance, "held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409}, "recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200, "pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true, - "receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps}, - "refusals": refused}) + "receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps, + "object_delete_markers_equal": true}, + "es_refusals": refused, "object_refusals": objectRefused, "object_forwarded_mutations": objectForwarded, + "publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)", + "object_wire_events_after": len(objectEventsAfter)}) t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d", job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries)) } @@ -875,12 +1193,27 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput { t.Helper() - out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)}) - if e != nil { - t.Fatalf("object inventory: %v", e) - } - if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 { - t.Fatal("incomplete object inventory or unexpected delete markers") + out := &s3.ListObjectVersionsOutput{} + var keyMarker, versionMarker *string + for page := 0; ; page++ { + if page > 1000 { + t.Fatal("object inventory pagination did not terminate") + } + result, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: aws.String(fx.bucket), KeyMarker: keyMarker, VersionIdMarker: versionMarker}) + if e != nil { + t.Fatalf("object inventory: %v", e) + } + out.Versions = append(out.Versions, result.Versions...) + out.DeleteMarkers = append(out.DeleteMarkers, result.DeleteMarkers...) + out.CommonPrefixes = append(out.CommonPrefixes, result.CommonPrefixes...) + if !aws.ToBool(result.IsTruncated) { + break + } + if result.NextKeyMarker == nil && result.NextVersionIdMarker == nil { + t.Fatal("truncated object inventory has no continuation markers") + } + keyMarker, versionMarker = result.NextKeyMarker, result.NextVersionIdMarker } return out } @@ -888,7 +1221,17 @@ func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string { ids := make([]string, 0, len(inventory.Versions)) for _, version := range inventory.Versions { - ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag)) + ids = append(ids, fmt.Sprintf("%s@%s#%s#%d", aws.ToString(version.Key), aws.ToString(version.VersionId), + aws.ToString(version.ETag), aws.ToInt64(version.Size))) + } + sort.Strings(ids) + return ids +} + +func objectDeleteMarkers(inventory *s3.ListObjectVersionsOutput) []string { + ids := make([]string, 0, len(inventory.DeleteMarkers)) + for _, marker := range inventory.DeleteMarkers { + ids = append(ids, aws.ToString(marker.Key)+"@"+aws.ToString(marker.VersionId)) } sort.Strings(ids) return ids @@ -946,6 +1289,37 @@ func effectPhases(effects map[string]map[string]any) []string { return phases } +// r38ObjectEffectPhases 汇总对象恢复在冻结计划下已落盘的阶段证据(phase -> 内容)。 +func r38ObjectEffectPhases(t *testing.T, archiveRoot, key string) map[string]map[string]any { + t.Helper() + out := map[string]map[string]any{} + _ = filepath.Walk(filepath.Join(archiveRoot, "effects"), func(path string, info os.FileInfo, err error) error { + if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") || + !strings.Contains(path, string(filepath.Separator)+"objects"+string(filepath.Separator)) { + return nil + } + raw, e := os.ReadFile(path) + if e != nil { + return nil + } + var effect struct { + Plan string `json:"plan"` + Object map[string]any `json:"object"` + Observation map[string]any `json:"observation"` + } + if json.Unmarshal(raw, &effect) != nil || effect.Object == nil { + return nil + } + if got, _ := effect.Object["Key"].(string); got != key { + return nil + } + out[strings.TrimSuffix(filepath.Base(path), ".json")] = map[string]any{ + "plan": effect.Plan, "object": effect.Object, "observation": effect.Observation} + return nil + }) + return out +} + func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) { t.Helper() rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1")