test(ds03): 对象恢复传输失败后同窗口恢复
build / build (push) Successful in 1m26s

对象分支在对象网关前拒绝一次回滚删除请求(派发前 503、转发 0、无 intent),
让自动回滚真实失败并进入 held;撤障后用同一 recover API/窗口/备份恢复,
并读回十表整行、ES 完整 _source、对象全版本与 delete markers。

同时补四类进程 executable/hash 身份与对象清单分页;旧的 ES 分支保持不变。
This commit is contained in:
姚章浩
2026-11-02 04:37:16 +08:00
parent 93c0f34aac
commit 68f03bb30e
2 changed files with 423 additions and 23 deletions
+26
View File
@@ -1,5 +1,31 @@
# Changelog
## 开发快照(DS03 对象恢复传输失败 → held → 同窗口恢复)
### Added
- 新增对象恢复失败分支的真实隔离用例 `TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered`:
在对象网关前拒绝一次"回滚删除本轮创建对象"的请求(派发前 503,转发计数 0,不产生网关
intent/receipt),让自动回滚真实失败并进入窗口 held + scope/instance `needs_restore`;
撤障后仍用同一 `POST /api/v1/library/maintenance/recover`、同一 window/job/backup/plan/mapping
完成恢复,并逐项读回 PG 十表整行、ES 完整 `_source`、对象全部分页版本/delete markers 与字节摘要。
- 新增四类进程身份证据:api / worker / object-gateway / 测试进程分别绑定最终源码树、binary
sha256、`/proc/<pid>/exe` 实际 executable 与 commit;测试进程自证 executable 摘要,不匹配即失败。
- 新增对象网关拒绝现场:被拒请求的 operation/key/action/plan/epoch、目标 receiver、转发计数、
网关侧 intent/receipt 行数、引用到的 durable create receipt,以及对象恢复的 `delete-intent` 效果。
### Changed
- r38 对象分支不再把旧的 ES clean-dispatch 503 改名成对象故障:ES 回滚写入必须真正完成,
失败必须来自对象恢复边界;未走正式发布 API 的入口在证据中明确标注"发布 API 入口未证"。
- 对象清单改为分页枚举全部 versions 与 delete markers,版本指纹包含 ETag 与字节数。
### Fixed
- 修正隔离运行器工具(证据根,不改变产品版本):对象删除前先核对容器 name/Id/label,之后
破坏性操作只用不可变 Id;清理异常(含 PG 读回 RuntimeError)统一落 `cleanup-incomplete`
失败终态与清理结果文件并非零退出,不再持久 `passed`。
## 开发快照(DS02 oracle 独立性、映射语义与运行器失败关闭)
### Added
@@ -19,6 +19,7 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -203,6 +204,116 @@ func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) {
return out, p.refused
}
// r38ObjectRefusalProxy 是对象网关前的传输层拒绝:被选中的恢复型对象操作在派发前
// 直接 503,网关一次都没收到;其余请求原样转发并记录现场。拒绝发生在 reserve 之前,
// 因此不会产生"已落盘但结果未知"的 intent,撤障后同一 operation 仍可安全重试。
type r38ObjectRefusalProxy struct {
url string
upstream string
root string
mu sync.Mutex
armed bool
events []map[string]any
refused int
forwarded int
}
func newR38ObjectRefusalProxy(t *testing.T, root, upstream string) *r38ObjectRefusalProxy {
t.Helper()
p := &r38ObjectRefusalProxy{root: root, upstream: upstream}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, e := io.ReadAll(r.Body)
if e != nil {
http.Error(w, "read failed", 500)
return
}
var envelope struct {
Authorization map[string]any `json:"authorization"`
Request map[string]any `json:"request"`
}
_ = json.Unmarshal(raw, &envelope)
p.mu.Lock()
armed := p.armed
p.mu.Unlock()
action, _ := envelope.Request["action"].(string)
selected := armed && r.Method == http.MethodPost && r.URL.Path == "/v1/execute" && action == "delete"
if selected {
p.mu.Lock()
p.refused++
p.events = append(p.events, map[string]any{
"method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request,
"authorization": envelope.Authorization, "body": string(raw),
"receiver": p.upstream, "gateway_scope": r.Header.Get("X-KG-Gateway-Scope"),
"dispatched": false, "forwarded": false, "refused": true,
"at": time.Now().UTC().Format(time.RFC3339Nano)})
p.flush()
p.mu.Unlock()
http.Error(w, "r38 injected object restore refusal before dispatch", http.StatusServiceUnavailable)
return
}
request, e := http.NewRequestWithContext(r.Context(), r.Method, p.upstream+r.URL.RequestURI(), bytes.NewReader(raw))
if e != nil {
http.Error(w, "request failed", 500)
return
}
request.Header = r.Header.Clone()
request.Header.Del("Accept-Encoding")
response, e := http.DefaultClient.Do(request)
if e != nil {
http.Error(w, "upstream failed", 502)
return
}
body, e := io.ReadAll(response.Body)
response.Body.Close()
if e != nil {
http.Error(w, "upstream body failed", 502)
return
}
if r.Method == http.MethodPost {
p.mu.Lock()
p.events = append(p.events, map[string]any{
"method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request,
"status": response.StatusCode, "receiver": p.upstream,
"dispatched": true, "forwarded": true, "refused": false,
"at": time.Now().UTC().Format(time.RFC3339Nano)})
if !selected {
p.forwarded++
}
p.mu.Unlock()
}
for k, values := range response.Header {
for _, v := range values {
w.Header().Add(k, v)
}
}
w.WriteHeader(response.StatusCode)
_, _ = w.Write(body)
}))
t.Cleanup(server.Close)
p.url = server.URL
p.flush()
return p
}
func (p *r38ObjectRefusalProxy) flush() {
encoded, _ := json.MarshalIndent(p.events, "", " ")
_ = os.WriteFile(filepath.Join(p.root, "object-wire.json"), encoded, 0600)
}
func (p *r38ObjectRefusalProxy) setArmed(armed bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.armed = armed
p.flush()
}
func (p *r38ObjectRefusalProxy) snapshot() ([]map[string]any, int, int) {
p.mu.Lock()
defer p.mu.Unlock()
out := append([]map[string]any(nil), p.events...)
return out, p.refused, p.forwarded
}
type r38API struct {
base string
client *http.Client
@@ -238,6 +349,19 @@ func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[str
return response.StatusCode, out
}
// TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered 是本轮对象恢复分支的
// 显式入口:隔离部署里必须由 runner 传入 KG_R38_SCENARIO=object-restore;
// 在隔离部署中未设置直接失败,不把旧 ES 分支的运行冒名成对象分支。
func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) {
if os.Getenv("KG_R38_RUN") == "" {
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
}
if os.Getenv("KG_R38_SCENARIO") != "object-restore" {
t.Fatalf("object recovery branch requires KG_R38_SCENARIO=object-restore, got %q", os.Getenv("KG_R38_SCENARIO"))
}
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
}
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
root := os.Getenv("KG_R38_RUN")
if root == "" {
@@ -309,6 +433,28 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
project := os.Getenv("KG_R38_PROJECT")
proxy := newR38RefusalProxy(t, root, fx.searchBase)
scenario := os.Getenv("KG_R38_SCENARIO")
objectBranch := scenario == "object-restore"
if objectBranch {
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
// 否则失败会被误报成旧的 ES clean-dispatch 分支。
proxy.setRefuseClean(false)
}
objectProxy := newR38ObjectRefusalProxy(t, root, "http://"+os.Getenv("KG_R38_GATEWAY_ADDR"))
save("00-scenario.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
// 四类进程身份:测试进程自证 executable hash 与 runner 期望一致;产品三类在 start()
// 里按 /proc/<pid>/exe 读回真实 executable 并逐字节比对构建产物。
testExecutable, e := os.Executable()
must(e)
testHash := sha256.Sum256(func() []byte { raw, _ := os.ReadFile(testExecutable); return raw }())
testIdentity := map[string]any{"label": "test", "pid": os.Getpid(), "executable": testExecutable,
"executable_sha256": hex.EncodeToString(testHash[:]), "expected_sha256": os.Getenv("KG_R38_TEST_BINARY_SHA"),
"commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")}
if testIdentity["expected_sha256"] == "" || testIdentity["executable_sha256"] != testIdentity["expected_sha256"] {
t.Fatalf("test process identity mismatch: %v", testIdentity)
}
save("00-test-process-identity.json", testIdentity)
env := map[string]string{
"KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor",
@@ -339,7 +485,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
must(e)
dataToken := objectgateway.Digest([]byte(root + "data"))
controlToken := objectgateway.Digest([]byte(root + "control"))
env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR")
env["LIBRARY_OBJECT_GATEWAY_URL"] = objectProxy.url
env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken
// worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。
@@ -395,6 +541,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600))
processes := []map[string]any{}
expectedBinaries := map[string]string{}
must(json.Unmarshal([]byte(os.Getenv("KG_R38_BINARY_SHA")), &expectedBinaries))
start := func(binary, label string, values map[string]string) *exec.Cmd {
t.Helper()
cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary))
@@ -407,7 +555,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
cmd.Stdout = f
cmd.Stderr = f
must(cmd.Start())
processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary})
executable, e := os.Readlink(fmt.Sprintf("/proc/%d/exe", cmd.Process.Pid))
must(e)
raw, e := os.ReadFile(executable)
must(e)
sum := sha256.Sum256(raw)
digest := hex.EncodeToString(sum[:])
expected := expectedBinaries[binary]
record := map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary,
"executable": executable, "executable_sha256": digest, "expected_sha256": expected,
"match": expected != "" && digest == expected,
"commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")}
if !record["match"].(bool) {
t.Fatalf("%s process identity mismatch: %v", label, record)
}
processes = append(processes, record)
save("06-processes.json", processes)
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() })
return cmd
@@ -535,9 +697,13 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
for i := 0; i < 900; i++ {
if _, e := os.Stat(hitPath); e == nil {
proxy.armVerifyCorruption()
if objectBranch {
objectProxy.setArmed(true)
}
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
"verify_corruption_armed": true, "refuse_clean_dispatch": true})
"verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch,
"refuse_object_restore_dispatch": objectBranch, "scenario": scenario})
released = true
break
}
@@ -561,7 +727,15 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
}
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
if objectBranch {
// 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") {
t.Fatalf("publish failure does not name the refused object restore: %q", failureText)
}
if strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText)
}
} else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
}
if string(libraryAfterPublish.Stage) != "rollback" {
@@ -605,13 +779,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
save("12-index-effects-held.json", effects)
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
if _, ok := effects[phase]; !ok {
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
if objectBranch {
// 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。
if _, ok := effects["restored"]; !ok {
t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects))
}
} else {
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
if _, ok := effects[phase]; !ok {
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
}
}
if _, ok := effects["restored"]; ok {
t.Fatal("held boundary must not carry a restored receipt")
}
}
if _, ok := effects["restored"]; ok {
t.Fatal("held boundary must not carry a restored receipt")
}
heldIndex := recoveryIndexReadback(t, fx)
save("13-index-held.json", heldIndex)
@@ -621,12 +802,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
}
heldSource := heldDoc["_source"].(map[string]any)
heldContent, _ := heldSource["content"].(string)
if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
if objectBranch {
// 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。
if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
t.Fatalf("object branch left the ES restore marker in place: %q", heldContent)
}
oldSource := beforeIndex["chunk-target"]["_source"]
if !reflect.DeepEqual(oldSource, heldSource) {
t.Fatalf("object branch index was not restored to its before-image")
}
} else if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
t.Fatalf("held index must still carry the restore marker, got %q", heldContent)
}
events, refused := proxy.snapshot()
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
if refused != 1 {
if objectBranch {
if refused != 0 {
t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused)
}
} else if refused != 1 {
t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused)
}
forwarded := 0
@@ -638,9 +832,77 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
forwarded++
}
}
if forwarded != 0 {
if !objectBranch && forwarded != 0 {
t.Fatalf("refused clean write was forwarded %d times", forwarded)
}
if objectBranch {
objectEvents, objectRefused, objectForwarded := objectProxy.snapshot()
save("14-object-wire-held.json", map[string]any{"events": objectEvents, "refused": objectRefused,
"forwarded_mutations": objectForwarded})
if objectRefused != 1 {
t.Fatalf("object restore dispatch refusals = %d, want exactly 1", objectRefused)
}
var refusedRequest map[string]any
refusedForwarded := 0
for _, event := range objectEvents {
request, _ := event["request"].(map[string]any)
if event["refused"] == true {
refusedRequest = request
}
if request != nil && request["action"] == "delete" && event["dispatched"] == true {
refusedForwarded++
}
}
if refusedRequest == nil || refusedRequest["action"] != "delete" || refusedRequest["key"] == "" {
t.Fatalf("refused object request not captured: %v", refusedRequest)
}
if refusedForwarded != 0 {
t.Fatalf("refused object delete was forwarded %d times", refusedForwarded)
}
operation, _ := refusedRequest["operation"].(string)
createdBy, _ := refusedRequest["created_by"].(string)
key, _ := refusedRequest["key"].(string)
var refusedIntent, refusedReceipt, createReceipt int
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE operation=$1", operation).Scan(&refusedIntent))
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&refusedReceipt))
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createReceipt))
if refusedIntent != 0 || refusedReceipt != 0 {
t.Fatalf("pre-dispatch refusal must leave no gateway intent/receipt, got intent=%d receipt=%d", refusedIntent, refusedReceipt)
}
if createReceipt != 1 {
t.Fatalf("refused delete does not reference a durable create receipt (created_by=%s rows=%d)", createdBy, createReceipt)
}
var createState string
var createRaw []byte
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createRaw))
var createView struct {
State string `json:"state"`
}
must(json.Unmarshal(createRaw, &createView))
createState = createView.State
if createState != "committed" {
t.Fatalf("referenced create receipt state = %q", createState)
}
deleteIntent := r38ObjectEffectPhases(t, archiveRoot, key)
if _, ok := deleteIntent["delete-intent"]; !ok {
t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent))
}
heldObjects := objectInventoryR38(t, ctx, fx)
save("13a-objects-held.json", heldObjects)
heldKeys := map[string]bool{}
for _, version := range heldObjects.Versions {
heldKeys[aws.ToString(version.Key)] = true
}
if !heldKeys[key] {
t.Fatalf("created object %s disappeared despite the rollback refusal", key)
}
save("14a-object-refusal.json", map[string]any{"refused_request": refusedRequest,
"receiver": objectProxy.upstream, "forwarded_delete_count": refusedForwarded,
"gateway_intent_rows": refusedIntent, "gateway_receipt_rows": refusedReceipt,
"referenced_create_receipt": map[string]any{"operation": createdBy, "state": createState},
"durable_effect_phases": effectPhases(deleteIntent), "created_key": key,
"held_object_present": heldKeys[key]})
}
// HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。
code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
@@ -684,6 +946,12 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
proxy.setRefuseClean(false)
if objectBranch {
objectProxy.setArmed(false)
}
save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal",
"backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch,
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
map[string]any{"scope": "internal", "window_job_id": job.ID})
if code != http.StatusAccepted {
@@ -759,6 +1027,47 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v",
objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects))
}
if !reflect.DeepEqual(objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects)) {
t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v",
objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects))
}
if objectBranch {
// 恢复后的对象写入必须来自真实网关:被拒的 delete operation 现在有 removed 回执,
// 版本与计划都对应同一冻结身份,而不是测试手工造的回执。
objectEvents, _, _ := objectProxy.snapshot()
var operation string
for _, event := range objectEvents {
request, _ := event["request"].(map[string]any)
if event["refused"] == true && request != nil {
operation, _ = request["operation"].(string)
}
}
if operation == "" {
t.Fatal("object branch lost the refused operation identity")
}
var removedRaw []byte
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw))
var removed struct {
Request objectgateway.Request `json:"request"`
State string `json:"state"`
Version objectgateway.Version `json:"version"`
}
must(json.Unmarshal(removedRaw, &removed))
if removed.State != "removed" {
t.Fatalf("recovered object delete receipt state = %q", removed.State)
}
save("23a-object-recovered-receipt.json", map[string]any{"operation": operation,
"state": removed.State, "request": removed.Request, "version": removed.Version})
found := false
for _, inventory := range beforeObjects.Versions {
if aws.ToString(inventory.Key) == removed.Request.Key && aws.ToString(inventory.VersionId) == removed.Version.ID {
found = true
}
}
if found {
t.Fatal("removed object version still present in the before-image")
}
}
afterIndex := recoveryIndexReadback(t, fx)
save("24-index-after.json", afterIndex)
if len(afterIndex) != len(beforeIndex) {
@@ -857,16 +1166,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) {
t.Fatal("replay rewrote index generations")
}
if !reflect.DeepEqual(objectVersionIDs(afterObjects), objectVersionIDs(objectInventoryR38(t, ctx, fx))) ||
!reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(objectInventoryR38(t, ctx, fx))) {
t.Fatal("replay rewrote object versions or delete markers")
}
must(worker.Process.Signal(os.Interrupt))
_ = worker.Wait()
save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID,
objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot()
save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
"window_job": job.ID, "publish_queue": queueID,
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
"pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true,
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps},
"refusals": refused})
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps,
"object_delete_markers_equal": true},
"es_refusals": refused, "object_refusals": objectRefused, "object_forwarded_mutations": objectForwarded,
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)",
"object_wire_events_after": len(objectEventsAfter)})
t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d",
job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries))
}
@@ -875,12 +1193,27 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput {
t.Helper()
out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)})
if e != nil {
t.Fatalf("object inventory: %v", e)
}
if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 {
t.Fatal("incomplete object inventory or unexpected delete markers")
out := &s3.ListObjectVersionsOutput{}
var keyMarker, versionMarker *string
for page := 0; ; page++ {
if page > 1000 {
t.Fatal("object inventory pagination did not terminate")
}
result, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: aws.String(fx.bucket), KeyMarker: keyMarker, VersionIdMarker: versionMarker})
if e != nil {
t.Fatalf("object inventory: %v", e)
}
out.Versions = append(out.Versions, result.Versions...)
out.DeleteMarkers = append(out.DeleteMarkers, result.DeleteMarkers...)
out.CommonPrefixes = append(out.CommonPrefixes, result.CommonPrefixes...)
if !aws.ToBool(result.IsTruncated) {
break
}
if result.NextKeyMarker == nil && result.NextVersionIdMarker == nil {
t.Fatal("truncated object inventory has no continuation markers")
}
keyMarker, versionMarker = result.NextKeyMarker, result.NextVersionIdMarker
}
return out
}
@@ -888,7 +1221,17 @@ func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture)
func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string {
ids := make([]string, 0, len(inventory.Versions))
for _, version := range inventory.Versions {
ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag))
ids = append(ids, fmt.Sprintf("%s@%s#%s#%d", aws.ToString(version.Key), aws.ToString(version.VersionId),
aws.ToString(version.ETag), aws.ToInt64(version.Size)))
}
sort.Strings(ids)
return ids
}
func objectDeleteMarkers(inventory *s3.ListObjectVersionsOutput) []string {
ids := make([]string, 0, len(inventory.DeleteMarkers))
for _, marker := range inventory.DeleteMarkers {
ids = append(ids, aws.ToString(marker.Key)+"@"+aws.ToString(marker.VersionId))
}
sort.Strings(ids)
return ids
@@ -946,6 +1289,37 @@ func effectPhases(effects map[string]map[string]any) []string {
return phases
}
// r38ObjectEffectPhases 汇总对象恢复在冻结计划下已落盘的阶段证据(phase -> 内容)。
func r38ObjectEffectPhases(t *testing.T, archiveRoot, key string) map[string]map[string]any {
t.Helper()
out := map[string]map[string]any{}
_ = filepath.Walk(filepath.Join(archiveRoot, "effects"), func(path string, info os.FileInfo, err error) error {
if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") ||
!strings.Contains(path, string(filepath.Separator)+"objects"+string(filepath.Separator)) {
return nil
}
raw, e := os.ReadFile(path)
if e != nil {
return nil
}
var effect struct {
Plan string `json:"plan"`
Object map[string]any `json:"object"`
Observation map[string]any `json:"observation"`
}
if json.Unmarshal(raw, &effect) != nil || effect.Object == nil {
return nil
}
if got, _ := effect.Object["Key"].(string); got != key {
return nil
}
out[strings.TrimSuffix(filepath.Base(path), ".json")] = map[string]any{
"plan": effect.Plan, "object": effect.Object, "observation": effect.Observation}
return nil
})
return out
}
func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) {
t.Helper()
rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1")