对象分支在对象网关前拒绝一次回滚删除请求(派发前 503、转发 0、无 intent), 让自动回滚真实失败并进入 held;撤障后用同一 recover API/窗口/备份恢复, 并读回十表整行、ES 完整 _source、对象全版本与 delete markers。 同时补四类进程 executable/hash 身份与对象清单分页;旧的 ES 分支保持不变。
This commit is contained in:
@@ -1,5 +1,31 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(DS03 对象恢复传输失败 → held → 同窗口恢复)
|
||||
|
||||
### Added
|
||||
|
||||
- 新增对象恢复失败分支的真实隔离用例 `TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered`:
|
||||
在对象网关前拒绝一次"回滚删除本轮创建对象"的请求(派发前 503,转发计数 0,不产生网关
|
||||
intent/receipt),让自动回滚真实失败并进入窗口 held + scope/instance `needs_restore`;
|
||||
撤障后仍用同一 `POST /api/v1/library/maintenance/recover`、同一 window/job/backup/plan/mapping
|
||||
完成恢复,并逐项读回 PG 十表整行、ES 完整 `_source`、对象全部分页版本/delete markers 与字节摘要。
|
||||
- 新增四类进程身份证据:api / worker / object-gateway / 测试进程分别绑定最终源码树、binary
|
||||
sha256、`/proc/<pid>/exe` 实际 executable 与 commit;测试进程自证 executable 摘要,不匹配即失败。
|
||||
- 新增对象网关拒绝现场:被拒请求的 operation/key/action/plan/epoch、目标 receiver、转发计数、
|
||||
网关侧 intent/receipt 行数、引用到的 durable create receipt,以及对象恢复的 `delete-intent` 效果。
|
||||
|
||||
### Changed
|
||||
|
||||
- r38 对象分支不再把旧的 ES clean-dispatch 503 改名成对象故障:ES 回滚写入必须真正完成,
|
||||
失败必须来自对象恢复边界;未走正式发布 API 的入口在证据中明确标注"发布 API 入口未证"。
|
||||
- 对象清单改为分页枚举全部 versions 与 delete markers,版本指纹包含 ETag 与字节数。
|
||||
|
||||
### Fixed
|
||||
|
||||
- 修正隔离运行器工具(证据根,不改变产品版本):对象删除前先核对容器 name/Id/label,之后
|
||||
破坏性操作只用不可变 Id;清理异常(含 PG 读回 RuntimeError)统一落 `cleanup-incomplete`
|
||||
失败终态与清理结果文件并非零退出,不再持久 `passed`。
|
||||
|
||||
## 开发快照(DS02 oracle 独立性、映射语义与运行器失败关闭)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@@ -203,6 +204,116 @@ func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) {
|
||||
return out, p.refused
|
||||
}
|
||||
|
||||
// r38ObjectRefusalProxy 是对象网关前的传输层拒绝:被选中的恢复型对象操作在派发前
|
||||
// 直接 503,网关一次都没收到;其余请求原样转发并记录现场。拒绝发生在 reserve 之前,
|
||||
// 因此不会产生"已落盘但结果未知"的 intent,撤障后同一 operation 仍可安全重试。
|
||||
type r38ObjectRefusalProxy struct {
|
||||
url string
|
||||
upstream string
|
||||
root string
|
||||
mu sync.Mutex
|
||||
armed bool
|
||||
events []map[string]any
|
||||
refused int
|
||||
forwarded int
|
||||
}
|
||||
|
||||
func newR38ObjectRefusalProxy(t *testing.T, root, upstream string) *r38ObjectRefusalProxy {
|
||||
t.Helper()
|
||||
p := &r38ObjectRefusalProxy{root: root, upstream: upstream}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, e := io.ReadAll(r.Body)
|
||||
if e != nil {
|
||||
http.Error(w, "read failed", 500)
|
||||
return
|
||||
}
|
||||
var envelope struct {
|
||||
Authorization map[string]any `json:"authorization"`
|
||||
Request map[string]any `json:"request"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &envelope)
|
||||
p.mu.Lock()
|
||||
armed := p.armed
|
||||
p.mu.Unlock()
|
||||
action, _ := envelope.Request["action"].(string)
|
||||
selected := armed && r.Method == http.MethodPost && r.URL.Path == "/v1/execute" && action == "delete"
|
||||
if selected {
|
||||
p.mu.Lock()
|
||||
p.refused++
|
||||
p.events = append(p.events, map[string]any{
|
||||
"method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request,
|
||||
"authorization": envelope.Authorization, "body": string(raw),
|
||||
"receiver": p.upstream, "gateway_scope": r.Header.Get("X-KG-Gateway-Scope"),
|
||||
"dispatched": false, "forwarded": false, "refused": true,
|
||||
"at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
p.flush()
|
||||
p.mu.Unlock()
|
||||
http.Error(w, "r38 injected object restore refusal before dispatch", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, e := http.NewRequestWithContext(r.Context(), r.Method, p.upstream+r.URL.RequestURI(), bytes.NewReader(raw))
|
||||
if e != nil {
|
||||
http.Error(w, "request failed", 500)
|
||||
return
|
||||
}
|
||||
request.Header = r.Header.Clone()
|
||||
request.Header.Del("Accept-Encoding")
|
||||
response, e := http.DefaultClient.Do(request)
|
||||
if e != nil {
|
||||
http.Error(w, "upstream failed", 502)
|
||||
return
|
||||
}
|
||||
body, e := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if e != nil {
|
||||
http.Error(w, "upstream body failed", 502)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
p.mu.Lock()
|
||||
p.events = append(p.events, map[string]any{
|
||||
"method": r.Method, "path": r.URL.RequestURI(), "request": envelope.Request,
|
||||
"status": response.StatusCode, "receiver": p.upstream,
|
||||
"dispatched": true, "forwarded": true, "refused": false,
|
||||
"at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if !selected {
|
||||
p.forwarded++
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
for k, values := range response.Header {
|
||||
for _, v := range values {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(response.StatusCode)
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
p.url = server.URL
|
||||
p.flush()
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *r38ObjectRefusalProxy) flush() {
|
||||
encoded, _ := json.MarshalIndent(p.events, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(p.root, "object-wire.json"), encoded, 0600)
|
||||
}
|
||||
|
||||
func (p *r38ObjectRefusalProxy) setArmed(armed bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.armed = armed
|
||||
p.flush()
|
||||
}
|
||||
|
||||
func (p *r38ObjectRefusalProxy) snapshot() ([]map[string]any, int, int) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := append([]map[string]any(nil), p.events...)
|
||||
return out, p.refused, p.forwarded
|
||||
}
|
||||
|
||||
type r38API struct {
|
||||
base string
|
||||
client *http.Client
|
||||
@@ -238,6 +349,19 @@ func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[str
|
||||
return response.StatusCode, out
|
||||
}
|
||||
|
||||
// TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered 是本轮对象恢复分支的
|
||||
// 显式入口:隔离部署里必须由 runner 传入 KG_R38_SCENARIO=object-restore;
|
||||
// 在隔离部署中未设置直接失败,不把旧 ES 分支的运行冒名成对象分支。
|
||||
func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) {
|
||||
if os.Getenv("KG_R38_RUN") == "" {
|
||||
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
|
||||
}
|
||||
if os.Getenv("KG_R38_SCENARIO") != "object-restore" {
|
||||
t.Fatalf("object recovery branch requires KG_R38_SCENARIO=object-restore, got %q", os.Getenv("KG_R38_SCENARIO"))
|
||||
}
|
||||
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
|
||||
}
|
||||
|
||||
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
root := os.Getenv("KG_R38_RUN")
|
||||
if root == "" {
|
||||
@@ -309,6 +433,28 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
|
||||
project := os.Getenv("KG_R38_PROJECT")
|
||||
proxy := newR38RefusalProxy(t, root, fx.searchBase)
|
||||
scenario := os.Getenv("KG_R38_SCENARIO")
|
||||
objectBranch := scenario == "object-restore"
|
||||
if objectBranch {
|
||||
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
|
||||
// 否则失败会被误报成旧的 ES clean-dispatch 分支。
|
||||
proxy.setRefuseClean(false)
|
||||
}
|
||||
objectProxy := newR38ObjectRefusalProxy(t, root, "http://"+os.Getenv("KG_R38_GATEWAY_ADDR"))
|
||||
save("00-scenario.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
|
||||
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
|
||||
// 四类进程身份:测试进程自证 executable hash 与 runner 期望一致;产品三类在 start()
|
||||
// 里按 /proc/<pid>/exe 读回真实 executable 并逐字节比对构建产物。
|
||||
testExecutable, e := os.Executable()
|
||||
must(e)
|
||||
testHash := sha256.Sum256(func() []byte { raw, _ := os.ReadFile(testExecutable); return raw }())
|
||||
testIdentity := map[string]any{"label": "test", "pid": os.Getpid(), "executable": testExecutable,
|
||||
"executable_sha256": hex.EncodeToString(testHash[:]), "expected_sha256": os.Getenv("KG_R38_TEST_BINARY_SHA"),
|
||||
"commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")}
|
||||
if testIdentity["expected_sha256"] == "" || testIdentity["executable_sha256"] != testIdentity["expected_sha256"] {
|
||||
t.Fatalf("test process identity mismatch: %v", testIdentity)
|
||||
}
|
||||
save("00-test-process-identity.json", testIdentity)
|
||||
|
||||
env := map[string]string{
|
||||
"KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor",
|
||||
@@ -339,7 +485,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
must(e)
|
||||
dataToken := objectgateway.Digest([]byte(root + "data"))
|
||||
controlToken := objectgateway.Digest([]byte(root + "control"))
|
||||
env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR")
|
||||
env["LIBRARY_OBJECT_GATEWAY_URL"] = objectProxy.url
|
||||
env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken
|
||||
|
||||
// worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。
|
||||
@@ -395,6 +541,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600))
|
||||
|
||||
processes := []map[string]any{}
|
||||
expectedBinaries := map[string]string{}
|
||||
must(json.Unmarshal([]byte(os.Getenv("KG_R38_BINARY_SHA")), &expectedBinaries))
|
||||
start := func(binary, label string, values map[string]string) *exec.Cmd {
|
||||
t.Helper()
|
||||
cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary))
|
||||
@@ -407,7 +555,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
cmd.Stdout = f
|
||||
cmd.Stderr = f
|
||||
must(cmd.Start())
|
||||
processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary})
|
||||
executable, e := os.Readlink(fmt.Sprintf("/proc/%d/exe", cmd.Process.Pid))
|
||||
must(e)
|
||||
raw, e := os.ReadFile(executable)
|
||||
must(e)
|
||||
sum := sha256.Sum256(raw)
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
expected := expectedBinaries[binary]
|
||||
record := map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary,
|
||||
"executable": executable, "executable_sha256": digest, "expected_sha256": expected,
|
||||
"match": expected != "" && digest == expected,
|
||||
"commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE")}
|
||||
if !record["match"].(bool) {
|
||||
t.Fatalf("%s process identity mismatch: %v", label, record)
|
||||
}
|
||||
processes = append(processes, record)
|
||||
save("06-processes.json", processes)
|
||||
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() })
|
||||
return cmd
|
||||
@@ -535,9 +697,13 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
for i := 0; i < 900; i++ {
|
||||
if _, e := os.Stat(hitPath); e == nil {
|
||||
proxy.armVerifyCorruption()
|
||||
if objectBranch {
|
||||
objectProxy.setArmed(true)
|
||||
}
|
||||
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
|
||||
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
|
||||
"verify_corruption_armed": true, "refuse_clean_dispatch": true})
|
||||
"verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch,
|
||||
"refuse_object_restore_dispatch": objectBranch, "scenario": scenario})
|
||||
released = true
|
||||
break
|
||||
}
|
||||
@@ -561,7 +727,15 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
|
||||
}
|
||||
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
|
||||
if objectBranch {
|
||||
// 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") {
|
||||
t.Fatalf("publish failure does not name the refused object restore: %q", failureText)
|
||||
}
|
||||
if strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText)
|
||||
}
|
||||
} else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
|
||||
}
|
||||
if string(libraryAfterPublish.Stage) != "rollback" {
|
||||
@@ -605,13 +779,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
|
||||
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
|
||||
save("12-index-effects-held.json", effects)
|
||||
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
|
||||
if _, ok := effects[phase]; !ok {
|
||||
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
|
||||
if objectBranch {
|
||||
// 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。
|
||||
if _, ok := effects["restored"]; !ok {
|
||||
t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects))
|
||||
}
|
||||
} else {
|
||||
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
|
||||
if _, ok := effects[phase]; !ok {
|
||||
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
|
||||
}
|
||||
}
|
||||
if _, ok := effects["restored"]; ok {
|
||||
t.Fatal("held boundary must not carry a restored receipt")
|
||||
}
|
||||
}
|
||||
if _, ok := effects["restored"]; ok {
|
||||
t.Fatal("held boundary must not carry a restored receipt")
|
||||
}
|
||||
heldIndex := recoveryIndexReadback(t, fx)
|
||||
save("13-index-held.json", heldIndex)
|
||||
@@ -621,12 +802,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
}
|
||||
heldSource := heldDoc["_source"].(map[string]any)
|
||||
heldContent, _ := heldSource["content"].(string)
|
||||
if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
|
||||
if objectBranch {
|
||||
// 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。
|
||||
if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
|
||||
t.Fatalf("object branch left the ES restore marker in place: %q", heldContent)
|
||||
}
|
||||
oldSource := beforeIndex["chunk-target"]["_source"]
|
||||
if !reflect.DeepEqual(oldSource, heldSource) {
|
||||
t.Fatalf("object branch index was not restored to its before-image")
|
||||
}
|
||||
} else if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
|
||||
t.Fatalf("held index must still carry the restore marker, got %q", heldContent)
|
||||
}
|
||||
events, refused := proxy.snapshot()
|
||||
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
|
||||
if refused != 1 {
|
||||
if objectBranch {
|
||||
if refused != 0 {
|
||||
t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused)
|
||||
}
|
||||
} else if refused != 1 {
|
||||
t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused)
|
||||
}
|
||||
forwarded := 0
|
||||
@@ -638,9 +832,77 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
forwarded++
|
||||
}
|
||||
}
|
||||
if forwarded != 0 {
|
||||
if !objectBranch && forwarded != 0 {
|
||||
t.Fatalf("refused clean write was forwarded %d times", forwarded)
|
||||
}
|
||||
if objectBranch {
|
||||
objectEvents, objectRefused, objectForwarded := objectProxy.snapshot()
|
||||
save("14-object-wire-held.json", map[string]any{"events": objectEvents, "refused": objectRefused,
|
||||
"forwarded_mutations": objectForwarded})
|
||||
if objectRefused != 1 {
|
||||
t.Fatalf("object restore dispatch refusals = %d, want exactly 1", objectRefused)
|
||||
}
|
||||
var refusedRequest map[string]any
|
||||
refusedForwarded := 0
|
||||
for _, event := range objectEvents {
|
||||
request, _ := event["request"].(map[string]any)
|
||||
if event["refused"] == true {
|
||||
refusedRequest = request
|
||||
}
|
||||
if request != nil && request["action"] == "delete" && event["dispatched"] == true {
|
||||
refusedForwarded++
|
||||
}
|
||||
}
|
||||
if refusedRequest == nil || refusedRequest["action"] != "delete" || refusedRequest["key"] == "" {
|
||||
t.Fatalf("refused object request not captured: %v", refusedRequest)
|
||||
}
|
||||
if refusedForwarded != 0 {
|
||||
t.Fatalf("refused object delete was forwarded %d times", refusedForwarded)
|
||||
}
|
||||
operation, _ := refusedRequest["operation"].(string)
|
||||
createdBy, _ := refusedRequest["created_by"].(string)
|
||||
key, _ := refusedRequest["key"].(string)
|
||||
var refusedIntent, refusedReceipt, createReceipt int
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE operation=$1", operation).Scan(&refusedIntent))
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&refusedReceipt))
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createReceipt))
|
||||
if refusedIntent != 0 || refusedReceipt != 0 {
|
||||
t.Fatalf("pre-dispatch refusal must leave no gateway intent/receipt, got intent=%d receipt=%d", refusedIntent, refusedReceipt)
|
||||
}
|
||||
if createReceipt != 1 {
|
||||
t.Fatalf("refused delete does not reference a durable create receipt (created_by=%s rows=%d)", createdBy, createReceipt)
|
||||
}
|
||||
var createState string
|
||||
var createRaw []byte
|
||||
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", createdBy).Scan(&createRaw))
|
||||
var createView struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
must(json.Unmarshal(createRaw, &createView))
|
||||
createState = createView.State
|
||||
if createState != "committed" {
|
||||
t.Fatalf("referenced create receipt state = %q", createState)
|
||||
}
|
||||
deleteIntent := r38ObjectEffectPhases(t, archiveRoot, key)
|
||||
if _, ok := deleteIntent["delete-intent"]; !ok {
|
||||
t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent))
|
||||
}
|
||||
heldObjects := objectInventoryR38(t, ctx, fx)
|
||||
save("13a-objects-held.json", heldObjects)
|
||||
heldKeys := map[string]bool{}
|
||||
for _, version := range heldObjects.Versions {
|
||||
heldKeys[aws.ToString(version.Key)] = true
|
||||
}
|
||||
if !heldKeys[key] {
|
||||
t.Fatalf("created object %s disappeared despite the rollback refusal", key)
|
||||
}
|
||||
save("14a-object-refusal.json", map[string]any{"refused_request": refusedRequest,
|
||||
"receiver": objectProxy.upstream, "forwarded_delete_count": refusedForwarded,
|
||||
"gateway_intent_rows": refusedIntent, "gateway_receipt_rows": refusedReceipt,
|
||||
"referenced_create_receipt": map[string]any{"operation": createdBy, "state": createState},
|
||||
"durable_effect_phases": effectPhases(deleteIntent), "created_key": key,
|
||||
"held_object_present": heldKeys[key]})
|
||||
}
|
||||
|
||||
// HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。
|
||||
code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
|
||||
@@ -684,6 +946,12 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
|
||||
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
|
||||
proxy.setRefuseClean(false)
|
||||
if objectBranch {
|
||||
objectProxy.setArmed(false)
|
||||
}
|
||||
save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal",
|
||||
"backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch,
|
||||
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
|
||||
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||||
map[string]any{"scope": "internal", "window_job_id": job.ID})
|
||||
if code != http.StatusAccepted {
|
||||
@@ -759,6 +1027,47 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v",
|
||||
objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects))
|
||||
}
|
||||
if !reflect.DeepEqual(objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects)) {
|
||||
t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v",
|
||||
objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects))
|
||||
}
|
||||
if objectBranch {
|
||||
// 恢复后的对象写入必须来自真实网关:被拒的 delete operation 现在有 removed 回执,
|
||||
// 版本与计划都对应同一冻结身份,而不是测试手工造的回执。
|
||||
objectEvents, _, _ := objectProxy.snapshot()
|
||||
var operation string
|
||||
for _, event := range objectEvents {
|
||||
request, _ := event["request"].(map[string]any)
|
||||
if event["refused"] == true && request != nil {
|
||||
operation, _ = request["operation"].(string)
|
||||
}
|
||||
}
|
||||
if operation == "" {
|
||||
t.Fatal("object branch lost the refused operation identity")
|
||||
}
|
||||
var removedRaw []byte
|
||||
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw))
|
||||
var removed struct {
|
||||
Request objectgateway.Request `json:"request"`
|
||||
State string `json:"state"`
|
||||
Version objectgateway.Version `json:"version"`
|
||||
}
|
||||
must(json.Unmarshal(removedRaw, &removed))
|
||||
if removed.State != "removed" {
|
||||
t.Fatalf("recovered object delete receipt state = %q", removed.State)
|
||||
}
|
||||
save("23a-object-recovered-receipt.json", map[string]any{"operation": operation,
|
||||
"state": removed.State, "request": removed.Request, "version": removed.Version})
|
||||
found := false
|
||||
for _, inventory := range beforeObjects.Versions {
|
||||
if aws.ToString(inventory.Key) == removed.Request.Key && aws.ToString(inventory.VersionId) == removed.Version.ID {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
t.Fatal("removed object version still present in the before-image")
|
||||
}
|
||||
}
|
||||
afterIndex := recoveryIndexReadback(t, fx)
|
||||
save("24-index-after.json", afterIndex)
|
||||
if len(afterIndex) != len(beforeIndex) {
|
||||
@@ -857,16 +1166,25 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) {
|
||||
t.Fatal("replay rewrote index generations")
|
||||
}
|
||||
if !reflect.DeepEqual(objectVersionIDs(afterObjects), objectVersionIDs(objectInventoryR38(t, ctx, fx))) ||
|
||||
!reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(objectInventoryR38(t, ctx, fx))) {
|
||||
t.Fatal("replay rewrote object versions or delete markers")
|
||||
}
|
||||
|
||||
must(worker.Process.Signal(os.Interrupt))
|
||||
_ = worker.Wait()
|
||||
save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID,
|
||||
objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot()
|
||||
save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
|
||||
"window_job": job.ID, "publish_queue": queueID,
|
||||
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
|
||||
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
|
||||
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
|
||||
"pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true,
|
||||
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps},
|
||||
"refusals": refused})
|
||||
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps,
|
||||
"object_delete_markers_equal": true},
|
||||
"es_refusals": refused, "object_refusals": objectRefused, "object_forwarded_mutations": objectForwarded,
|
||||
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)",
|
||||
"object_wire_events_after": len(objectEventsAfter)})
|
||||
t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d",
|
||||
job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries))
|
||||
}
|
||||
@@ -875,12 +1193,27 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
|
||||
func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput {
|
||||
t.Helper()
|
||||
out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)})
|
||||
if e != nil {
|
||||
t.Fatalf("object inventory: %v", e)
|
||||
}
|
||||
if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 {
|
||||
t.Fatal("incomplete object inventory or unexpected delete markers")
|
||||
out := &s3.ListObjectVersionsOutput{}
|
||||
var keyMarker, versionMarker *string
|
||||
for page := 0; ; page++ {
|
||||
if page > 1000 {
|
||||
t.Fatal("object inventory pagination did not terminate")
|
||||
}
|
||||
result, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(fx.bucket), KeyMarker: keyMarker, VersionIdMarker: versionMarker})
|
||||
if e != nil {
|
||||
t.Fatalf("object inventory: %v", e)
|
||||
}
|
||||
out.Versions = append(out.Versions, result.Versions...)
|
||||
out.DeleteMarkers = append(out.DeleteMarkers, result.DeleteMarkers...)
|
||||
out.CommonPrefixes = append(out.CommonPrefixes, result.CommonPrefixes...)
|
||||
if !aws.ToBool(result.IsTruncated) {
|
||||
break
|
||||
}
|
||||
if result.NextKeyMarker == nil && result.NextVersionIdMarker == nil {
|
||||
t.Fatal("truncated object inventory has no continuation markers")
|
||||
}
|
||||
keyMarker, versionMarker = result.NextKeyMarker, result.NextVersionIdMarker
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -888,7 +1221,17 @@ func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture)
|
||||
func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string {
|
||||
ids := make([]string, 0, len(inventory.Versions))
|
||||
for _, version := range inventory.Versions {
|
||||
ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag))
|
||||
ids = append(ids, fmt.Sprintf("%s@%s#%s#%d", aws.ToString(version.Key), aws.ToString(version.VersionId),
|
||||
aws.ToString(version.ETag), aws.ToInt64(version.Size)))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func objectDeleteMarkers(inventory *s3.ListObjectVersionsOutput) []string {
|
||||
ids := make([]string, 0, len(inventory.DeleteMarkers))
|
||||
for _, marker := range inventory.DeleteMarkers {
|
||||
ids = append(ids, aws.ToString(marker.Key)+"@"+aws.ToString(marker.VersionId))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
@@ -946,6 +1289,37 @@ func effectPhases(effects map[string]map[string]any) []string {
|
||||
return phases
|
||||
}
|
||||
|
||||
// r38ObjectEffectPhases 汇总对象恢复在冻结计划下已落盘的阶段证据(phase -> 内容)。
|
||||
func r38ObjectEffectPhases(t *testing.T, archiveRoot, key string) map[string]map[string]any {
|
||||
t.Helper()
|
||||
out := map[string]map[string]any{}
|
||||
_ = filepath.Walk(filepath.Join(archiveRoot, "effects"), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") ||
|
||||
!strings.Contains(path, string(filepath.Separator)+"objects"+string(filepath.Separator)) {
|
||||
return nil
|
||||
}
|
||||
raw, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return nil
|
||||
}
|
||||
var effect struct {
|
||||
Plan string `json:"plan"`
|
||||
Object map[string]any `json:"object"`
|
||||
Observation map[string]any `json:"observation"`
|
||||
}
|
||||
if json.Unmarshal(raw, &effect) != nil || effect.Object == nil {
|
||||
return nil
|
||||
}
|
||||
if got, _ := effect.Object["Key"].(string); got != key {
|
||||
return nil
|
||||
}
|
||||
out[strings.TrimSuffix(filepath.Base(path), ".json")] = map[string]any{
|
||||
"plan": effect.Plan, "object": effect.Object, "observation": effect.Observation}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) {
|
||||
t.Helper()
|
||||
rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1")
|
||||
|
||||
Reference in New Issue
Block a user