test(library): 补 D16 N2 真实 legacy 意图与缺失目标边界

N2-missing 场景在旧协议回滚的恢复标记 create 之前拒绝传输,使边界成为
"真实 mark-intent 已落盘、目标文档确实缺失、零 DS01 台账"。随后由同一冻结
二进制的新正式 worker 走 recover API;观测 PG/对象/索引/effect 文件零变更、
台账不回填、入口 409。旧 N2 的 marker-present 事实与 raw 不覆盖。
This commit is contained in:
姚章浩
2026-11-02 20:41:25 +08:00
parent 1aa60c159e
commit 7cdbb74c8f
2 changed files with 60 additions and 10 deletions
@@ -26,7 +26,7 @@ import (
// held,且不得回填台账。
func r38RunDS01Negative(t *testing.T, ctx context.Context, root string, api r38API, gov *pgxpool.Pool,
store *postgresstore.Store, fx *writeSetFixture, job governance.LibraryJob, backupID string,
n1 bool, worker *exec.Cmd, workerEnv map[string]string, proxy *r38RefusalProxy,
n1, missing bool, worker *exec.Cmd, workerEnv map[string]string, proxy *r38RefusalProxy,
start func(string, string, map[string]string) *exec.Cmd) {
t.Helper()
save := func(name string, value any) {
@@ -73,6 +73,28 @@ WHERE index_name=$1 AND document_id='chunk-target'`, fx.index).Scan(&state, &own
if creates := proxy.markerCreates(); creates != 0 {
t.Fatalf("dispatching boundary already forwarded %d marker creates", creates)
}
} else if missing {
// N2-missing 边界必须是"真实 legacy mark-intent 已落盘、目标确实缺失、零 DS01
// 台账"。三个事实都由现场独立读回,不从旧 run 推断。
effects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
intent, hasIntent := effects["restore-mark-intent"]
index := recoveryIndexReadback(t, fx)
_, present := index["chunk-target"]
state, owner, nonce, count := ledger()
facts := map[string]any{"mark_intent_phases": effectPhases(effects), "has_mark_intent": hasIntent,
"target_document_present": present, "dispatch_rows": count, "dispatch_state": state,
"dispatch_owner": owner, "dispatch_nonce": nonce, "marker_creates": proxy.markerCreates(),
"refused_before_dispatch": proxy.refused}
save("17n2-legacy-missing-boundary.json", facts)
if !hasIntent || intent == nil {
t.Fatalf("legacy boundary has no real mark-intent: %v", facts)
}
if present {
t.Fatalf("legacy boundary target document is not missing: %v", facts)
}
if count != 0 || state != "" {
t.Fatalf("legacy boundary unexpectedly has %d dispatch rows (state=%q)", count, state)
}
} else {
if state, _, _, count := ledger(); count != 0 || state != "" {
t.Fatalf("legacy boundary unexpectedly has %d dispatch rows (state=%q)", count, state)
@@ -108,6 +130,7 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
beforePG := universeDump(t, fx.pool)
beforeIndex := recoveryIndexReadback(t, fx)
beforeObjects := objectVersionSet(t, fx)
beforeEffects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
save("17n-held-snapshot.json", map[string]any{"old_pid": oldPID, "marker_creates": beforeDispatching})
// 同一冻结二进制的新正式 worker;不携带一次性接缝。
@@ -160,13 +183,15 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
afterPG := universeDump(t, fx.pool)
afterIndex := recoveryIndexReadback(t, fx)
afterObjects := objectVersionSet(t, fx)
afterEffects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
state, owner, nonce, count := ledger()
creates := proxy.markerCreates()
save("17n-oracle.json", map[string]any{"ledger_state": state, "ledger_owner": owner, "ledger_nonce": nonce,
"ledger_rows": count, "marker_creates_after": creates,
"pg_diff": diffUniverse(beforePG, afterPG),
"index_equal": reflect.DeepEqual(beforeIndex, afterIndex),
"objects_equal": reflect.DeepEqual(beforeObjects, afterObjects)})
"objects_equal": reflect.DeepEqual(beforeObjects, afterObjects),
"effects_equal": reflect.DeepEqual(beforeEffects, afterEffects)})
if diff := diffUniverse(beforePG, afterPG); diff != "" {
t.Fatalf("negative recovery wrote PG: %s", diff)
}
@@ -176,6 +201,9 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
if !reflect.DeepEqual(beforeObjects, afterObjects) {
t.Fatal("negative recovery changed objects")
}
if !reflect.DeepEqual(beforeEffects, afterEffects) {
t.Fatal("negative recovery changed the persisted index effect files")
}
if creates != beforeDispatching {
t.Fatalf("negative recovery forwarded %d new marker creates", creates-beforeDispatching)
}
@@ -60,6 +60,7 @@ type r38RefusalProxy struct {
url string
mu sync.Mutex
refuseClean bool
refuseMarker bool
corruptVerify bool
events []map[string]any
refused int
@@ -78,20 +79,26 @@ func newR38RefusalProxy(t *testing.T, root, upstream string) *r38RefusalProxy {
}
p.mu.Lock()
refuseClean := p.refuseClean
refuseMarker := p.refuseMarker
p.mu.Unlock()
selected := refuseClean && r.Method == http.MethodPut &&
strings.Contains(r.URL.Path, "/_doc/chunk-target") &&
!bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
if selected {
// N2-missing:拒绝**恢复标记 create 本身**,让旧协议回滚停在"意图已落盘、目标
// 确实缺失"的边界上;这不是 clean 拒绝,也不写任何目标字节。
markerRefused := refuseMarker && (r.Method == http.MethodPut || r.Method == http.MethodPost) &&
strings.Contains(r.URL.Path, "/_create/") && bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
if selected || markerRefused {
p.mu.Lock()
p.refused++
p.events = append(p.events, map[string]any{
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
"dispatched": false, "refused": true, "at": time.Now().UTC().Format(time.RFC3339Nano)})
"marker_refused": markerRefused, "dispatched": false, "refused": true,
"at": time.Now().UTC().Format(time.RFC3339Nano)})
p.flush()
p.mu.Unlock()
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份 clean 写入。
http.Error(w, "r38 injected refusal before clean dispatch", http.StatusServiceUnavailable)
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份写入。
http.Error(w, "r38 injected refusal before dispatch", http.StatusServiceUnavailable)
return
}
request, e := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
@@ -163,6 +170,13 @@ func (p *r38RefusalProxy) setRefuseClean(refuse bool) {
p.flush()
}
func (p *r38RefusalProxy) setRefuseMarker(refuse bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.refuseMarker = refuse
p.flush()
}
// markerCreates 只数**真实转发**到 ES 的恢复标记 create(路径 _create + claim 前缀)。
func (p *r38RefusalProxy) markerCreates() int {
p.mu.Lock()
@@ -557,7 +571,8 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
ds01Positive := scenario == "ds01"
ds01N1 := scenario == "ds01-n1"
ds01N2 := scenario == "ds01-n2"
ds01Branch := ds01Positive || ds01N1 || ds01N2
ds01N2Missing := scenario == "ds01-n2-missing"
ds01Branch := ds01Positive || ds01N1 || ds01N2 || ds01N2Missing
// 三条分支共用同一套 fullshape/真实入库装配;对象分支与两种 PG 分支都让 ES 恢复不成为
// 第一个失败点:ES 分支拒绝 clean 派发,PG 分支在恢复自己的事务里拿到真实 SQLSTATE。
fullShape := objectBranch || pgBranch || earlyBranch
@@ -708,6 +723,12 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
project := os.Getenv("KG_R38_PROJECT")
proxy := newR38RefusalProxy(t, root, fx.searchBase)
if ds01N2Missing {
// N2-missing:只拒绝旧协议的恢复标记 create,回滚因此停在"意图已落盘、
// 目标确实缺失"的边界;clean 放行以免把失败误归因到 clean 拒绝。
proxy.setRefuseClean(false)
proxy.setRefuseMarker(true)
}
if fullShape || ds01Positive || ds01N1 {
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
// 否则失败会被误报成旧的 ES clean-dispatch 分支。DS01 分支要让恢复在
@@ -1063,7 +1084,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if backupID == "" {
t.Fatal("N1 needs the window backup identity")
}
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, true, worker, workerEnv, proxy, start)
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, true, false, worker, workerEnv, proxy, start)
return
}
if got := waitJob(queueID, func(s string) bool { return s == "failed" }, 180, "publish rollback must fail at the restore boundary"); got != "failed" {
@@ -1407,8 +1428,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("recover with another window => %d %v", code, mismatch)
}
save("17-window-binding-409.json", map[string]any{"http": code, "body": mismatch})
if ds01N1 || ds01N2 {
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, ds01N1, worker, workerEnv, proxy, start)
if ds01N1 || ds01N2 || ds01N2Missing {
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, ds01N1, ds01N2Missing, worker, workerEnv, proxy, start)
return
}
if ds01Positive {
@@ -1427,6 +1448,7 @@ FROM library_index_restore_dispatch WHERE index_name=$1 AND document_id='chunk-t
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
proxy.setRefuseClean(false)
proxy.setRefuseMarker(false)
if objectBranch {
objectProxy.setArmed(false)
}