test(library): 补 D16 N2 真实 legacy 意图与缺失目标边界
N2-missing 场景在旧协议回滚的恢复标记 create 之前拒绝传输,使边界成为 "真实 mark-intent 已落盘、目标文档确实缺失、零 DS01 台账"。随后由同一冻结 二进制的新正式 worker 走 recover API;观测 PG/对象/索引/effect 文件零变更、 台账不回填、入口 409。旧 N2 的 marker-present 事实与 raw 不覆盖。
This commit is contained in:
@@ -26,7 +26,7 @@ import (
|
||||
// held,且不得回填台账。
|
||||
func r38RunDS01Negative(t *testing.T, ctx context.Context, root string, api r38API, gov *pgxpool.Pool,
|
||||
store *postgresstore.Store, fx *writeSetFixture, job governance.LibraryJob, backupID string,
|
||||
n1 bool, worker *exec.Cmd, workerEnv map[string]string, proxy *r38RefusalProxy,
|
||||
n1, missing bool, worker *exec.Cmd, workerEnv map[string]string, proxy *r38RefusalProxy,
|
||||
start func(string, string, map[string]string) *exec.Cmd) {
|
||||
t.Helper()
|
||||
save := func(name string, value any) {
|
||||
@@ -73,6 +73,28 @@ WHERE index_name=$1 AND document_id='chunk-target'`, fx.index).Scan(&state, &own
|
||||
if creates := proxy.markerCreates(); creates != 0 {
|
||||
t.Fatalf("dispatching boundary already forwarded %d marker creates", creates)
|
||||
}
|
||||
} else if missing {
|
||||
// N2-missing 边界必须是"真实 legacy mark-intent 已落盘、目标确实缺失、零 DS01
|
||||
// 台账"。三个事实都由现场独立读回,不从旧 run 推断。
|
||||
effects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
|
||||
intent, hasIntent := effects["restore-mark-intent"]
|
||||
index := recoveryIndexReadback(t, fx)
|
||||
_, present := index["chunk-target"]
|
||||
state, owner, nonce, count := ledger()
|
||||
facts := map[string]any{"mark_intent_phases": effectPhases(effects), "has_mark_intent": hasIntent,
|
||||
"target_document_present": present, "dispatch_rows": count, "dispatch_state": state,
|
||||
"dispatch_owner": owner, "dispatch_nonce": nonce, "marker_creates": proxy.markerCreates(),
|
||||
"refused_before_dispatch": proxy.refused}
|
||||
save("17n2-legacy-missing-boundary.json", facts)
|
||||
if !hasIntent || intent == nil {
|
||||
t.Fatalf("legacy boundary has no real mark-intent: %v", facts)
|
||||
}
|
||||
if present {
|
||||
t.Fatalf("legacy boundary target document is not missing: %v", facts)
|
||||
}
|
||||
if count != 0 || state != "" {
|
||||
t.Fatalf("legacy boundary unexpectedly has %d dispatch rows (state=%q)", count, state)
|
||||
}
|
||||
} else {
|
||||
if state, _, _, count := ledger(); count != 0 || state != "" {
|
||||
t.Fatalf("legacy boundary unexpectedly has %d dispatch rows (state=%q)", count, state)
|
||||
@@ -108,6 +130,7 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
beforeIndex := recoveryIndexReadback(t, fx)
|
||||
beforeObjects := objectVersionSet(t, fx)
|
||||
beforeEffects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
|
||||
save("17n-held-snapshot.json", map[string]any{"old_pid": oldPID, "marker_creates": beforeDispatching})
|
||||
|
||||
// 同一冻结二进制的新正式 worker;不携带一次性接缝。
|
||||
@@ -160,13 +183,15 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
|
||||
afterPG := universeDump(t, fx.pool)
|
||||
afterIndex := recoveryIndexReadback(t, fx)
|
||||
afterObjects := objectVersionSet(t, fx)
|
||||
afterEffects := r38IndexEffects(t, filepath.Join(root, "archive"), "chunk-target")
|
||||
state, owner, nonce, count := ledger()
|
||||
creates := proxy.markerCreates()
|
||||
save("17n-oracle.json", map[string]any{"ledger_state": state, "ledger_owner": owner, "ledger_nonce": nonce,
|
||||
"ledger_rows": count, "marker_creates_after": creates,
|
||||
"pg_diff": diffUniverse(beforePG, afterPG),
|
||||
"index_equal": reflect.DeepEqual(beforeIndex, afterIndex),
|
||||
"objects_equal": reflect.DeepEqual(beforeObjects, afterObjects)})
|
||||
"objects_equal": reflect.DeepEqual(beforeObjects, afterObjects),
|
||||
"effects_equal": reflect.DeepEqual(beforeEffects, afterEffects)})
|
||||
if diff := diffUniverse(beforePG, afterPG); diff != "" {
|
||||
t.Fatalf("negative recovery wrote PG: %s", diff)
|
||||
}
|
||||
@@ -176,6 +201,9 @@ FROM library_jobs WHERE id=$1`, job.ID).Scan(&expired); err != nil {
|
||||
if !reflect.DeepEqual(beforeObjects, afterObjects) {
|
||||
t.Fatal("negative recovery changed objects")
|
||||
}
|
||||
if !reflect.DeepEqual(beforeEffects, afterEffects) {
|
||||
t.Fatal("negative recovery changed the persisted index effect files")
|
||||
}
|
||||
if creates != beforeDispatching {
|
||||
t.Fatalf("negative recovery forwarded %d new marker creates", creates-beforeDispatching)
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ type r38RefusalProxy struct {
|
||||
url string
|
||||
mu sync.Mutex
|
||||
refuseClean bool
|
||||
refuseMarker bool
|
||||
corruptVerify bool
|
||||
events []map[string]any
|
||||
refused int
|
||||
@@ -78,20 +79,26 @@ func newR38RefusalProxy(t *testing.T, root, upstream string) *r38RefusalProxy {
|
||||
}
|
||||
p.mu.Lock()
|
||||
refuseClean := p.refuseClean
|
||||
refuseMarker := p.refuseMarker
|
||||
p.mu.Unlock()
|
||||
selected := refuseClean && r.Method == http.MethodPut &&
|
||||
strings.Contains(r.URL.Path, "/_doc/chunk-target") &&
|
||||
!bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
|
||||
if selected {
|
||||
// N2-missing:拒绝**恢复标记 create 本身**,让旧协议回滚停在"意图已落盘、目标
|
||||
// 确实缺失"的边界上;这不是 clean 拒绝,也不写任何目标字节。
|
||||
markerRefused := refuseMarker && (r.Method == http.MethodPut || r.Method == http.MethodPost) &&
|
||||
strings.Contains(r.URL.Path, "/_create/") && bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
|
||||
if selected || markerRefused {
|
||||
p.mu.Lock()
|
||||
p.refused++
|
||||
p.events = append(p.events, map[string]any{
|
||||
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
|
||||
"dispatched": false, "refused": true, "at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
"marker_refused": markerRefused, "dispatched": false, "refused": true,
|
||||
"at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
p.flush()
|
||||
p.mu.Unlock()
|
||||
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份 clean 写入。
|
||||
http.Error(w, "r38 injected refusal before clean dispatch", http.StatusServiceUnavailable)
|
||||
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份写入。
|
||||
http.Error(w, "r38 injected refusal before dispatch", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, e := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
|
||||
@@ -163,6 +170,13 @@ func (p *r38RefusalProxy) setRefuseClean(refuse bool) {
|
||||
p.flush()
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) setRefuseMarker(refuse bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.refuseMarker = refuse
|
||||
p.flush()
|
||||
}
|
||||
|
||||
// markerCreates 只数**真实转发**到 ES 的恢复标记 create(路径 _create + claim 前缀)。
|
||||
func (p *r38RefusalProxy) markerCreates() int {
|
||||
p.mu.Lock()
|
||||
@@ -557,7 +571,8 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
ds01Positive := scenario == "ds01"
|
||||
ds01N1 := scenario == "ds01-n1"
|
||||
ds01N2 := scenario == "ds01-n2"
|
||||
ds01Branch := ds01Positive || ds01N1 || ds01N2
|
||||
ds01N2Missing := scenario == "ds01-n2-missing"
|
||||
ds01Branch := ds01Positive || ds01N1 || ds01N2 || ds01N2Missing
|
||||
// 三条分支共用同一套 fullshape/真实入库装配;对象分支与两种 PG 分支都让 ES 恢复不成为
|
||||
// 第一个失败点:ES 分支拒绝 clean 派发,PG 分支在恢复自己的事务里拿到真实 SQLSTATE。
|
||||
fullShape := objectBranch || pgBranch || earlyBranch
|
||||
@@ -708,6 +723,12 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
|
||||
project := os.Getenv("KG_R38_PROJECT")
|
||||
proxy := newR38RefusalProxy(t, root, fx.searchBase)
|
||||
if ds01N2Missing {
|
||||
// N2-missing:只拒绝旧协议的恢复标记 create,回滚因此停在"意图已落盘、
|
||||
// 目标确实缺失"的边界;clean 放行以免把失败误归因到 clean 拒绝。
|
||||
proxy.setRefuseClean(false)
|
||||
proxy.setRefuseMarker(true)
|
||||
}
|
||||
if fullShape || ds01Positive || ds01N1 {
|
||||
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
|
||||
// 否则失败会被误报成旧的 ES clean-dispatch 分支。DS01 分支要让恢复在
|
||||
@@ -1063,7 +1084,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if backupID == "" {
|
||||
t.Fatal("N1 needs the window backup identity")
|
||||
}
|
||||
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, true, worker, workerEnv, proxy, start)
|
||||
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, true, false, worker, workerEnv, proxy, start)
|
||||
return
|
||||
}
|
||||
if got := waitJob(queueID, func(s string) bool { return s == "failed" }, 180, "publish rollback must fail at the restore boundary"); got != "failed" {
|
||||
@@ -1407,8 +1428,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("recover with another window => %d %v", code, mismatch)
|
||||
}
|
||||
save("17-window-binding-409.json", map[string]any{"http": code, "body": mismatch})
|
||||
if ds01N1 || ds01N2 {
|
||||
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, ds01N1, worker, workerEnv, proxy, start)
|
||||
if ds01N1 || ds01N2 || ds01N2Missing {
|
||||
r38RunDS01Negative(t, ctx, root, api, gov, store, fx, job, backupID, ds01N1, ds01N2Missing, worker, workerEnv, proxy, start)
|
||||
return
|
||||
}
|
||||
if ds01Positive {
|
||||
@@ -1427,6 +1448,7 @@ FROM library_index_restore_dispatch WHERE index_name=$1 AND document_id='chunk-t
|
||||
|
||||
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
|
||||
proxy.setRefuseClean(false)
|
||||
proxy.setRefuseMarker(false)
|
||||
if objectBranch {
|
||||
objectProxy.setArmed(false)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user