test(ds03): 新增PG恢复失败后同窗口续作用例
build / build (push) Successful in 1m25s

为什么:B29.2 的 PG 子断言缺一条真实命中 PG 恢复步骤的失败闭环。故障不改产品
返回值、不停共享 PG、不扩 ACL、不改 schema:恢复最后一段 PG 写入(wiki 阶段)
会把 before-image 的日志行按原 id 插回目标范围;先用一行"外部范围、同主键"的
日志占住这个 id,那条真实 INSERT 就得到 23505。用例记录服务器 SQLSTATE、同一
语句的探针读回(没有多出行)、失败阶段事务回滚,以及真实的部分恢复现场:
database 阶段已提交、wiki 表仍是发布写入、共享 resources 的新增行按设计留到
四阶段全部成功后再清。解除只删这一行,随后走同一 window_job_id 的正式 recover。

同时把夹具的索引登记接通运行器(KG_R38_INDEX 被真实消费),并落实际 namespace,
使 wrapper 在夹具未执行 t.Cleanup 时也只清登记的那一个索引。

验证:make verify 16/16 success(local 档案,identity=skipped 属本地常态);
PG 闭环的真实运行在提交后按同一 SHA 单独执行并留 raw。
This commit is contained in:
姚章浩
2026-11-02 06:22:14 +08:00
parent 98e7279632
commit e760d85427
3 changed files with 307 additions and 27 deletions
+31
View File
@@ -1,5 +1,36 @@
# Changelog
## 开发快照(DS03 PG 恢复失败:held → 同窗口二次恢复)
### Added
- 新增 PG 恢复失败用例 `TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered`:复用已证明的
`after:wiki` 定点进入回滚,唯一新增故障来自本轮一次性业务库里真实的 PG 错误——恢复事务写
`knowledges` 时触发器 `RAISE EXCEPTION ... USING ERRCODE='42501'`。用例记录触发器 DDL、
服务器返回的 SQLSTATE、探针语句与读回(该语句没有留下任何行),并断言失败事务整体回滚后
目标仍是发布写入后的内容;`恢复阶段 database 失败` 与 `SQLSTATE 42501` 必须同时出现在失败文案里。
- 新增 held 现场证据:故障未被自动解除(开关仍开、触发器仍在),目标行逐表等于 `after:wiki`
处的 applied 读回,索引仍是发布写入后的内容(恢复没有走到索引阶段)。
- 新增同窗口续作证据:解除故障只 DROP 本轮触发器/函数/开关表;同一 `window_job_id` 的
`POST /api/v1/library/maintenance/recover` 完成后复核十表、引用承诺、对象版本字节、索引
与网关回执/投递,并保存 replay 原始读回。
### Changed
- 隔离夹具在 `reset` 时消费运行器登记的唯一索引(`KG_R38_INDEX`),并在第一次写入前把实际
namespace 落到本轮目录;清理器核对「运行器登记 / 夹具落盘 / 实际清理」是同一个名字,
登记不一致时拒绝删除,不再只依赖 `t.Cleanup`。
### Fixed
- 证据目录的运行器:构建后/执行后比较 HEAD、干净状态与源码树摘要,构建后不符直接拒跑用例、
执行后不符结论作废(不再出现源码已改却 `passed-cleaned`);finally 里的 provenance 异常
不再跳过精确清理。
- 证据目录的清理器:PG 环境发现等前置动作失败也落 `resources-cleaned.json` 与非零终态;
已有的 `cleanup-incomplete` 结果文件不再因"文件存在"而返回 0。
---
## 开发快照(DS03 对象恢复父复验:十表全形状与对象版本字节)
### Added
@@ -23,6 +23,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -40,6 +41,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/crypto/bcrypt"
@@ -364,6 +366,20 @@ func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) {
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
}
// TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered 是 PG 恢复失败分支的显式入口:
// 隔离部署里必须由 runner 传入 KG_R38_SCENARIO=pg-restore。发布失败入口复用已证明的
// after:wiki 定点;唯一新增故障来自这次专用恢复事务里的真实 PG 错误(触发器 + SQLSTATE),
// 不靠停共享 PG、扩大 ACL 或改产品返回值制造。
func TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered(t *testing.T) {
if os.Getenv("KG_R38_RUN") == "" {
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
}
if os.Getenv("KG_R38_SCENARIO") != "pg-restore" {
t.Fatalf("pg recovery branch requires KG_R38_SCENARIO=pg-restore, got %q", os.Getenv("KG_R38_SCENARIO"))
}
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
}
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
root := os.Getenv("KG_R38_RUN")
if root == "" {
@@ -371,6 +387,10 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
}
scenario := os.Getenv("KG_R38_SCENARIO")
objectBranch := scenario == "object-restore"
pgBranch := scenario == "pg-restore"
// 三条分支共用同一套 fullshape/真实入库装配;对象分支与 PG 分支都让 ES 恢复成功,
// 只有 ES 分支把 clean 派发拒掉。
fullShape := objectBranch || pgBranch
ctx := context.Background()
save := func(name string, value any) {
t.Helper()
@@ -398,7 +418,7 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
// 目标 before-image 十表全非空。旧 ES 分支保持原最小夹具语义,不在本轮扩围。
var declaredBefore map[string][]map[string]any
var beforeSentinels map[string][]map[string]any
if objectBranch {
if fullShape {
declaredBefore = r38SeedFullShape(t, fx)
} else {
seedTargetData(t, fx)
@@ -412,7 +432,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
must(e)
beforePG := universeDump(t, fx.pool)
save("01-pg-before.json", beforePG)
if objectBranch {
// PG 分支的故障锚点:恢复的最后一段 PG 写入(wiki 阶段)会把 before-image 的
// 日志行按原 id 插回目标范围,所以一个同主键、不同范围的"外来行"会让那条真实
// INSERT 报 23505;发布/恢复都不改 schema,也不动共享 PG。
var conflictLog int64
var beforeTarget map[string][]map[string]any
if pgBranch {
must(fx.pool.QueryRow(ctx, `SELECT id FROM wiki_log_entries WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY id DESC LIMIT 1`,
wsTargetTenant, wsTargetKB).Scan(&conflictLog))
if conflictLog == 0 {
t.Fatal("pg branch needs a before-image wiki log identity to collide with")
}
beforeTarget = r38TargetRows(t, fx)
}
if fullShape {
for _, table := range ds02TargetTables {
if len(beforePG[table]) == 0 {
t.Fatalf("before-image table %s is empty: the ten-table claim would be vacuous", table)
@@ -432,7 +465,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
beforeObjects := objectInventoryR38(t, ctx, fx)
save("03-objects-before.json", beforeObjects)
var beforeDigests map[string]string
if objectBranch {
if fullShape {
beforeDigests = r38ObjectVersionDigests(t, ctx, fx, beforeObjects)
if len(beforeDigests) == 0 {
t.Fatal("before-image has no object versions to fingerprint")
@@ -449,7 +482,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
var expectation ds02Expectation
var source string
var sourceParts map[string]int64
if objectBranch {
if fullShape {
spec = ds02SourceSpec(t, fx)
expectation = ds02Expected(t, fx, spec)
source = r38SealFullShape(t, fx, spec, mapping)
@@ -482,7 +515,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
project := os.Getenv("KG_R38_PROJECT")
proxy := newR38RefusalProxy(t, root, fx.searchBase)
if objectBranch {
if fullShape {
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
// 否则失败会被误报成旧的 ES clean-dispatch 分支。
proxy.setRefuseClean(false)
@@ -743,13 +776,19 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
// 放行后由生产自己的校验判定失败并进入回滚。
hitPath := filepath.Join(root, "barriers", "after-wiki.hit")
released := false
var appliedRows map[string][]map[string]any
var appliedIndex map[string]map[string]any
var pgFault map[string]any
for i := 0; i < 900; i++ {
if _, e := os.Stat(hitPath); e == nil {
if objectBranch {
if fullShape {
// 屏障停在发布已写完、校验尚未开始时:此时目标里是"这次发布真正写进去的
// 内容"。在这里独立读回十表并核 LocalIDs,而不是等回滚后拿结果反推。
applied := r38TargetRows(t, fx)
appliedRows = applied
save("08b-applied-pg.json", applied)
appliedIndex = recoveryIndexReadback(t, fx)
save("08f-index-applied.json", appliedIndex)
save("08c-applied-reference-promises.json", r38RequireReferences(t, "applied publish rows", applied))
ledger := r38FindWriteSet(t, archiveRoot, job.ID)
mappingDiffs := ds02CheckLocalIDMapping(r38LocalColumns(t, fx, spec, expectation), ledger.LocalIDs)
@@ -768,10 +807,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if objectBranch {
objectProxy.setArmed(true)
}
if pgBranch {
pgFault = r38ArmPGFault(t, fx, conflictLog)
save("08e-pg-fault-armed.json", pgFault)
}
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
"verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch,
"refuse_object_restore_dispatch": objectBranch, "scenario": scenario})
"verify_corruption_armed": true, "refuse_clean_dispatch": !fullShape,
"refuse_object_restore_dispatch": objectBranch, "refuse_pg_restore": pgBranch, "scenario": scenario})
released = true
break
}
@@ -795,7 +838,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
}
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
if objectBranch {
switch {
case objectBranch:
// 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") {
t.Fatalf("publish failure does not name the refused object restore: %q", failureText)
@@ -803,8 +847,24 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText)
}
} else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
case pgBranch:
// 失败必须来自 PG 恢复步骤本身:恢复阶段 database + 真实 SQLSTATE,而不是 ES/对象文案。
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "恢复阶段 wiki 失败") {
t.Fatalf("publish failure does not name the failed PG restore stage: %q", failureText)
}
if !strings.Contains(failureText, "SQLSTATE "+r38PGSQLState) {
t.Fatalf("publish failure carries no SQLSTATE %s: %q", r38PGSQLState, failureText)
}
if strings.Contains(failureText, "index effect outcome unknown") || strings.Contains(failureText, "conditional object cleanup not confirmed") {
t.Fatalf("pg branch was mislabelled as an ES or object failure: %q", failureText)
}
save("09a-pg-failure.json", map[string]any{"job": job.ID, "error": libraryAfterPublish.Error,
"recovery": libraryAfterPublish.Recovery, "failed_stage": libraryAfterPublish.Stage,
"expected_sqlstate": r38PGSQLState, "fault_armed": pgFault, "matched_sqlstate": true})
default:
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
}
}
if string(libraryAfterPublish.Stage) != "rollback" {
t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage)
@@ -843,16 +903,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("scope maintenance = %+v", scopeRow)
}
save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow})
if pgBranch {
// 失败现场必须保留:故障未自行解除,失败事务没有提交,目标仍是发布写入后的内容。
save("11a-pg-failed-restore.json", r38PGFaultHeld(t, fx, beforeTarget, appliedRows, failureText, conflictLog))
}
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
save("12-index-effects-held.json", effects)
if objectBranch {
switch {
case fullShape:
// 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。
if _, ok := effects["restored"]; !ok {
t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects))
}
} else {
default:
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
if _, ok := effects[phase]; !ok {
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
@@ -870,7 +935,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
}
heldSource := heldDoc["_source"].(map[string]any)
heldContent, _ := heldSource["content"].(string)
if objectBranch {
if fullShape {
// 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。
if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
t.Fatalf("object branch left the ES restore marker in place: %q", heldContent)
@@ -885,7 +950,9 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
events, refused := proxy.snapshot()
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
var heldObjects *s3.ListObjectVersionsOutput
if objectBranch {
if fullShape {
heldObjects = objectInventoryR38(t, ctx, fx)
save("13a-objects-held.json", heldObjects)
if refused != 0 {
t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused)
}
@@ -901,7 +968,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
forwarded++
}
}
if !objectBranch && forwarded != 0 {
if !fullShape && forwarded != 0 {
t.Fatalf("refused clean write was forwarded %d times", forwarded)
}
if objectBranch {
@@ -956,8 +1023,6 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if _, ok := deleteIntent["delete-intent"]; !ok {
t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent))
}
heldObjects = objectInventoryR38(t, ctx, fx)
save("13a-objects-held.json", heldObjects)
heldKeys := map[string]bool{}
for _, version := range heldObjects.Versions {
heldKeys[aws.ToString(version.Key)] = true
@@ -1018,9 +1083,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if objectBranch {
objectProxy.setArmed(false)
}
if pgBranch {
// 只解除本轮注入的 PG 故障(drop trigger/function/switch table),不动共享 PG 本身。
save("17b-pg-fault-disarmed.json", r38DisarmPGFault(t, fx, conflictLog))
}
save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal",
"backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch,
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
"pg_fault_removed": pgBranch,
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
map[string]any{"scope": "internal", "window_job_id": job.ID})
if code != http.StatusAccepted {
@@ -1090,7 +1160,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if diff := diffUniverse(beforePG, afterPG); diff != "" {
t.Fatalf("PG not restored row-by-row: %s", diff)
}
if objectBranch {
if fullShape {
// 独立 oracle:目标范围十表必须逐列等于声明 before-image(不是"和上一份读回相同"),
// 邻库/外部租户哨兵必须原样保留,恢复后的引用承诺逐种 matched>0。
restored := r38TargetRows(t, fx)
@@ -1123,7 +1193,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v",
objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects))
}
if objectBranch {
if fullShape {
// 字节指纹:对 before-image 里每个受保护 versionId 单独 GET 算 SHA256,恢复后
// 同 version 再独立算一次;ETag/size 只当旁证,绝不拿元数据当"逐字节相同"。
afterDigests := r38ObjectVersionDigests(t, ctx, fx, afterObjects)
@@ -1172,7 +1242,33 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
}
}
if operation == "" {
t.Fatal("object branch lost the refused operation identity")
// PG 分支在 held 时还没走到对象删除,没有"被拒请求"可以回溯;按发布新增的
// key@version 精确匹配 removed 回执,拿到的仍是真实网关写入的操作身份。
wanted := map[string]bool{}
for _, id := range createdVersions {
wanted[id] = true
}
rows, e := gov.Query(ctx, `SELECT receipt FROM kg_object_gateway_receipts`)
must(e)
for rows.Next() {
var raw []byte
must(rows.Scan(&raw))
var receipt objectgateway.Receipt
must(json.Unmarshal(raw, &receipt))
if receipt.State == "removed" && wanted[receipt.Request.Key+"@"+receipt.Version.ID] {
operation = receipt.Request.Operation
}
}
must(rows.Err())
rows.Close()
if objectBranch {
t.Fatal("object branch lost the refused operation identity")
}
if operation == "" {
t.Fatal("pg branch found no removed receipt bound to a version created by the failed publish")
}
save("23d-removal-bound-by-receipt.json", map[string]any{"operation": operation,
"created_versions": createdVersions})
}
var removedRaw []byte
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw))
@@ -1235,7 +1331,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
if len(receipts) == 0 || len(deliveries) == 0 {
t.Fatal("no gateway receipt/delivery evidence for real object writes")
}
if objectBranch {
if fullShape {
// 发布期与恢复期两条独立记录必须指向同一套 mapping:发布时落盘的写集合摘要,
// 必须同时出现在作业锚点、预备份版本证据、网关回执/投递计划里;恢复意图的
// request_digest 由窗口/作业/目标/来源/备份/该计划独立重算并与持久记录逐字符比较。
@@ -1399,7 +1495,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
!reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(replayObjects)) {
t.Fatal("replay rewrote object versions or delete markers")
}
if objectBranch {
if fullShape {
replayDigests := r38ObjectVersionDigests(t, ctx, fx, replayObjects)
save("31a-object-byte-digests-after-replay.json", replayDigests)
if !reflect.DeepEqual(beforeDigests, replayDigests) {
@@ -1416,7 +1512,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
_ = worker.Wait()
objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot()
save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
"window_job": job.ID, "publish_queue": queueID,
"pg_restore_branch": pgBranch,
"window_job": job.ID, "publish_queue": queueID,
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
@@ -1429,8 +1526,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
"run": map[string]any{"label": os.Getenv("KG_R38_LABEL"), "commit": os.Getenv("KG_R38_COMMIT"),
"source_tree": os.Getenv("KG_R38_SOURCE_TREE"), "version": os.Getenv("KG_R38_VERSION"),
"release_version": os.Getenv("KG_R38_RELEASE_VERSION")},
"full_shape": map[string]any{"ten_table_before_nonempty": objectBranch,
"ten_table_source_nonempty": objectBranch, "before_tables": len(beforePG),
"full_shape": map[string]any{"ten_table_before_nonempty": fullShape,
"ten_table_source_nonempty": fullShape, "before_tables": len(beforePG),
"source_parts": sourceParts, "reference_promises": "all matched>0",
"local_id_mapping": "publish ledger vs independent oracle + recovery digest",
"object_byte_sha256_versions": len(beforeDigests)}})
@@ -1611,3 +1708,151 @@ func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *write
deliveryRows.Close()
return receipts, deliveries
}
// ---------- PG 恢复故障注入(只在本轮一次性业务库) ----------
const r38PGSQLState = "23505"
// r38PGFaultInsert 是恢复自己的 wiki 阶段写法(jsonb_populate_record + SELECT *),
// 这里用同一条语句做"故障探针",证明报错来自恢复会执行的那类 SQL。
const r38PGFaultInsert = `INSERT INTO wiki_log_entries SELECT * FROM jsonb_populate_record(NULL::wiki_log_entries, $1::jsonb)`
// r38ArmPGFault 是唯一新增的故障:before-image 的一条 wiki 日志在恢复的最后一段 PG 写入里
// 会按原 id 插回目标范围;先用一行"外部范围、同主键"的日志占住这个 id,恢复的真实 INSERT
// 就会得到 23505。它不改 schema(不加触发器/约束/索引)、不停共享 PG、不动 ACL;解除就是删这一行。
func r38ArmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any {
t.Helper()
if logID == 0 {
t.Fatal("pg fault armed without a wiki log identity")
}
var raw []byte
if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x ORDER BY id LIMIT 1`).Scan(&raw); e != nil {
t.Fatalf("pg fault needs one existing wiki log row as a shape template: %v", e)
}
var row map[string]any
if e := json.Unmarshal(raw, &row); e != nil {
t.Fatal(e)
}
row["id"] = logID
row["tenant_id"] = 20000
row["knowledge_base_id"] = "kb-r38-foreign"
encoded, e := json.Marshal(row)
if e != nil {
t.Fatal(e)
}
if _, e := fx.pool.Exec(fx.ctx, r38PGFaultInsert, encoded); e != nil {
t.Fatalf("arm pg fault (foreign row holding wiki log %d): %v", logID, e)
}
var tenant int64
var kb string
if e := fx.pool.QueryRow(fx.ctx, `SELECT tenant_id,knowledge_base_id FROM wiki_log_entries WHERE id=$1`, logID).Scan(&tenant, &kb); e != nil {
t.Fatal(e)
}
if tenant != 20000 || kb != "kb-r38-foreign" {
t.Fatalf("pg fault row readback = %d/%s, want 20000/kb-r38-foreign", tenant, kb)
}
return map[string]any{"stage": "wiki", "table": "wiki_log_entries", "conflict_id": logID,
"conflict_tenant": tenant, "conflict_kb": kb, "insert_statement": r38PGFaultInsert,
"expected_sqlstate": r38PGSQLState,
"mechanism": "foreign-scope row holds the primary key the recovery wiki stage restores",
"schema_changed": false,
"scope": "one disposable business database; shared PG, ACL and sessions untouched"}
}
// r38HeldAppliedTables 是"wiki 阶段失败时仍保持发布写入内容"的表:wiki_* 属于失败并回滚的
// 那一阶段;resources 是多知识库共享表,恢复把它的精确清理推迟到四阶段全部成功之后,
// 所以此刻仍带着发布新增的资源行。其余表都属于已提交的 database 阶段。
func r38HeldAppliedTables() map[string]bool {
return map[string]bool{"wiki_folders": true, "wiki_pages": true,
"wiki_page_issues": true, "wiki_log_entries": true, "resources": true}
}
// r38PGFaultHeld 证明三件事:故障行还在现场;同一 INSERT 再执行一次仍拿到服务器 SQLSTATE
// 且没有多出行;失败后目标是一个真实的"部分恢复"状态——database 阶段已提交(非 wiki 表
// 回到 before-image),wiki 阶段失败并整体回滚(wiki 表仍是发布写入后的内容)。
func r38PGFaultHeld(t *testing.T, fx *writeSetFixture, before, applied map[string][]map[string]any, failureText string, logID int64) map[string]any {
t.Helper()
if len(before) == 0 || len(applied) == 0 {
t.Fatal("held pg evidence needs both the before-image and the applied readback")
}
var rows int
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`,
logID).Scan(&rows); e != nil {
t.Fatal(e)
}
if rows != 1 {
t.Fatalf("pg fault row was not preserved at held: %d rows for wiki log %d", rows, logID)
}
var raw []byte
if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x WHERE id=$1`, logID).Scan(&raw); e != nil {
t.Fatal(e)
}
_, probeErr := fx.pool.Exec(fx.ctx, r38PGFaultInsert, raw)
var pgErr *pgconn.PgError
if probeErr == nil || !errors.As(probeErr, &pgErr) {
t.Fatalf("fault probe did not raise a PG error: %v", probeErr)
}
if pgErr.Code != r38PGSQLState {
t.Fatalf("fault probe SQLSTATE = %q, want %s", pgErr.Code, r38PGSQLState)
}
var probeRows int
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1`, logID).Scan(&probeRows); e != nil {
t.Fatal(e)
}
if probeRows != 1 {
t.Fatalf("failed probe left %d rows for wiki log %d: the statement transaction did not roll back", probeRows, logID)
}
held := r38TargetRows(t, fx)
stillApplied := r38HeldAppliedTables()
committed, rolledBack := []string{}, []string{}
for _, table := range ds02TargetTables {
want, label := before[table], "before-image"
if stillApplied[table] {
want, label = applied[table], "applied publish"
}
if diff := r38CompareRows("held "+table, want, held[table]); diff != "" {
t.Fatalf("held %s does not match the %s (earlier stages commit per stage, the failed stage rolls back): %s", table, label, diff)
}
if stillApplied[table] {
rolledBack = append(rolledBack, table)
} else {
committed = append(committed, table)
}
}
encoded, e := json.Marshal(held)
if e != nil {
t.Fatal(e)
}
sum := sha256.Sum256(encoded)
counts := map[string]int{}
for table, values := range held {
counts[table] = len(values)
}
return map[string]any{"conflict_id": logID, "conflict_rows": rows,
"probe_statement": r38PGFaultInsert, "sqlstate": pgErr.Code, "probe_error": pgErr.Error(),
"probe_rows_after_error": probeRows,
"transaction": "wiki stage INSERT failed; that stage transaction rolled back; earlier database stage stayed committed",
"before_image_tables": committed, "still_applied_tables": rolledBack,
"held_row_counts": counts, "held_pg_sha256": hex.EncodeToString(sum[:]),
"matches_before_and_applied": true,
"failure_text_has_sqlstate": strings.Contains(failureText, "SQLSTATE "+r38PGSQLState),
"failure_text_has_stage": strings.Contains(failureText, "恢复阶段 wiki 失败")}
}
func r38DisarmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any {
t.Helper()
statement := `DELETE FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`
of, e := fx.pool.Exec(fx.ctx, statement, logID)
if e != nil {
t.Fatal(e)
}
var left int
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`).Scan(&left); e != nil {
t.Fatal(e)
}
if of.RowsAffected() != 1 || left != 0 {
t.Fatalf("pg fault not removed: deleted=%d left=%d", of.RowsAffected(), left)
}
return map[string]any{"statement": statement, "deleted": of.RowsAffected(), "foreign_rows_left": left,
"readback_absent": true}
}
@@ -203,6 +203,10 @@ func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writ
index := "kg_library_test_" + hex.EncodeToString(token)
if options.index != "" {
index = options.index
} else if registered := os.Getenv("KG_R38_INDEX"); registered != "" {
// 隔离运行器在创建前登记了唯一索引;夹具必须消费它,否则 wrapper 清理的是
// 一个从未被使用的名字(真索引只能靠 t.Cleanup,进程提前退出就漏掉)。
index = registered
}
// Register each disposable namespace before its first ES/object mutation.
r10Save(t, "namespace", map[string]string{"bucket": wsBucket, "index": index})