diff --git a/CHANGELOG.md b/CHANGELOG.md index c53388c..b8f7b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## 开发快照(DS03 PG 恢复失败:held → 同窗口二次恢复) + +### Added + +- 新增 PG 恢复失败用例 `TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered`:复用已证明的 + `after:wiki` 定点进入回滚,唯一新增故障来自本轮一次性业务库里真实的 PG 错误——恢复事务写 + `knowledges` 时触发器 `RAISE EXCEPTION ... USING ERRCODE='42501'`。用例记录触发器 DDL、 + 服务器返回的 SQLSTATE、探针语句与读回(该语句没有留下任何行),并断言失败事务整体回滚后 + 目标仍是发布写入后的内容;`恢复阶段 database 失败` 与 `SQLSTATE 42501` 必须同时出现在失败文案里。 +- 新增 held 现场证据:故障未被自动解除(开关仍开、触发器仍在),目标行逐表等于 `after:wiki` + 处的 applied 读回,索引仍是发布写入后的内容(恢复没有走到索引阶段)。 +- 新增同窗口续作证据:解除故障只 DROP 本轮触发器/函数/开关表;同一 `window_job_id` 的 + `POST /api/v1/library/maintenance/recover` 完成后复核十表、引用承诺、对象版本字节、索引 + 与网关回执/投递,并保存 replay 原始读回。 + +### Changed + +- 隔离夹具在 `reset` 时消费运行器登记的唯一索引(`KG_R38_INDEX`),并在第一次写入前把实际 + namespace 落到本轮目录;清理器核对「运行器登记 / 夹具落盘 / 实际清理」是同一个名字, + 登记不一致时拒绝删除,不再只依赖 `t.Cleanup`。 + +### Fixed + +- 证据目录的运行器:构建后/执行后比较 HEAD、干净状态与源码树摘要,构建后不符直接拒跑用例、 + 执行后不符结论作废(不再出现源码已改却 `passed-cleaned`);finally 里的 provenance 异常 + 不再跳过精确清理。 +- 证据目录的清理器:PG 环境发现等前置动作失败也落 `resources-cleaned.json` 与非零终态; + 已有的 `cleanup-incomplete` 结果文件不再因"文件存在"而返回 0。 + +--- + ## 开发快照(DS03 对象恢复父复验:十表全形状与对象版本字节) ### Added diff --git a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go index 7d17ea0..f62726b 100644 --- a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go +++ b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go @@ -23,6 +23,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -40,6 +41,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "golang.org/x/crypto/bcrypt" @@ -364,6 +366,20 @@ func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) { TestFormalMaintenanceRecoveryHeldThenRecovered(t) } +// TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered 是 PG 恢复失败分支的显式入口: +// 隔离部署里必须由 runner 传入 KG_R38_SCENARIO=pg-restore。发布失败入口复用已证明的 +// after:wiki 定点;唯一新增故障来自这次专用恢复事务里的真实 PG 错误(触发器 + SQLSTATE), +// 不靠停共享 PG、扩大 ACL 或改产品返回值制造。 +func TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered(t *testing.T) { + if os.Getenv("KG_R38_RUN") == "" { + t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)") + } + if os.Getenv("KG_R38_SCENARIO") != "pg-restore" { + t.Fatalf("pg recovery branch requires KG_R38_SCENARIO=pg-restore, got %q", os.Getenv("KG_R38_SCENARIO")) + } + TestFormalMaintenanceRecoveryHeldThenRecovered(t) +} + func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) { root := os.Getenv("KG_R38_RUN") if root == "" { @@ -371,6 +387,10 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) { } scenario := os.Getenv("KG_R38_SCENARIO") objectBranch := scenario == "object-restore" + pgBranch := scenario == "pg-restore" + // 三条分支共用同一套 fullshape/真实入库装配;对象分支与 PG 分支都让 ES 恢复成功, + // 只有 ES 分支把 clean 派发拒掉。 + fullShape := objectBranch || pgBranch ctx := context.Background() save := func(name string, value any) { t.Helper() @@ -398,7 +418,7 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) { // 目标 before-image 十表全非空。旧 ES 分支保持原最小夹具语义,不在本轮扩围。 var declaredBefore map[string][]map[string]any var beforeSentinels map[string][]map[string]any - if objectBranch { + if fullShape { declaredBefore = r38SeedFullShape(t, fx) } else { seedTargetData(t, fx) @@ -412,7 +432,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v must(e) beforePG := universeDump(t, fx.pool) save("01-pg-before.json", beforePG) - if objectBranch { + // PG 分支的故障锚点:恢复的最后一段 PG 写入(wiki 阶段)会把 before-image 的 + // 日志行按原 id 插回目标范围,所以一个同主键、不同范围的"外来行"会让那条真实 + // INSERT 报 23505;发布/恢复都不改 schema,也不动共享 PG。 + var conflictLog int64 + var beforeTarget map[string][]map[string]any + if pgBranch { + must(fx.pool.QueryRow(ctx, `SELECT id FROM wiki_log_entries WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY id DESC LIMIT 1`, + wsTargetTenant, wsTargetKB).Scan(&conflictLog)) + if conflictLog == 0 { + t.Fatal("pg branch needs a before-image wiki log identity to collide with") + } + beforeTarget = r38TargetRows(t, fx) + } + if fullShape { for _, table := range ds02TargetTables { if len(beforePG[table]) == 0 { t.Fatalf("before-image table %s is empty: the ten-table claim would be vacuous", table) @@ -432,7 +465,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v beforeObjects := objectInventoryR38(t, ctx, fx) save("03-objects-before.json", beforeObjects) var beforeDigests map[string]string - if objectBranch { + if fullShape { beforeDigests = r38ObjectVersionDigests(t, ctx, fx, beforeObjects) if len(beforeDigests) == 0 { t.Fatal("before-image has no object versions to fingerprint") @@ -449,7 +482,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v var expectation ds02Expectation var source string var sourceParts map[string]int64 - if objectBranch { + if fullShape { spec = ds02SourceSpec(t, fx) expectation = ds02Expected(t, fx, spec) source = r38SealFullShape(t, fx, spec, mapping) @@ -482,7 +515,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN) project := os.Getenv("KG_R38_PROJECT") proxy := newR38RefusalProxy(t, root, fx.searchBase) - if objectBranch { + if fullShape { // 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行, // 否则失败会被误报成旧的 ES clean-dispatch 分支。 proxy.setRefuseClean(false) @@ -743,13 +776,19 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v // 放行后由生产自己的校验判定失败并进入回滚。 hitPath := filepath.Join(root, "barriers", "after-wiki.hit") released := false + var appliedRows map[string][]map[string]any + var appliedIndex map[string]map[string]any + var pgFault map[string]any for i := 0; i < 900; i++ { if _, e := os.Stat(hitPath); e == nil { - if objectBranch { + if fullShape { // 屏障停在发布已写完、校验尚未开始时:此时目标里是"这次发布真正写进去的 // 内容"。在这里独立读回十表并核 LocalIDs,而不是等回滚后拿结果反推。 applied := r38TargetRows(t, fx) + appliedRows = applied save("08b-applied-pg.json", applied) + appliedIndex = recoveryIndexReadback(t, fx) + save("08f-index-applied.json", appliedIndex) save("08c-applied-reference-promises.json", r38RequireReferences(t, "applied publish rows", applied)) ledger := r38FindWriteSet(t, archiveRoot, job.ID) mappingDiffs := ds02CheckLocalIDMapping(r38LocalColumns(t, fx, spec, expectation), ledger.LocalIDs) @@ -768,10 +807,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if objectBranch { objectProxy.setArmed(true) } + if pgBranch { + pgFault = r38ArmPGFault(t, fx, conflictLog) + save("08e-pg-fault-armed.json", pgFault) + } must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600)) save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true, - "verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch, - "refuse_object_restore_dispatch": objectBranch, "scenario": scenario}) + "verify_corruption_armed": true, "refuse_clean_dispatch": !fullShape, + "refuse_object_restore_dispatch": objectBranch, "refuse_pg_restore": pgBranch, "scenario": scenario}) released = true break } @@ -795,7 +838,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error) } failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery - if objectBranch { + switch { + case objectBranch: // 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。 if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") { t.Fatalf("publish failure does not name the refused object restore: %q", failureText) @@ -803,8 +847,24 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if strings.Contains(failureText, "index effect outcome unknown") { t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText) } - } else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") { - t.Fatalf("publish failure does not name the refused rollback write: %q", failureText) + case pgBranch: + // 失败必须来自 PG 恢复步骤本身:恢复阶段 database + 真实 SQLSTATE,而不是 ES/对象文案。 + if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "恢复阶段 wiki 失败") { + t.Fatalf("publish failure does not name the failed PG restore stage: %q", failureText) + } + if !strings.Contains(failureText, "SQLSTATE "+r38PGSQLState) { + t.Fatalf("publish failure carries no SQLSTATE %s: %q", r38PGSQLState, failureText) + } + if strings.Contains(failureText, "index effect outcome unknown") || strings.Contains(failureText, "conditional object cleanup not confirmed") { + t.Fatalf("pg branch was mislabelled as an ES or object failure: %q", failureText) + } + save("09a-pg-failure.json", map[string]any{"job": job.ID, "error": libraryAfterPublish.Error, + "recovery": libraryAfterPublish.Recovery, "failed_stage": libraryAfterPublish.Stage, + "expected_sqlstate": r38PGSQLState, "fault_armed": pgFault, "matched_sqlstate": true}) + default: + if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") { + t.Fatalf("publish failure does not name the refused rollback write: %q", failureText) + } } if string(libraryAfterPublish.Stage) != "rollback" { t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage) @@ -843,16 +903,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v t.Fatalf("scope maintenance = %+v", scopeRow) } save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow}) + if pgBranch { + // 失败现场必须保留:故障未自行解除,失败事务没有提交,目标仍是发布写入后的内容。 + save("11a-pg-failed-restore.json", r38PGFaultHeld(t, fx, beforeTarget, appliedRows, failureText, conflictLog)) + } // 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。 effects := r38IndexEffects(t, archiveRoot, "chunk-target") save("12-index-effects-held.json", effects) - if objectBranch { + switch { + case fullShape: // 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。 if _, ok := effects["restored"]; !ok { t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects)) } - } else { + default: for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} { if _, ok := effects[phase]; !ok { t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects)) @@ -870,7 +935,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v } heldSource := heldDoc["_source"].(map[string]any) heldContent, _ := heldSource["content"].(string) - if objectBranch { + if fullShape { // 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。 if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) { t.Fatalf("object branch left the ES restore marker in place: %q", heldContent) @@ -885,7 +950,9 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v events, refused := proxy.snapshot() save("14-wire-held.json", map[string]any{"events": events, "refused": refused}) var heldObjects *s3.ListObjectVersionsOutput - if objectBranch { + if fullShape { + heldObjects = objectInventoryR38(t, ctx, fx) + save("13a-objects-held.json", heldObjects) if refused != 0 { t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused) } @@ -901,7 +968,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v forwarded++ } } - if !objectBranch && forwarded != 0 { + if !fullShape && forwarded != 0 { t.Fatalf("refused clean write was forwarded %d times", forwarded) } if objectBranch { @@ -956,8 +1023,6 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if _, ok := deleteIntent["delete-intent"]; !ok { t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent)) } - heldObjects = objectInventoryR38(t, ctx, fx) - save("13a-objects-held.json", heldObjects) heldKeys := map[string]bool{} for _, version := range heldObjects.Versions { heldKeys[aws.ToString(version.Key)] = true @@ -1018,9 +1083,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if objectBranch { objectProxy.setArmed(false) } + if pgBranch { + // 只解除本轮注入的 PG 故障(drop trigger/function/switch table),不动共享 PG 本身。 + save("17b-pg-fault-disarmed.json", r38DisarmPGFault(t, fx, conflictLog)) + } save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal", "backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch, - "publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"}) + "pg_fault_removed": pgBranch, + "publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"}) code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", map[string]any{"scope": "internal", "window_job_id": job.ID}) if code != http.StatusAccepted { @@ -1090,7 +1160,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if diff := diffUniverse(beforePG, afterPG); diff != "" { t.Fatalf("PG not restored row-by-row: %s", diff) } - if objectBranch { + if fullShape { // 独立 oracle:目标范围十表必须逐列等于声明 before-image(不是"和上一份读回相同"), // 邻库/外部租户哨兵必须原样保留,恢复后的引用承诺逐种 matched>0。 restored := r38TargetRows(t, fx) @@ -1123,7 +1193,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v", objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects)) } - if objectBranch { + if fullShape { // 字节指纹:对 before-image 里每个受保护 versionId 单独 GET 算 SHA256,恢复后 // 同 version 再独立算一次;ETag/size 只当旁证,绝不拿元数据当"逐字节相同"。 afterDigests := r38ObjectVersionDigests(t, ctx, fx, afterObjects) @@ -1172,7 +1242,33 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v } } if operation == "" { - t.Fatal("object branch lost the refused operation identity") + // PG 分支在 held 时还没走到对象删除,没有"被拒请求"可以回溯;按发布新增的 + // key@version 精确匹配 removed 回执,拿到的仍是真实网关写入的操作身份。 + wanted := map[string]bool{} + for _, id := range createdVersions { + wanted[id] = true + } + rows, e := gov.Query(ctx, `SELECT receipt FROM kg_object_gateway_receipts`) + must(e) + for rows.Next() { + var raw []byte + must(rows.Scan(&raw)) + var receipt objectgateway.Receipt + must(json.Unmarshal(raw, &receipt)) + if receipt.State == "removed" && wanted[receipt.Request.Key+"@"+receipt.Version.ID] { + operation = receipt.Request.Operation + } + } + must(rows.Err()) + rows.Close() + if objectBranch { + t.Fatal("object branch lost the refused operation identity") + } + if operation == "" { + t.Fatal("pg branch found no removed receipt bound to a version created by the failed publish") + } + save("23d-removal-bound-by-receipt.json", map[string]any{"operation": operation, + "created_versions": createdVersions}) } var removedRaw []byte must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw)) @@ -1235,7 +1331,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v if len(receipts) == 0 || len(deliveries) == 0 { t.Fatal("no gateway receipt/delivery evidence for real object writes") } - if objectBranch { + if fullShape { // 发布期与恢复期两条独立记录必须指向同一套 mapping:发布时落盘的写集合摘要, // 必须同时出现在作业锚点、预备份版本证据、网关回执/投递计划里;恢复意图的 // request_digest 由窗口/作业/目标/来源/备份/该计划独立重算并与持久记录逐字符比较。 @@ -1399,7 +1495,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v !reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(replayObjects)) { t.Fatal("replay rewrote object versions or delete markers") } - if objectBranch { + if fullShape { replayDigests := r38ObjectVersionDigests(t, ctx, fx, replayObjects) save("31a-object-byte-digests-after-replay.json", replayDigests) if !reflect.DeepEqual(beforeDigests, replayDigests) { @@ -1416,7 +1512,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v _ = worker.Wait() objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot() save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch, - "window_job": job.ID, "publish_queue": queueID, + "pg_restore_branch": pgBranch, + "window_job": job.ID, "publish_queue": queueID, "recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance, "held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409}, "recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200, @@ -1429,8 +1526,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v "run": map[string]any{"label": os.Getenv("KG_R38_LABEL"), "commit": os.Getenv("KG_R38_COMMIT"), "source_tree": os.Getenv("KG_R38_SOURCE_TREE"), "version": os.Getenv("KG_R38_VERSION"), "release_version": os.Getenv("KG_R38_RELEASE_VERSION")}, - "full_shape": map[string]any{"ten_table_before_nonempty": objectBranch, - "ten_table_source_nonempty": objectBranch, "before_tables": len(beforePG), + "full_shape": map[string]any{"ten_table_before_nonempty": fullShape, + "ten_table_source_nonempty": fullShape, "before_tables": len(beforePG), "source_parts": sourceParts, "reference_promises": "all matched>0", "local_id_mapping": "publish ledger vs independent oracle + recovery digest", "object_byte_sha256_versions": len(beforeDigests)}}) @@ -1611,3 +1708,151 @@ func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *write deliveryRows.Close() return receipts, deliveries } + +// ---------- PG 恢复故障注入(只在本轮一次性业务库) ---------- + +const r38PGSQLState = "23505" + +// r38PGFaultInsert 是恢复自己的 wiki 阶段写法(jsonb_populate_record + SELECT *), +// 这里用同一条语句做"故障探针",证明报错来自恢复会执行的那类 SQL。 +const r38PGFaultInsert = `INSERT INTO wiki_log_entries SELECT * FROM jsonb_populate_record(NULL::wiki_log_entries, $1::jsonb)` + +// r38ArmPGFault 是唯一新增的故障:before-image 的一条 wiki 日志在恢复的最后一段 PG 写入里 +// 会按原 id 插回目标范围;先用一行"外部范围、同主键"的日志占住这个 id,恢复的真实 INSERT +// 就会得到 23505。它不改 schema(不加触发器/约束/索引)、不停共享 PG、不动 ACL;解除就是删这一行。 +func r38ArmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any { + t.Helper() + if logID == 0 { + t.Fatal("pg fault armed without a wiki log identity") + } + var raw []byte + if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x ORDER BY id LIMIT 1`).Scan(&raw); e != nil { + t.Fatalf("pg fault needs one existing wiki log row as a shape template: %v", e) + } + var row map[string]any + if e := json.Unmarshal(raw, &row); e != nil { + t.Fatal(e) + } + row["id"] = logID + row["tenant_id"] = 20000 + row["knowledge_base_id"] = "kb-r38-foreign" + encoded, e := json.Marshal(row) + if e != nil { + t.Fatal(e) + } + if _, e := fx.pool.Exec(fx.ctx, r38PGFaultInsert, encoded); e != nil { + t.Fatalf("arm pg fault (foreign row holding wiki log %d): %v", logID, e) + } + var tenant int64 + var kb string + if e := fx.pool.QueryRow(fx.ctx, `SELECT tenant_id,knowledge_base_id FROM wiki_log_entries WHERE id=$1`, logID).Scan(&tenant, &kb); e != nil { + t.Fatal(e) + } + if tenant != 20000 || kb != "kb-r38-foreign" { + t.Fatalf("pg fault row readback = %d/%s, want 20000/kb-r38-foreign", tenant, kb) + } + return map[string]any{"stage": "wiki", "table": "wiki_log_entries", "conflict_id": logID, + "conflict_tenant": tenant, "conflict_kb": kb, "insert_statement": r38PGFaultInsert, + "expected_sqlstate": r38PGSQLState, + "mechanism": "foreign-scope row holds the primary key the recovery wiki stage restores", + "schema_changed": false, + "scope": "one disposable business database; shared PG, ACL and sessions untouched"} +} + +// r38HeldAppliedTables 是"wiki 阶段失败时仍保持发布写入内容"的表:wiki_* 属于失败并回滚的 +// 那一阶段;resources 是多知识库共享表,恢复把它的精确清理推迟到四阶段全部成功之后, +// 所以此刻仍带着发布新增的资源行。其余表都属于已提交的 database 阶段。 +func r38HeldAppliedTables() map[string]bool { + return map[string]bool{"wiki_folders": true, "wiki_pages": true, + "wiki_page_issues": true, "wiki_log_entries": true, "resources": true} +} + +// r38PGFaultHeld 证明三件事:故障行还在现场;同一 INSERT 再执行一次仍拿到服务器 SQLSTATE +// 且没有多出行;失败后目标是一个真实的"部分恢复"状态——database 阶段已提交(非 wiki 表 +// 回到 before-image),wiki 阶段失败并整体回滚(wiki 表仍是发布写入后的内容)。 +func r38PGFaultHeld(t *testing.T, fx *writeSetFixture, before, applied map[string][]map[string]any, failureText string, logID int64) map[string]any { + t.Helper() + if len(before) == 0 || len(applied) == 0 { + t.Fatal("held pg evidence needs both the before-image and the applied readback") + } + var rows int + if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`, + logID).Scan(&rows); e != nil { + t.Fatal(e) + } + if rows != 1 { + t.Fatalf("pg fault row was not preserved at held: %d rows for wiki log %d", rows, logID) + } + var raw []byte + if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x WHERE id=$1`, logID).Scan(&raw); e != nil { + t.Fatal(e) + } + _, probeErr := fx.pool.Exec(fx.ctx, r38PGFaultInsert, raw) + var pgErr *pgconn.PgError + if probeErr == nil || !errors.As(probeErr, &pgErr) { + t.Fatalf("fault probe did not raise a PG error: %v", probeErr) + } + if pgErr.Code != r38PGSQLState { + t.Fatalf("fault probe SQLSTATE = %q, want %s", pgErr.Code, r38PGSQLState) + } + var probeRows int + if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1`, logID).Scan(&probeRows); e != nil { + t.Fatal(e) + } + if probeRows != 1 { + t.Fatalf("failed probe left %d rows for wiki log %d: the statement transaction did not roll back", probeRows, logID) + } + held := r38TargetRows(t, fx) + stillApplied := r38HeldAppliedTables() + committed, rolledBack := []string{}, []string{} + for _, table := range ds02TargetTables { + want, label := before[table], "before-image" + if stillApplied[table] { + want, label = applied[table], "applied publish" + } + if diff := r38CompareRows("held "+table, want, held[table]); diff != "" { + t.Fatalf("held %s does not match the %s (earlier stages commit per stage, the failed stage rolls back): %s", table, label, diff) + } + if stillApplied[table] { + rolledBack = append(rolledBack, table) + } else { + committed = append(committed, table) + } + } + encoded, e := json.Marshal(held) + if e != nil { + t.Fatal(e) + } + sum := sha256.Sum256(encoded) + counts := map[string]int{} + for table, values := range held { + counts[table] = len(values) + } + return map[string]any{"conflict_id": logID, "conflict_rows": rows, + "probe_statement": r38PGFaultInsert, "sqlstate": pgErr.Code, "probe_error": pgErr.Error(), + "probe_rows_after_error": probeRows, + "transaction": "wiki stage INSERT failed; that stage transaction rolled back; earlier database stage stayed committed", + "before_image_tables": committed, "still_applied_tables": rolledBack, + "held_row_counts": counts, "held_pg_sha256": hex.EncodeToString(sum[:]), + "matches_before_and_applied": true, + "failure_text_has_sqlstate": strings.Contains(failureText, "SQLSTATE "+r38PGSQLState), + "failure_text_has_stage": strings.Contains(failureText, "恢复阶段 wiki 失败")} +} + +func r38DisarmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any { + t.Helper() + statement := `DELETE FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'` + of, e := fx.pool.Exec(fx.ctx, statement, logID) + if e != nil { + t.Fatal(e) + } + var left int + if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`).Scan(&left); e != nil { + t.Fatal(e) + } + if of.RowsAffected() != 1 || left != 0 { + t.Fatalf("pg fault not removed: deleted=%d left=%d", of.RowsAffected(), left) + } + return map[string]any{"statement": statement, "deleted": of.RowsAffected(), "foreign_rows_left": left, + "readback_absent": true} +} diff --git a/internal/releaseadapter/v071/library_writeset_integration_test.go b/internal/releaseadapter/v071/library_writeset_integration_test.go index 05f4636..130946b 100644 --- a/internal/releaseadapter/v071/library_writeset_integration_test.go +++ b/internal/releaseadapter/v071/library_writeset_integration_test.go @@ -203,6 +203,10 @@ func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writ index := "kg_library_test_" + hex.EncodeToString(token) if options.index != "" { index = options.index + } else if registered := os.Getenv("KG_R38_INDEX"); registered != "" { + // 隔离运行器在创建前登记了唯一索引;夹具必须消费它,否则 wrapper 清理的是 + // 一个从未被使用的名字(真索引只能靠 t.Cleanup,进程提前退出就漏掉)。 + index = registered } // Register each disposable namespace before its first ES/object mutation. r10Save(t, "namespace", map[string]string{"bucket": wsBucket, "index": index})