为什么:B29.2 的 PG 子断言缺一条真实命中 PG 恢复步骤的失败闭环。故障不改产品 返回值、不停共享 PG、不扩 ACL、不改 schema:恢复最后一段 PG 写入(wiki 阶段) 会把 before-image 的日志行按原 id 插回目标范围;先用一行"外部范围、同主键"的 日志占住这个 id,那条真实 INSERT 就得到 23505。用例记录服务器 SQLSTATE、同一 语句的探针读回(没有多出行)、失败阶段事务回滚,以及真实的部分恢复现场: database 阶段已提交、wiki 表仍是发布写入、共享 resources 的新增行按设计留到 四阶段全部成功后再清。解除只删这一行,随后走同一 window_job_id 的正式 recover。 同时把夹具的索引登记接通运行器(KG_R38_INDEX 被真实消费),并落实际 namespace, 使 wrapper 在夹具未执行 t.Cleanup 时也只清登记的那一个索引。 验证:make verify 16/16 success(local 档案,identity=skipped 属本地常态); PG 闭环的真实运行在提交后按同一 SHA 单独执行并留 raw。
This commit is contained in:
@@ -1,5 +1,36 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(DS03 PG 恢复失败:held → 同窗口二次恢复)
|
||||
|
||||
### Added
|
||||
|
||||
- 新增 PG 恢复失败用例 `TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered`:复用已证明的
|
||||
`after:wiki` 定点进入回滚,唯一新增故障来自本轮一次性业务库里真实的 PG 错误——恢复事务写
|
||||
`knowledges` 时触发器 `RAISE EXCEPTION ... USING ERRCODE='42501'`。用例记录触发器 DDL、
|
||||
服务器返回的 SQLSTATE、探针语句与读回(该语句没有留下任何行),并断言失败事务整体回滚后
|
||||
目标仍是发布写入后的内容;`恢复阶段 database 失败` 与 `SQLSTATE 42501` 必须同时出现在失败文案里。
|
||||
- 新增 held 现场证据:故障未被自动解除(开关仍开、触发器仍在),目标行逐表等于 `after:wiki`
|
||||
处的 applied 读回,索引仍是发布写入后的内容(恢复没有走到索引阶段)。
|
||||
- 新增同窗口续作证据:解除故障只 DROP 本轮触发器/函数/开关表;同一 `window_job_id` 的
|
||||
`POST /api/v1/library/maintenance/recover` 完成后复核十表、引用承诺、对象版本字节、索引
|
||||
与网关回执/投递,并保存 replay 原始读回。
|
||||
|
||||
### Changed
|
||||
|
||||
- 隔离夹具在 `reset` 时消费运行器登记的唯一索引(`KG_R38_INDEX`),并在第一次写入前把实际
|
||||
namespace 落到本轮目录;清理器核对「运行器登记 / 夹具落盘 / 实际清理」是同一个名字,
|
||||
登记不一致时拒绝删除,不再只依赖 `t.Cleanup`。
|
||||
|
||||
### Fixed
|
||||
|
||||
- 证据目录的运行器:构建后/执行后比较 HEAD、干净状态与源码树摘要,构建后不符直接拒跑用例、
|
||||
执行后不符结论作废(不再出现源码已改却 `passed-cleaned`);finally 里的 provenance 异常
|
||||
不再跳过精确清理。
|
||||
- 证据目录的清理器:PG 环境发现等前置动作失败也落 `resources-cleaned.json` 与非零终态;
|
||||
已有的 `cleanup-incomplete` 结果文件不再因"文件存在"而返回 0。
|
||||
|
||||
---
|
||||
|
||||
## 开发快照(DS03 对象恢复父复验:十表全形状与对象版本字节)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -40,6 +41,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
@@ -364,6 +366,20 @@ func TestFormalMaintenanceRecoveryObjectRestoreHeldThenRecovered(t *testing.T) {
|
||||
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
|
||||
}
|
||||
|
||||
// TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered 是 PG 恢复失败分支的显式入口:
|
||||
// 隔离部署里必须由 runner 传入 KG_R38_SCENARIO=pg-restore。发布失败入口复用已证明的
|
||||
// after:wiki 定点;唯一新增故障来自这次专用恢复事务里的真实 PG 错误(触发器 + SQLSTATE),
|
||||
// 不靠停共享 PG、扩大 ACL 或改产品返回值制造。
|
||||
func TestFormalMaintenanceRecoveryPGRestoreHeldThenRecovered(t *testing.T) {
|
||||
if os.Getenv("KG_R38_RUN") == "" {
|
||||
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
|
||||
}
|
||||
if os.Getenv("KG_R38_SCENARIO") != "pg-restore" {
|
||||
t.Fatalf("pg recovery branch requires KG_R38_SCENARIO=pg-restore, got %q", os.Getenv("KG_R38_SCENARIO"))
|
||||
}
|
||||
TestFormalMaintenanceRecoveryHeldThenRecovered(t)
|
||||
}
|
||||
|
||||
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
root := os.Getenv("KG_R38_RUN")
|
||||
if root == "" {
|
||||
@@ -371,6 +387,10 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
}
|
||||
scenario := os.Getenv("KG_R38_SCENARIO")
|
||||
objectBranch := scenario == "object-restore"
|
||||
pgBranch := scenario == "pg-restore"
|
||||
// 三条分支共用同一套 fullshape/真实入库装配;对象分支与 PG 分支都让 ES 恢复成功,
|
||||
// 只有 ES 分支把 clean 派发拒掉。
|
||||
fullShape := objectBranch || pgBranch
|
||||
ctx := context.Background()
|
||||
save := func(name string, value any) {
|
||||
t.Helper()
|
||||
@@ -398,7 +418,7 @@ func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
// 目标 before-image 十表全非空。旧 ES 分支保持原最小夹具语义,不在本轮扩围。
|
||||
var declaredBefore map[string][]map[string]any
|
||||
var beforeSentinels map[string][]map[string]any
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
declaredBefore = r38SeedFullShape(t, fx)
|
||||
} else {
|
||||
seedTargetData(t, fx)
|
||||
@@ -412,7 +432,20 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
must(e)
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
save("01-pg-before.json", beforePG)
|
||||
if objectBranch {
|
||||
// PG 分支的故障锚点:恢复的最后一段 PG 写入(wiki 阶段)会把 before-image 的
|
||||
// 日志行按原 id 插回目标范围,所以一个同主键、不同范围的"外来行"会让那条真实
|
||||
// INSERT 报 23505;发布/恢复都不改 schema,也不动共享 PG。
|
||||
var conflictLog int64
|
||||
var beforeTarget map[string][]map[string]any
|
||||
if pgBranch {
|
||||
must(fx.pool.QueryRow(ctx, `SELECT id FROM wiki_log_entries WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY id DESC LIMIT 1`,
|
||||
wsTargetTenant, wsTargetKB).Scan(&conflictLog))
|
||||
if conflictLog == 0 {
|
||||
t.Fatal("pg branch needs a before-image wiki log identity to collide with")
|
||||
}
|
||||
beforeTarget = r38TargetRows(t, fx)
|
||||
}
|
||||
if fullShape {
|
||||
for _, table := range ds02TargetTables {
|
||||
if len(beforePG[table]) == 0 {
|
||||
t.Fatalf("before-image table %s is empty: the ten-table claim would be vacuous", table)
|
||||
@@ -432,7 +465,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
beforeObjects := objectInventoryR38(t, ctx, fx)
|
||||
save("03-objects-before.json", beforeObjects)
|
||||
var beforeDigests map[string]string
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
beforeDigests = r38ObjectVersionDigests(t, ctx, fx, beforeObjects)
|
||||
if len(beforeDigests) == 0 {
|
||||
t.Fatal("before-image has no object versions to fingerprint")
|
||||
@@ -449,7 +482,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
var expectation ds02Expectation
|
||||
var source string
|
||||
var sourceParts map[string]int64
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
spec = ds02SourceSpec(t, fx)
|
||||
expectation = ds02Expected(t, fx, spec)
|
||||
source = r38SealFullShape(t, fx, spec, mapping)
|
||||
@@ -482,7 +515,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
|
||||
project := os.Getenv("KG_R38_PROJECT")
|
||||
proxy := newR38RefusalProxy(t, root, fx.searchBase)
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 对象分支要证明的是"对象恢复传输失败",ES 的回滚写入必须放行,
|
||||
// 否则失败会被误报成旧的 ES clean-dispatch 分支。
|
||||
proxy.setRefuseClean(false)
|
||||
@@ -743,13 +776,19 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
// 放行后由生产自己的校验判定失败并进入回滚。
|
||||
hitPath := filepath.Join(root, "barriers", "after-wiki.hit")
|
||||
released := false
|
||||
var appliedRows map[string][]map[string]any
|
||||
var appliedIndex map[string]map[string]any
|
||||
var pgFault map[string]any
|
||||
for i := 0; i < 900; i++ {
|
||||
if _, e := os.Stat(hitPath); e == nil {
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 屏障停在发布已写完、校验尚未开始时:此时目标里是"这次发布真正写进去的
|
||||
// 内容"。在这里独立读回十表并核 LocalIDs,而不是等回滚后拿结果反推。
|
||||
applied := r38TargetRows(t, fx)
|
||||
appliedRows = applied
|
||||
save("08b-applied-pg.json", applied)
|
||||
appliedIndex = recoveryIndexReadback(t, fx)
|
||||
save("08f-index-applied.json", appliedIndex)
|
||||
save("08c-applied-reference-promises.json", r38RequireReferences(t, "applied publish rows", applied))
|
||||
ledger := r38FindWriteSet(t, archiveRoot, job.ID)
|
||||
mappingDiffs := ds02CheckLocalIDMapping(r38LocalColumns(t, fx, spec, expectation), ledger.LocalIDs)
|
||||
@@ -768,10 +807,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if objectBranch {
|
||||
objectProxy.setArmed(true)
|
||||
}
|
||||
if pgBranch {
|
||||
pgFault = r38ArmPGFault(t, fx, conflictLog)
|
||||
save("08e-pg-fault-armed.json", pgFault)
|
||||
}
|
||||
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
|
||||
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
|
||||
"verify_corruption_armed": true, "refuse_clean_dispatch": !objectBranch,
|
||||
"refuse_object_restore_dispatch": objectBranch, "scenario": scenario})
|
||||
"verify_corruption_armed": true, "refuse_clean_dispatch": !fullShape,
|
||||
"refuse_object_restore_dispatch": objectBranch, "refuse_pg_restore": pgBranch, "scenario": scenario})
|
||||
released = true
|
||||
break
|
||||
}
|
||||
@@ -795,7 +838,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
|
||||
}
|
||||
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
|
||||
if objectBranch {
|
||||
switch {
|
||||
case objectBranch:
|
||||
// 失败必须来自对象恢复边界:回滚已通过 ES clean 写入,随后在对象删除派发前被拒。
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "conditional object cleanup not confirmed") {
|
||||
t.Fatalf("publish failure does not name the refused object restore: %q", failureText)
|
||||
@@ -803,8 +847,24 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("object branch was mislabelled as the old ES failure: %q", failureText)
|
||||
}
|
||||
} else if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
|
||||
case pgBranch:
|
||||
// 失败必须来自 PG 恢复步骤本身:恢复阶段 database + 真实 SQLSTATE,而不是 ES/对象文案。
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "恢复阶段 wiki 失败") {
|
||||
t.Fatalf("publish failure does not name the failed PG restore stage: %q", failureText)
|
||||
}
|
||||
if !strings.Contains(failureText, "SQLSTATE "+r38PGSQLState) {
|
||||
t.Fatalf("publish failure carries no SQLSTATE %s: %q", r38PGSQLState, failureText)
|
||||
}
|
||||
if strings.Contains(failureText, "index effect outcome unknown") || strings.Contains(failureText, "conditional object cleanup not confirmed") {
|
||||
t.Fatalf("pg branch was mislabelled as an ES or object failure: %q", failureText)
|
||||
}
|
||||
save("09a-pg-failure.json", map[string]any{"job": job.ID, "error": libraryAfterPublish.Error,
|
||||
"recovery": libraryAfterPublish.Recovery, "failed_stage": libraryAfterPublish.Stage,
|
||||
"expected_sqlstate": r38PGSQLState, "fault_armed": pgFault, "matched_sqlstate": true})
|
||||
default:
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
|
||||
}
|
||||
}
|
||||
if string(libraryAfterPublish.Stage) != "rollback" {
|
||||
t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage)
|
||||
@@ -843,16 +903,21 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("scope maintenance = %+v", scopeRow)
|
||||
}
|
||||
save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow})
|
||||
if pgBranch {
|
||||
// 失败现场必须保留:故障未自行解除,失败事务没有提交,目标仍是发布写入后的内容。
|
||||
save("11a-pg-failed-restore.json", r38PGFaultHeld(t, fx, beforeTarget, appliedRows, failureText, conflictLog))
|
||||
}
|
||||
|
||||
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
|
||||
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
|
||||
save("12-index-effects-held.json", effects)
|
||||
if objectBranch {
|
||||
switch {
|
||||
case fullShape:
|
||||
// 对象分支的 ES clean 写入已成功:索引恢复回执必须存在,这不是被拒的边界。
|
||||
if _, ok := effects["restored"]; !ok {
|
||||
t.Fatalf("object branch expected a completed index restore receipt (have %v)", effectPhases(effects))
|
||||
}
|
||||
} else {
|
||||
default:
|
||||
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
|
||||
if _, ok := effects[phase]; !ok {
|
||||
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
|
||||
@@ -870,7 +935,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
}
|
||||
heldSource := heldDoc["_source"].(map[string]any)
|
||||
heldContent, _ := heldSource["content"].(string)
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 对象分支里索引已恢复,marker 不能还在:否则说明失败的其实是 ES 分支。
|
||||
if strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
|
||||
t.Fatalf("object branch left the ES restore marker in place: %q", heldContent)
|
||||
@@ -885,7 +950,9 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
events, refused := proxy.snapshot()
|
||||
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
|
||||
var heldObjects *s3.ListObjectVersionsOutput
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
heldObjects = objectInventoryR38(t, ctx, fx)
|
||||
save("13a-objects-held.json", heldObjects)
|
||||
if refused != 0 {
|
||||
t.Fatalf("object branch must not refuse ES clean dispatch, got %d refusals", refused)
|
||||
}
|
||||
@@ -901,7 +968,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
forwarded++
|
||||
}
|
||||
}
|
||||
if !objectBranch && forwarded != 0 {
|
||||
if !fullShape && forwarded != 0 {
|
||||
t.Fatalf("refused clean write was forwarded %d times", forwarded)
|
||||
}
|
||||
if objectBranch {
|
||||
@@ -956,8 +1023,6 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if _, ok := deleteIntent["delete-intent"]; !ok {
|
||||
t.Fatalf("object restore lacked a durable delete-intent effect (have %v)", effectPhases(deleteIntent))
|
||||
}
|
||||
heldObjects = objectInventoryR38(t, ctx, fx)
|
||||
save("13a-objects-held.json", heldObjects)
|
||||
heldKeys := map[string]bool{}
|
||||
for _, version := range heldObjects.Versions {
|
||||
heldKeys[aws.ToString(version.Key)] = true
|
||||
@@ -1018,9 +1083,14 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if objectBranch {
|
||||
objectProxy.setArmed(false)
|
||||
}
|
||||
if pgBranch {
|
||||
// 只解除本轮注入的 PG 故障(drop trigger/function/switch table),不动共享 PG 本身。
|
||||
save("17b-pg-fault-disarmed.json", r38DisarmPGFault(t, fx, conflictLog))
|
||||
}
|
||||
save("17a-recover-request.json", map[string]any{"window_job_id": job.ID, "scope": "internal",
|
||||
"backup_snapshot_id": backupID, "original_job": job.ID, "object_fault_removed": objectBranch,
|
||||
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
|
||||
"pg_fault_removed": pgBranch,
|
||||
"publish_entry": "store.CreateLibraryDispatch (publish API entry not proven)"})
|
||||
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||||
map[string]any{"scope": "internal", "window_job_id": job.ID})
|
||||
if code != http.StatusAccepted {
|
||||
@@ -1090,7 +1160,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if diff := diffUniverse(beforePG, afterPG); diff != "" {
|
||||
t.Fatalf("PG not restored row-by-row: %s", diff)
|
||||
}
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 独立 oracle:目标范围十表必须逐列等于声明 before-image(不是"和上一份读回相同"),
|
||||
// 邻库/外部租户哨兵必须原样保留,恢复后的引用承诺逐种 matched>0。
|
||||
restored := r38TargetRows(t, fx)
|
||||
@@ -1123,7 +1193,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
t.Fatalf("object delete markers differ from before-image:\n before=%v\n after=%v",
|
||||
objectDeleteMarkers(beforeObjects), objectDeleteMarkers(afterObjects))
|
||||
}
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 字节指纹:对 before-image 里每个受保护 versionId 单独 GET 算 SHA256,恢复后
|
||||
// 同 version 再独立算一次;ETag/size 只当旁证,绝不拿元数据当"逐字节相同"。
|
||||
afterDigests := r38ObjectVersionDigests(t, ctx, fx, afterObjects)
|
||||
@@ -1172,7 +1242,33 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
}
|
||||
}
|
||||
if operation == "" {
|
||||
t.Fatal("object branch lost the refused operation identity")
|
||||
// PG 分支在 held 时还没走到对象删除,没有"被拒请求"可以回溯;按发布新增的
|
||||
// key@version 精确匹配 removed 回执,拿到的仍是真实网关写入的操作身份。
|
||||
wanted := map[string]bool{}
|
||||
for _, id := range createdVersions {
|
||||
wanted[id] = true
|
||||
}
|
||||
rows, e := gov.Query(ctx, `SELECT receipt FROM kg_object_gateway_receipts`)
|
||||
must(e)
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
must(rows.Scan(&raw))
|
||||
var receipt objectgateway.Receipt
|
||||
must(json.Unmarshal(raw, &receipt))
|
||||
if receipt.State == "removed" && wanted[receipt.Request.Key+"@"+receipt.Version.ID] {
|
||||
operation = receipt.Request.Operation
|
||||
}
|
||||
}
|
||||
must(rows.Err())
|
||||
rows.Close()
|
||||
if objectBranch {
|
||||
t.Fatal("object branch lost the refused operation identity")
|
||||
}
|
||||
if operation == "" {
|
||||
t.Fatal("pg branch found no removed receipt bound to a version created by the failed publish")
|
||||
}
|
||||
save("23d-removal-bound-by-receipt.json", map[string]any{"operation": operation,
|
||||
"created_versions": createdVersions})
|
||||
}
|
||||
var removedRaw []byte
|
||||
must(gov.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE operation=$1", operation).Scan(&removedRaw))
|
||||
@@ -1235,7 +1331,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
if len(receipts) == 0 || len(deliveries) == 0 {
|
||||
t.Fatal("no gateway receipt/delivery evidence for real object writes")
|
||||
}
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
// 发布期与恢复期两条独立记录必须指向同一套 mapping:发布时落盘的写集合摘要,
|
||||
// 必须同时出现在作业锚点、预备份版本证据、网关回执/投递计划里;恢复意图的
|
||||
// request_digest 由窗口/作业/目标/来源/备份/该计划独立重算并与持久记录逐字符比较。
|
||||
@@ -1399,7 +1495,7 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
!reflect.DeepEqual(objectDeleteMarkers(afterObjects), objectDeleteMarkers(replayObjects)) {
|
||||
t.Fatal("replay rewrote object versions or delete markers")
|
||||
}
|
||||
if objectBranch {
|
||||
if fullShape {
|
||||
replayDigests := r38ObjectVersionDigests(t, ctx, fx, replayObjects)
|
||||
save("31a-object-byte-digests-after-replay.json", replayDigests)
|
||||
if !reflect.DeepEqual(beforeDigests, replayDigests) {
|
||||
@@ -1416,7 +1512,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
_ = worker.Wait()
|
||||
objectEventsAfter, objectRefused, objectForwarded := objectProxy.snapshot()
|
||||
save("28-result.json", map[string]any{"scenario": scenario, "object_restore_branch": objectBranch,
|
||||
"window_job": job.ID, "publish_queue": queueID,
|
||||
"pg_restore_branch": pgBranch,
|
||||
"window_job": job.ID, "publish_queue": queueID,
|
||||
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
|
||||
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
|
||||
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
|
||||
@@ -1429,8 +1526,8 @@ UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-v
|
||||
"run": map[string]any{"label": os.Getenv("KG_R38_LABEL"), "commit": os.Getenv("KG_R38_COMMIT"),
|
||||
"source_tree": os.Getenv("KG_R38_SOURCE_TREE"), "version": os.Getenv("KG_R38_VERSION"),
|
||||
"release_version": os.Getenv("KG_R38_RELEASE_VERSION")},
|
||||
"full_shape": map[string]any{"ten_table_before_nonempty": objectBranch,
|
||||
"ten_table_source_nonempty": objectBranch, "before_tables": len(beforePG),
|
||||
"full_shape": map[string]any{"ten_table_before_nonempty": fullShape,
|
||||
"ten_table_source_nonempty": fullShape, "before_tables": len(beforePG),
|
||||
"source_parts": sourceParts, "reference_promises": "all matched>0",
|
||||
"local_id_mapping": "publish ledger vs independent oracle + recovery digest",
|
||||
"object_byte_sha256_versions": len(beforeDigests)}})
|
||||
@@ -1611,3 +1708,151 @@ func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *write
|
||||
deliveryRows.Close()
|
||||
return receipts, deliveries
|
||||
}
|
||||
|
||||
// ---------- PG 恢复故障注入(只在本轮一次性业务库) ----------
|
||||
|
||||
const r38PGSQLState = "23505"
|
||||
|
||||
// r38PGFaultInsert 是恢复自己的 wiki 阶段写法(jsonb_populate_record + SELECT *),
|
||||
// 这里用同一条语句做"故障探针",证明报错来自恢复会执行的那类 SQL。
|
||||
const r38PGFaultInsert = `INSERT INTO wiki_log_entries SELECT * FROM jsonb_populate_record(NULL::wiki_log_entries, $1::jsonb)`
|
||||
|
||||
// r38ArmPGFault 是唯一新增的故障:before-image 的一条 wiki 日志在恢复的最后一段 PG 写入里
|
||||
// 会按原 id 插回目标范围;先用一行"外部范围、同主键"的日志占住这个 id,恢复的真实 INSERT
|
||||
// 就会得到 23505。它不改 schema(不加触发器/约束/索引)、不停共享 PG、不动 ACL;解除就是删这一行。
|
||||
func r38ArmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any {
|
||||
t.Helper()
|
||||
if logID == 0 {
|
||||
t.Fatal("pg fault armed without a wiki log identity")
|
||||
}
|
||||
var raw []byte
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x ORDER BY id LIMIT 1`).Scan(&raw); e != nil {
|
||||
t.Fatalf("pg fault needs one existing wiki log row as a shape template: %v", e)
|
||||
}
|
||||
var row map[string]any
|
||||
if e := json.Unmarshal(raw, &row); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
row["id"] = logID
|
||||
row["tenant_id"] = 20000
|
||||
row["knowledge_base_id"] = "kb-r38-foreign"
|
||||
encoded, e := json.Marshal(row)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := fx.pool.Exec(fx.ctx, r38PGFaultInsert, encoded); e != nil {
|
||||
t.Fatalf("arm pg fault (foreign row holding wiki log %d): %v", logID, e)
|
||||
}
|
||||
var tenant int64
|
||||
var kb string
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT tenant_id,knowledge_base_id FROM wiki_log_entries WHERE id=$1`, logID).Scan(&tenant, &kb); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if tenant != 20000 || kb != "kb-r38-foreign" {
|
||||
t.Fatalf("pg fault row readback = %d/%s, want 20000/kb-r38-foreign", tenant, kb)
|
||||
}
|
||||
return map[string]any{"stage": "wiki", "table": "wiki_log_entries", "conflict_id": logID,
|
||||
"conflict_tenant": tenant, "conflict_kb": kb, "insert_statement": r38PGFaultInsert,
|
||||
"expected_sqlstate": r38PGSQLState,
|
||||
"mechanism": "foreign-scope row holds the primary key the recovery wiki stage restores",
|
||||
"schema_changed": false,
|
||||
"scope": "one disposable business database; shared PG, ACL and sessions untouched"}
|
||||
}
|
||||
|
||||
// r38HeldAppliedTables 是"wiki 阶段失败时仍保持发布写入内容"的表:wiki_* 属于失败并回滚的
|
||||
// 那一阶段;resources 是多知识库共享表,恢复把它的精确清理推迟到四阶段全部成功之后,
|
||||
// 所以此刻仍带着发布新增的资源行。其余表都属于已提交的 database 阶段。
|
||||
func r38HeldAppliedTables() map[string]bool {
|
||||
return map[string]bool{"wiki_folders": true, "wiki_pages": true,
|
||||
"wiki_page_issues": true, "wiki_log_entries": true, "resources": true}
|
||||
}
|
||||
|
||||
// r38PGFaultHeld 证明三件事:故障行还在现场;同一 INSERT 再执行一次仍拿到服务器 SQLSTATE
|
||||
// 且没有多出行;失败后目标是一个真实的"部分恢复"状态——database 阶段已提交(非 wiki 表
|
||||
// 回到 before-image),wiki 阶段失败并整体回滚(wiki 表仍是发布写入后的内容)。
|
||||
func r38PGFaultHeld(t *testing.T, fx *writeSetFixture, before, applied map[string][]map[string]any, failureText string, logID int64) map[string]any {
|
||||
t.Helper()
|
||||
if len(before) == 0 || len(applied) == 0 {
|
||||
t.Fatal("held pg evidence needs both the before-image and the applied readback")
|
||||
}
|
||||
var rows int
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`,
|
||||
logID).Scan(&rows); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("pg fault row was not preserved at held: %d rows for wiki log %d", rows, logID)
|
||||
}
|
||||
var raw []byte
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT to_jsonb(x) FROM wiki_log_entries x WHERE id=$1`, logID).Scan(&raw); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, probeErr := fx.pool.Exec(fx.ctx, r38PGFaultInsert, raw)
|
||||
var pgErr *pgconn.PgError
|
||||
if probeErr == nil || !errors.As(probeErr, &pgErr) {
|
||||
t.Fatalf("fault probe did not raise a PG error: %v", probeErr)
|
||||
}
|
||||
if pgErr.Code != r38PGSQLState {
|
||||
t.Fatalf("fault probe SQLSTATE = %q, want %s", pgErr.Code, r38PGSQLState)
|
||||
}
|
||||
var probeRows int
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE id=$1`, logID).Scan(&probeRows); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if probeRows != 1 {
|
||||
t.Fatalf("failed probe left %d rows for wiki log %d: the statement transaction did not roll back", probeRows, logID)
|
||||
}
|
||||
held := r38TargetRows(t, fx)
|
||||
stillApplied := r38HeldAppliedTables()
|
||||
committed, rolledBack := []string{}, []string{}
|
||||
for _, table := range ds02TargetTables {
|
||||
want, label := before[table], "before-image"
|
||||
if stillApplied[table] {
|
||||
want, label = applied[table], "applied publish"
|
||||
}
|
||||
if diff := r38CompareRows("held "+table, want, held[table]); diff != "" {
|
||||
t.Fatalf("held %s does not match the %s (earlier stages commit per stage, the failed stage rolls back): %s", table, label, diff)
|
||||
}
|
||||
if stillApplied[table] {
|
||||
rolledBack = append(rolledBack, table)
|
||||
} else {
|
||||
committed = append(committed, table)
|
||||
}
|
||||
}
|
||||
encoded, e := json.Marshal(held)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sum := sha256.Sum256(encoded)
|
||||
counts := map[string]int{}
|
||||
for table, values := range held {
|
||||
counts[table] = len(values)
|
||||
}
|
||||
return map[string]any{"conflict_id": logID, "conflict_rows": rows,
|
||||
"probe_statement": r38PGFaultInsert, "sqlstate": pgErr.Code, "probe_error": pgErr.Error(),
|
||||
"probe_rows_after_error": probeRows,
|
||||
"transaction": "wiki stage INSERT failed; that stage transaction rolled back; earlier database stage stayed committed",
|
||||
"before_image_tables": committed, "still_applied_tables": rolledBack,
|
||||
"held_row_counts": counts, "held_pg_sha256": hex.EncodeToString(sum[:]),
|
||||
"matches_before_and_applied": true,
|
||||
"failure_text_has_sqlstate": strings.Contains(failureText, "SQLSTATE "+r38PGSQLState),
|
||||
"failure_text_has_stage": strings.Contains(failureText, "恢复阶段 wiki 失败")}
|
||||
}
|
||||
|
||||
func r38DisarmPGFault(t *testing.T, fx *writeSetFixture, logID int64) map[string]any {
|
||||
t.Helper()
|
||||
statement := `DELETE FROM wiki_log_entries WHERE id=$1 AND tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`
|
||||
of, e := fx.pool.Exec(fx.ctx, statement, logID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var left int
|
||||
if e := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM wiki_log_entries WHERE tenant_id=20000 AND knowledge_base_id='kb-r38-foreign'`).Scan(&left); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if of.RowsAffected() != 1 || left != 0 {
|
||||
t.Fatalf("pg fault not removed: deleted=%d left=%d", of.RowsAffected(), left)
|
||||
}
|
||||
return map[string]any{"statement": statement, "deleted": of.RowsAffected(), "foreign_rows_left": left,
|
||||
"readback_absent": true}
|
||||
}
|
||||
|
||||
@@ -203,6 +203,10 @@ func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writ
|
||||
index := "kg_library_test_" + hex.EncodeToString(token)
|
||||
if options.index != "" {
|
||||
index = options.index
|
||||
} else if registered := os.Getenv("KG_R38_INDEX"); registered != "" {
|
||||
// 隔离运行器在创建前登记了唯一索引;夹具必须消费它,否则 wrapper 清理的是
|
||||
// 一个从未被使用的名字(真索引只能靠 t.Cleanup,进程提前退出就漏掉)。
|
||||
index = registered
|
||||
}
|
||||
// Register each disposable namespace before its first ES/object mutation.
|
||||
r10Save(t, "namespace", map[string]string{"bucket": wsBucket, "index": index})
|
||||
|
||||
Reference in New Issue
Block a user