test(library): 补孤儿校验锚点失败与介质失效的真实隔离用例

孤儿校验:缺 knowledge_id 的索引文档被拒绝、未知 knowledge_id 如实报孤儿、
数据库层中途失败(真实 statement timeout,SQLSTATE 57014)冒泡为错误而不是
"没有孤儿"。

锚点:真实 PG 约束拒绝写版本锚点时,作业在任何阶段写入前失败、目标零写入、
版本表不留本次行;去掉故障后同一请求写满四阶段。

介质失效:归档 part 缺失/摘要不符时恢复停在内容层,保持 held + RestoreBlocked,
服务与入口都不恢复、校验不再跑。
This commit is contained in:
姚章浩
2026-11-01 17:21:42 +08:00
parent 0df0e3cb1e
commit 99055a582e
4 changed files with 239 additions and 0 deletions
@@ -0,0 +1,124 @@
package main
import (
"context"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
)
// r13CountingTarget 记录 Apply 次数:锚点写失败时它必须是 0。
type r13CountingTarget struct {
ledgerTestTarget
applied int
}
func (t *r13CountingTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshot string) error {
t.applied++
return t.ledgerTestTarget.Apply(ctx, stage, snapshot)
}
// B25:写集合/预备份锚点写不进治理库时,必须在任何业务写入之前失败,目标零写入,
// 也不许留下"已有基线"的假象。故障来自真实 PG(版本表上的 CHECK 约束拒绝写入),
// 不是包一层必失败的假 store。
func TestR13AnchorWriteFailureBlocksTargetIsolated(t *testing.T) {
store, pool := openLedgerStore(t)
ctx := context.Background()
now := time.Now().UTC()
const (
scope = "internal"
jobID = "libsync-r13-anchor"
snapshot = "backup-snap-r13-anchor"
target = "target-snap-r13-anchor"
params = "params-r13-anchor"
)
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: jobID, Scope: scope,
Kind: governance.LibraryJobSync, IdempotencyKey: "library.sync:internal:r13-anchor",
RequestDigest: params, Actor: "admin", TargetSnapshotID: target}, governance.SyncStages(), now); err != nil {
t.Fatalf("create job: %v", err)
}
var exists bool
// 隔离夹具把表建在每次运行专属 schema 里(search_path 指向它),
// 因此这里用非限定名,不能写死 public。
if err := pool.QueryRow(ctx, `SELECT to_regclass('library_versions') IS NOT NULL`).Scan(&exists); err != nil {
t.Fatalf("locate version table: %v", err)
}
if !exists {
rows, err := pool.Query(ctx, `SELECT table_schema||'.'||table_name FROM information_schema.tables
WHERE table_name LIKE 'library%' ORDER BY 1`)
if err == nil {
var names []string
for rows.Next() {
var n string
rows.Scan(&n)
names = append(names, n)
}
rows.Close()
t.Logf("library tables visible: %v", names)
}
var ledger []string
if r2, err := pool.Query(ctx, `SELECT version::text FROM schema_migrations ORDER BY 1 DESC LIMIT 5`); err == nil {
for r2.Next() {
var v string
r2.Scan(&v)
ledger = append(ledger, v)
}
r2.Close()
}
t.Fatalf("版本表不存在:夹具/迁移不完整,本用例的前提不成立(最近迁移=%v)", ledger)
}
// NOT VALID:不动已有行,只让新的版本写入被真实 PG 拒绝。
if _, err := pool.Exec(ctx, `ALTER TABLE library_versions ADD CONSTRAINT r13_anchor_reject CHECK (false) NOT VALID`); err != nil {
t.Fatalf("inject anchor constraint: %v", err)
}
dropConstraint := func() {
pool.Exec(context.Background(), `ALTER TABLE library_versions DROP CONSTRAINT IF EXISTS r13_anchor_reject`)
}
t.Cleanup(dropConstraint)
recorder := libraryStageRecorder{store: store, jobID: jobID, scope: scope}
// 写集合必须落在一次性目录,不能把测试产物写进仓库。
inner := &r13CountingTarget{ledgerTestTarget: ledgerTestTarget{root: t.TempDir()}}
backups := ledgerTestBackups{ref: libraryrelease.BackupRef{ID: "backup-" + snapshot, SnapshotID: snapshot,
Bytes: 4096, Universe: map[string]libraryrelease.TableFingerprint{"resources": {Count: 1, Digest: "universe"}}}}
request := libraryrelease.Request{JobID: jobID, Target: scope, SnapshotID: target, ParamsDigest: params,
Now: now, Grant: libraryrelease.Grant{Target: scope, SnapshotID: target, ParamsDigest: params,
ExpiresAt: now.Add(time.Hour), AuthorizedBy: "tester"}}
outcome := libraryrelease.Run(ctx, request, inner, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("锚点写失败却继续执行:%+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("锚点失败必须早于任何阶段写入,failed_at=%q", outcome.FailedAt)
}
if inner.applied != 0 {
t.Fatalf("锚点失败后仍写了目标 %d 个阶段", inner.applied)
}
shown := false
for _, sub := range []string{"写集合", "登记", "constraint", "check", "23514"} {
if strings.Contains(outcome.Error, sub) {
shown = true
}
}
if !shown {
t.Fatalf("失败原因没有指向锚点写入:%q", outcome.Error)
}
versions, err := store.LibraryVersionsBySnapshot(ctx, scope, snapshot)
if err != nil {
t.Fatalf("read versions: %v", err)
}
if len(versions) != 0 {
t.Fatalf("锚点失败却留下版本证据:%+v", versions)
}
// 去掉故障后同一请求能完成登记:证明失败来自注入的锚点故障,不是环境坏掉。
dropConstraint()
retry := libraryrelease.Run(ctx, request, inner, backups, recorder)
if retry.State != "succeeded" {
t.Fatalf("去掉故障后应当能完成:%+v", retry)
}
if inner.applied != 4 {
t.Fatalf("去掉故障后应写满四个阶段,实际 %d", inner.applied)
}
}
@@ -177,3 +177,44 @@ func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(c
type verifierFunc func(context.Context, string) (string, error)
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }
// D04:备份介质失效(归档 part 缺失/摘要不符)时必须在内容层停住:保持 held +
// RestoreBlocked,服务与入口都不恢复,也不许跑校验或写终态。错误要能被人工按
// 归档处置(明确指向预备份/介质,而不是"未知失败")。
func TestR13RecoveryStaysHeldWhenBackupMediumFails(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-media", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
verified := false
_, err := e.RecoverWindow(context.Background(), "job-media",
restorerFunc(func(context.Context, string) error {
return errors.New("archive part chunks missing or digest mismatch: backup medium unusable")
}),
verifierFunc(func(context.Context, string) (string, error) { verified = true; return "d", nil }))
if err == nil {
t.Fatal("介质失效时恢复被当成成功")
}
if verified {
t.Fatal("归档读不出来却继续跑了内容校验")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("介质失效后服务 %s 被恢复", s.ID)
}
}
after, loadErr := e.load("job-media")
if loadErr != nil {
t.Fatal(loadErr)
}
if after.State != "held" || after.Code != "restore_failed" || !after.RestoreBlocked || after.RecoveredAt != nil {
t.Fatalf("journal=%+v:介质失效必须保持 held/RestoreBlocked", after)
}
}
@@ -0,0 +1,71 @@
package v071
import (
"fmt"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
// B21:孤儿校验的三条"不许假绿"路径——索引文档缺 knowledge_id、未知 knowledge_id
// 必须如实报孤儿、以及查询在迭代中途失败时必须冒泡成错误(而不是"没有孤儿")。
func TestR13OrphanOracleRejectsMalformedAndUnknownDocumentsIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
target := fx.target(t)
// 1) 缺 knowledge_id:必须报错,不能当作"无孤儿"。
broken := []SearchDocument{{ID: "chunk-broken", Source: []byte(`{"chunk_id":"chunk-broken","content":"x"}`)}}
if _, err := target.orphanIndexDocuments(fx.ctx, broken); err == nil ||
!strings.Contains(err.Error(), "knowledge_id") {
t.Fatalf("缺失 knowledge_id 未被判为错误:%v", err)
}
// 2) knowledge_id 指向本次范围外的文档:必须如实列出孤儿,而不是被静默忽略。
unknown := []SearchDocument{{ID: "chunk-unknown", Source: []byte(`{"chunk_id":"chunk-unknown","knowledge_id":"doc-not-in-scope"}`)}}
orphans, err := target.orphanIndexDocuments(fx.ctx, unknown)
if err != nil {
t.Fatalf("未知 knowledge_id 不应变成基础设施错误:%v", err)
}
if len(orphans) != 1 || orphans[0] != "chunk-unknown" {
t.Fatalf("孤儿列表 = %v,want [chunk-unknown]", orphans)
}
// 3) 范围内文档不能被误报成孤儿。
known := []SearchDocument{{ID: "chunk-target", Source: []byte(`{"chunk_id":"chunk-target","knowledge_id":"doc-target"}`)}}
if orphans, err := target.orphanIndexDocuments(fx.ctx, known); err != nil || len(orphans) != 0 {
t.Fatalf("正常文档被误报:%v %v", orphans, err)
}
r10Save(t, "orphan-paths", map[string]any{"missing_knowledge_id": "rejected", "unknown_reported": orphans})
}
// B21:真实驱动层失败(数据库 statement timeout 让 SELECT 在迭代中途中断)必须
// 返回错误。这条替换"用假 rows 造 Scan 错误"的做法:错误来自真实 PG。
func TestR13OrphanOraclePropagatesDriverFailureIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
// 足够多的行,保证 SELECT 一定超过下面的超时。
r10Exec(t, fx, `INSERT INTO knowledges(id,tenant_id,knowledge_base_id,parse_status,pending_subtasks_count,file_path,deleted_at)
SELECT 'bulk-'||i, 10001, 'kb-target', 'completed', 0, '', NULL FROM generate_series(1,200000) i`)
dbName := fx.pool.Config().ConnConfig.Database
r10Exec(t, fx, fmt.Sprintf(`ALTER DATABASE "%s" SET statement_timeout = '5ms'`, dbName))
t.Cleanup(func() {
if _, err := fx.pool.Exec(fx.ctx, fmt.Sprintf(`ALTER DATABASE "%s" RESET statement_timeout`, dbName)); err != nil {
t.Logf("reset statement_timeout: %v", err)
}
})
// ALTER DATABASE 只影响新会话:用一条新连接池执行被测查询。
slowPool, err := pgxpool.New(fx.ctx, fx.pool.Config().ConnString())
if err != nil {
t.Fatalf("open timing pool: %v", err)
}
defer slowPool.Close()
target := fx.target(t)
original := target.pool
target.pool = slowPool
defer func() { target.pool = original }()
documents := []SearchDocument{{ID: "chunk-target", Source: []byte(`{"chunk_id":"chunk-target","knowledge_id":"doc-target"}`)}}
if _, err := target.orphanIndexDocuments(fx.ctx, documents); err == nil {
t.Fatal("数据库迭代失败被当成'没有孤儿'")
} else {
r10Save(t, "orphan-driver-error", err.Error())
}
}
+3
View File
@@ -9,6 +9,9 @@
"⚠️ 文件缺失或解析不了 ⇒ 门禁红:范围声明不出来时,不许默认放行。"
],
"optional": [
{"package":"knowledge-governance/cmd/worker","test":"TestR13AnchorWriteFailureBlocksTargetIsolated","reason":"需显式一次性本机治理PG:真实约束拒绝写锚点,验证目标零写入且不留假基线;普通CI跳过不是通过"},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOraclePropagatesDriverFailureIsolated","reason":"需显式一次性本机PG:用真实 statement timeout 让校验查询中途失败,断言不是\"无孤儿\";普通CI跳过不是通过"},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOracleRejectsMalformedAndUnknownDocumentsIsolated","reason":"需显式一次性本机PG/ES/MinIO:孤儿校验的缺失/未知 knowledge_id 分支;普通CI跳过不是通过"},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12SharedResourceDifferenceMatrixIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12LateWikiFailureKeepsPreExistingObjectVersionsIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12IndexOracleDetectsContentVectorAndMetadataChangesIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},