test(library): 补孤儿校验锚点失败与介质失效的真实隔离用例
孤儿校验:缺 knowledge_id 的索引文档被拒绝、未知 knowledge_id 如实报孤儿、 数据库层中途失败(真实 statement timeout,SQLSTATE 57014)冒泡为错误而不是 "没有孤儿"。 锚点:真实 PG 约束拒绝写版本锚点时,作业在任何阶段写入前失败、目标零写入、 版本表不留本次行;去掉故障后同一请求写满四阶段。 介质失效:归档 part 缺失/摘要不符时恢复停在内容层,保持 held + RestoreBlocked, 服务与入口都不恢复、校验不再跑。
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// r13CountingTarget 记录 Apply 次数:锚点写失败时它必须是 0。
|
||||
type r13CountingTarget struct {
|
||||
ledgerTestTarget
|
||||
applied int
|
||||
}
|
||||
|
||||
func (t *r13CountingTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshot string) error {
|
||||
t.applied++
|
||||
return t.ledgerTestTarget.Apply(ctx, stage, snapshot)
|
||||
}
|
||||
|
||||
// B25:写集合/预备份锚点写不进治理库时,必须在任何业务写入之前失败,目标零写入,
|
||||
// 也不许留下"已有基线"的假象。故障来自真实 PG(版本表上的 CHECK 约束拒绝写入),
|
||||
// 不是包一层必失败的假 store。
|
||||
func TestR13AnchorWriteFailureBlocksTargetIsolated(t *testing.T) {
|
||||
store, pool := openLedgerStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
const (
|
||||
scope = "internal"
|
||||
jobID = "libsync-r13-anchor"
|
||||
snapshot = "backup-snap-r13-anchor"
|
||||
target = "target-snap-r13-anchor"
|
||||
params = "params-r13-anchor"
|
||||
)
|
||||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: jobID, Scope: scope,
|
||||
Kind: governance.LibraryJobSync, IdempotencyKey: "library.sync:internal:r13-anchor",
|
||||
RequestDigest: params, Actor: "admin", TargetSnapshotID: target}, governance.SyncStages(), now); err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
var exists bool
|
||||
// 隔离夹具把表建在每次运行专属 schema 里(search_path 指向它),
|
||||
// 因此这里用非限定名,不能写死 public。
|
||||
if err := pool.QueryRow(ctx, `SELECT to_regclass('library_versions') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
t.Fatalf("locate version table: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
rows, err := pool.Query(ctx, `SELECT table_schema||'.'||table_name FROM information_schema.tables
|
||||
WHERE table_name LIKE 'library%' ORDER BY 1`)
|
||||
if err == nil {
|
||||
var names []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
rows.Scan(&n)
|
||||
names = append(names, n)
|
||||
}
|
||||
rows.Close()
|
||||
t.Logf("library tables visible: %v", names)
|
||||
}
|
||||
var ledger []string
|
||||
if r2, err := pool.Query(ctx, `SELECT version::text FROM schema_migrations ORDER BY 1 DESC LIMIT 5`); err == nil {
|
||||
for r2.Next() {
|
||||
var v string
|
||||
r2.Scan(&v)
|
||||
ledger = append(ledger, v)
|
||||
}
|
||||
r2.Close()
|
||||
}
|
||||
t.Fatalf("版本表不存在:夹具/迁移不完整,本用例的前提不成立(最近迁移=%v)", ledger)
|
||||
}
|
||||
// NOT VALID:不动已有行,只让新的版本写入被真实 PG 拒绝。
|
||||
if _, err := pool.Exec(ctx, `ALTER TABLE library_versions ADD CONSTRAINT r13_anchor_reject CHECK (false) NOT VALID`); err != nil {
|
||||
t.Fatalf("inject anchor constraint: %v", err)
|
||||
}
|
||||
dropConstraint := func() {
|
||||
pool.Exec(context.Background(), `ALTER TABLE library_versions DROP CONSTRAINT IF EXISTS r13_anchor_reject`)
|
||||
}
|
||||
t.Cleanup(dropConstraint)
|
||||
recorder := libraryStageRecorder{store: store, jobID: jobID, scope: scope}
|
||||
// 写集合必须落在一次性目录,不能把测试产物写进仓库。
|
||||
inner := &r13CountingTarget{ledgerTestTarget: ledgerTestTarget{root: t.TempDir()}}
|
||||
backups := ledgerTestBackups{ref: libraryrelease.BackupRef{ID: "backup-" + snapshot, SnapshotID: snapshot,
|
||||
Bytes: 4096, Universe: map[string]libraryrelease.TableFingerprint{"resources": {Count: 1, Digest: "universe"}}}}
|
||||
request := libraryrelease.Request{JobID: jobID, Target: scope, SnapshotID: target, ParamsDigest: params,
|
||||
Now: now, Grant: libraryrelease.Grant{Target: scope, SnapshotID: target, ParamsDigest: params,
|
||||
ExpiresAt: now.Add(time.Hour), AuthorizedBy: "tester"}}
|
||||
outcome := libraryrelease.Run(ctx, request, inner, backups, recorder)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("锚点写失败却继续执行:%+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != "" {
|
||||
t.Fatalf("锚点失败必须早于任何阶段写入,failed_at=%q", outcome.FailedAt)
|
||||
}
|
||||
if inner.applied != 0 {
|
||||
t.Fatalf("锚点失败后仍写了目标 %d 个阶段", inner.applied)
|
||||
}
|
||||
shown := false
|
||||
for _, sub := range []string{"写集合", "登记", "constraint", "check", "23514"} {
|
||||
if strings.Contains(outcome.Error, sub) {
|
||||
shown = true
|
||||
}
|
||||
}
|
||||
if !shown {
|
||||
t.Fatalf("失败原因没有指向锚点写入:%q", outcome.Error)
|
||||
}
|
||||
versions, err := store.LibraryVersionsBySnapshot(ctx, scope, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("read versions: %v", err)
|
||||
}
|
||||
if len(versions) != 0 {
|
||||
t.Fatalf("锚点失败却留下版本证据:%+v", versions)
|
||||
}
|
||||
// 去掉故障后同一请求能完成登记:证明失败来自注入的锚点故障,不是环境坏掉。
|
||||
dropConstraint()
|
||||
retry := libraryrelease.Run(ctx, request, inner, backups, recorder)
|
||||
if retry.State != "succeeded" {
|
||||
t.Fatalf("去掉故障后应当能完成:%+v", retry)
|
||||
}
|
||||
if inner.applied != 4 {
|
||||
t.Fatalf("去掉故障后应写满四个阶段,实际 %d", inner.applied)
|
||||
}
|
||||
}
|
||||
@@ -177,3 +177,44 @@ func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(c
|
||||
type verifierFunc func(context.Context, string) (string, error)
|
||||
|
||||
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }
|
||||
|
||||
// D04:备份介质失效(归档 part 缺失/摘要不符)时必须在内容层停住:保持 held +
|
||||
// RestoreBlocked,服务与入口都不恢复,也不许跑校验或写终态。错误要能被人工按
|
||||
// 归档处置(明确指向预备份/介质,而不是"未知失败")。
|
||||
func TestR13RecoveryStaysHeldWhenBackupMediumFails(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
stopAllServices(c)
|
||||
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.save(heldJournal("job-media", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verified := false
|
||||
_, err := e.RecoverWindow(context.Background(), "job-media",
|
||||
restorerFunc(func(context.Context, string) error {
|
||||
return errors.New("archive part chunks missing or digest mismatch: backup medium unusable")
|
||||
}),
|
||||
verifierFunc(func(context.Context, string) (string, error) { verified = true; return "d", nil }))
|
||||
if err == nil {
|
||||
t.Fatal("介质失效时恢复被当成成功")
|
||||
}
|
||||
if verified {
|
||||
t.Fatal("归档读不出来却继续跑了内容校验")
|
||||
}
|
||||
for _, s := range c.services {
|
||||
if s.Running {
|
||||
t.Fatalf("介质失效后服务 %s 被恢复", s.ID)
|
||||
}
|
||||
}
|
||||
after, loadErr := e.load("job-media")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
if after.State != "held" || after.Code != "restore_failed" || !after.RestoreBlocked || after.RecoveredAt != nil {
|
||||
t.Fatalf("journal=%+v:介质失效必须保持 held/RestoreBlocked", after)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// B21:孤儿校验的三条"不许假绿"路径——索引文档缺 knowledge_id、未知 knowledge_id
|
||||
// 必须如实报孤儿、以及查询在迭代中途失败时必须冒泡成错误(而不是"没有孤儿")。
|
||||
func TestR13OrphanOracleRejectsMalformedAndUnknownDocumentsIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
target := fx.target(t)
|
||||
// 1) 缺 knowledge_id:必须报错,不能当作"无孤儿"。
|
||||
broken := []SearchDocument{{ID: "chunk-broken", Source: []byte(`{"chunk_id":"chunk-broken","content":"x"}`)}}
|
||||
if _, err := target.orphanIndexDocuments(fx.ctx, broken); err == nil ||
|
||||
!strings.Contains(err.Error(), "knowledge_id") {
|
||||
t.Fatalf("缺失 knowledge_id 未被判为错误:%v", err)
|
||||
}
|
||||
// 2) knowledge_id 指向本次范围外的文档:必须如实列出孤儿,而不是被静默忽略。
|
||||
unknown := []SearchDocument{{ID: "chunk-unknown", Source: []byte(`{"chunk_id":"chunk-unknown","knowledge_id":"doc-not-in-scope"}`)}}
|
||||
orphans, err := target.orphanIndexDocuments(fx.ctx, unknown)
|
||||
if err != nil {
|
||||
t.Fatalf("未知 knowledge_id 不应变成基础设施错误:%v", err)
|
||||
}
|
||||
if len(orphans) != 1 || orphans[0] != "chunk-unknown" {
|
||||
t.Fatalf("孤儿列表 = %v,want [chunk-unknown]", orphans)
|
||||
}
|
||||
// 3) 范围内文档不能被误报成孤儿。
|
||||
known := []SearchDocument{{ID: "chunk-target", Source: []byte(`{"chunk_id":"chunk-target","knowledge_id":"doc-target"}`)}}
|
||||
if orphans, err := target.orphanIndexDocuments(fx.ctx, known); err != nil || len(orphans) != 0 {
|
||||
t.Fatalf("正常文档被误报:%v %v", orphans, err)
|
||||
}
|
||||
r10Save(t, "orphan-paths", map[string]any{"missing_knowledge_id": "rejected", "unknown_reported": orphans})
|
||||
}
|
||||
|
||||
// B21:真实驱动层失败(数据库 statement timeout 让 SELECT 在迭代中途中断)必须
|
||||
// 返回错误。这条替换"用假 rows 造 Scan 错误"的做法:错误来自真实 PG。
|
||||
func TestR13OrphanOraclePropagatesDriverFailureIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
// 足够多的行,保证 SELECT 一定超过下面的超时。
|
||||
r10Exec(t, fx, `INSERT INTO knowledges(id,tenant_id,knowledge_base_id,parse_status,pending_subtasks_count,file_path,deleted_at)
|
||||
SELECT 'bulk-'||i, 10001, 'kb-target', 'completed', 0, '', NULL FROM generate_series(1,200000) i`)
|
||||
dbName := fx.pool.Config().ConnConfig.Database
|
||||
r10Exec(t, fx, fmt.Sprintf(`ALTER DATABASE "%s" SET statement_timeout = '5ms'`, dbName))
|
||||
t.Cleanup(func() {
|
||||
if _, err := fx.pool.Exec(fx.ctx, fmt.Sprintf(`ALTER DATABASE "%s" RESET statement_timeout`, dbName)); err != nil {
|
||||
t.Logf("reset statement_timeout: %v", err)
|
||||
}
|
||||
})
|
||||
// ALTER DATABASE 只影响新会话:用一条新连接池执行被测查询。
|
||||
slowPool, err := pgxpool.New(fx.ctx, fx.pool.Config().ConnString())
|
||||
if err != nil {
|
||||
t.Fatalf("open timing pool: %v", err)
|
||||
}
|
||||
defer slowPool.Close()
|
||||
target := fx.target(t)
|
||||
original := target.pool
|
||||
target.pool = slowPool
|
||||
defer func() { target.pool = original }()
|
||||
documents := []SearchDocument{{ID: "chunk-target", Source: []byte(`{"chunk_id":"chunk-target","knowledge_id":"doc-target"}`)}}
|
||||
if _, err := target.orphanIndexDocuments(fx.ctx, documents); err == nil {
|
||||
t.Fatal("数据库迭代失败被当成'没有孤儿'")
|
||||
} else {
|
||||
r10Save(t, "orphan-driver-error", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,9 @@
|
||||
"⚠️ 文件缺失或解析不了 ⇒ 门禁红:范围声明不出来时,不许默认放行。"
|
||||
],
|
||||
"optional": [
|
||||
{"package":"knowledge-governance/cmd/worker","test":"TestR13AnchorWriteFailureBlocksTargetIsolated","reason":"需显式一次性本机治理PG:真实约束拒绝写锚点,验证目标零写入且不留假基线;普通CI跳过不是通过"},
|
||||
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOraclePropagatesDriverFailureIsolated","reason":"需显式一次性本机PG:用真实 statement timeout 让校验查询中途失败,断言不是\"无孤儿\";普通CI跳过不是通过"},
|
||||
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOracleRejectsMalformedAndUnknownDocumentsIsolated","reason":"需显式一次性本机PG/ES/MinIO:孤儿校验的缺失/未知 knowledge_id 分支;普通CI跳过不是通过"},
|
||||
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12SharedResourceDifferenceMatrixIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},
|
||||
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12LateWikiFailureKeepsPreExistingObjectVersionsIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},
|
||||
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR12IndexOracleDetectsContentVectorAndMetadataChangesIsolated","reason":"需显式一次性本机PG/ES/MinIO:r12 软删除before-image/共享资源差异/外部索引文档字节/索引oracle/对象精确版本;普通CI跳过不是通过"},
|
||||
|
||||
Reference in New Issue
Block a user