Files
knowledge-governance/internal/releaseadapter/v071/library_export_integration_test.go
T
姚章浩 e0768f2caa
build / build (push) Successful in 1m34s
fix(library): 写集合台账与跨范围冲突预检
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它
知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可
恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例
证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。

改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据"
算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份
版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对
资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。
resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。
整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。

本地验证(已跑):
- make verify 15 项全 success(identity 因本地档案 skipped)
- go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES +
  真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/
  历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复;
  跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净
- SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变
- go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、
  接管不重新规划
2026-10-29 23:55:13 +08:00

216 lines
9.0 KiB
Go

package v071
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
// This fixture creates only synthetic content in an explicitly named disposable
// database. Never point this test at an application database.
func TestLibraryExportIsolatedPostgres(t *testing.T) {
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Fatal("isolated database required")
}
ctx := context.Background()
db, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
defer db.Close()
// 同一个隔离库会被本包的多个用例复用:不清掉上一次的 schema 就会
// "relation already exists",让人误以为是导出本身坏了。
if _, err = db.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, libraryTestSchema); err != nil {
t.Fatal(err)
}
scope := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}
run := func() PostgresSnapshot {
t.Helper()
s, e := ExportLibrary(ctx, db, scope)
if e != nil {
t.Fatal(e)
}
return s
}
snap := run()
if snap.Table("knowledges").RowCount != 1 || snap.Table("wiki_pages").RowCount != 1 || snap.Table("resources").RowCount != 1 {
t.Fatal("incorrect scoped export")
}
if snap.Table("wiki_log_entries").RowCount != 1 || snap.Table("wiki_page_issues").RowCount != 1 {
t.Fatal("Wiki associated data lost")
}
for _, tab := range snap.Tables {
for _, raw := range tab.Rows {
if strings.Contains(string(raw), "other-scope") {
t.Fatalf("cross scope leak in %s", tab.Name)
}
}
}
t.Run("whole source settlement", func(t *testing.T) {
if err := checkLibrarySourceSettlement(ctx, db); err != nil {
t.Fatal(err)
}
for _, tc := range []struct{ name, change, restore string }{
{"other tenant document", `UPDATE knowledges SET parse_status='processing' WHERE id='other-scope'`, `UPDATE knowledges SET parse_status='completed' WHERE id='other-scope'`},
{"other library pending", `INSERT INTO task_pending_ops VALUES(20000,'wiki:ingest','knowledge_base','other-library')`, `DELETE FROM task_pending_ops`},
{"deleted scope pending", `INSERT INTO task_pending_ops VALUES(20000,'wiki:retract','knowledge','physically-deleted-doc')`, `DELETE FROM task_pending_ops`},
{"unknown scope pending", `INSERT INTO task_pending_ops VALUES(20000,'unknown:task','unknown','unknown')`, `DELETE FROM task_pending_ops`},
{"other tenant dead letter", `INSERT INTO task_dead_letters VALUES(20000,'document:process','knowledge_base','other-library')`, `DELETE FROM task_dead_letters`},
} {
t.Run(tc.name, func(t *testing.T) {
if _, err := db.Exec(ctx, tc.change); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if _, err := db.Exec(ctx, tc.restore); err != nil {
t.Error(err)
}
})
if _, err := ExportLibrary(ctx, db, scope); err != nil {
t.Fatalf("selected library unexpectedly busy: %v", err)
}
if err := checkLibrarySourceSettlement(ctx, db); err == nil {
t.Fatal("source maintenance allowed while another scope has unfinished work")
}
})
}
if err := checkLibrarySourceSettlement(ctx, db); err != nil {
t.Fatalf("settled source did not recover: %v", err)
}
})
t.Run("MVCC concurrent update", func(t *testing.T) {
wrapper := libraryHookDB{pool: db, hook: func() {
if _, err := db.Exec(ctx, `UPDATE wiki_pages SET content='new-content' WHERE id='wiki-safe'`); err != nil {
t.Fatal(err)
}
}}
old, err := ExportLibrary(ctx, wrapper, scope)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(old.Table("wiki_pages").Rows[0]), "old-content") {
t.Fatal("mixed PostgreSQL versions during export")
}
fresh := run()
if !strings.Contains(string(fresh.Table("wiki_pages").Rows[0]), "new-content") {
t.Fatal("concurrent writer did not actually commit")
}
})
t.Run("deleted knowledge tasks", func(t *testing.T) {
for _, test := range []struct{ table, task, scope, id string }{
{"task_pending_ops", "wiki:ingest", "knowledge_base", "kb-safe"},
{"task_pending_ops", "wiki:finalize", "knowledge_base", "kb-safe"},
{"task_dead_letters", "wiki:ingest", "knowledge", "deleted-doc"},
} {
_, err := db.Exec(ctx, `INSERT INTO `+test.table+`(tenant_id,task_type,scope,scope_id) VALUES(10000,$1,$2,$3)`, test.task, test.scope, test.id)
if err != nil {
t.Fatal(err)
}
if _, err = ExportLibrary(ctx, db, scope); err == nil {
t.Fatalf("ignored %s %s", test.table, test.task)
}
if _, err = db.Exec(ctx, `DELETE FROM `+test.table); err != nil {
t.Fatal(err)
}
}
})
t.Run("dangling resource blocks", func(t *testing.T) {
_, err := db.Exec(ctx, `UPDATE wiki_pages SET content='resource://abcdefghijklmnopqrstuv' WHERE id='wiki-safe'`)
if err != nil {
t.Fatal(err)
}
if _, err = ExportLibrary(ctx, db, scope); err == nil {
t.Fatal("accepted missing Wiki resource dependency")
}
if _, err = db.Exec(ctx, `UPDATE wiki_pages SET content='new-content' WHERE id='wiki-safe'`); err != nil {
t.Fatal(err)
}
})
t.Run("empty library has explicit empty tables", func(t *testing.T) {
s, err := ExportLibrary(ctx, db, PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-empty"})
if err != nil {
t.Fatal(err)
}
if len(s.Tables) != 10 {
t.Fatal("empty library snapshot incomplete")
}
for _, tab := range s.Tables {
if tab.RowCount != 0 {
t.Fatal("empty library leaked content")
}
}
})
// The exporter may not mutate content: only fixture updates above are visible.
var count int
if err := db.QueryRow(ctx, `SELECT count(*) FROM sessions`).Scan(&count); err != nil || count != 1 {
t.Fatal("nonknowledge data changed")
}
encoded, err := json.Marshal(snap)
if err != nil || strings.Contains(string(encoded), "private-session-marker") {
t.Fatal("nonknowledge data exported")
}
}
type libraryHookDB struct {
pool *pgxpool.Pool
hook func()
}
func (d libraryHookDB) BeginTx(ctx context.Context, o pgx.TxOptions) (pgx.Tx, error) {
tx, err := d.pool.BeginTx(ctx, o)
return &libraryHookTx{Tx: tx, hook: d.hook}, err
}
type libraryHookTx struct {
pgx.Tx
hook func()
fired bool
}
func (d *libraryHookTx) Query(ctx context.Context, q string, args ...any) (pgx.Rows, error) {
if !d.fired {
d.fired = true
d.hook()
}
return d.Tx.Query(ctx, q, args...)
}
const libraryTestSchema = `
CREATE TABLE schema_migrations(version int,dirty bool); INSERT INTO schema_migrations VALUES(74,false);
CREATE TABLE knowledge_bases(id text,tenant_id bigint,indexing_strategy jsonb,deleted_at timestamptz);
INSERT INTO knowledge_bases VALUES('kb-safe',10000,'{}',NULL),('kb-empty',10000,'{}',NULL);
CREATE TABLE knowledges(id text PRIMARY KEY,tenant_id bigint,knowledge_base_id text,parse_status text,pending_subtasks_count int,file_path text,deleted_at timestamptz);
INSERT INTO knowledges VALUES('doc-safe',10000,'kb-safe','completed',0,'resource://ABCDEFGHIJKLMNOPQRSTUV',NULL),('other-scope',20000,'kb-safe','completed',0,'',NULL),('deleted-doc',10000,'kb-safe','completed',0,'',now());
CREATE TABLE chunks(id text,tenant_id bigint,knowledge_base_id text,knowledge_id text,content text,deleted_at timestamptz);
INSERT INTO chunks VALUES('chunk-safe',10000,'kb-safe','doc-safe','chunk',NULL),('other-scope',20000,'kb-safe','other-scope','secret',NULL);
CREATE TABLE knowledge_tags(id text,tenant_id bigint,knowledge_base_id text,deleted_at timestamptz);
CREATE TABLE knowledge_tag_relations(knowledge_id text,tag_id text);
CREATE TABLE resources(id text,handle text,tenant_id bigint,physical_path text,deleted_at timestamptz);
INSERT INTO resources VALUES('res-safe','ABCDEFGHIJKLMNOPQRSTUV',10000,'source.txt',NULL),('other-scope','abcdefghijklmnopqrstuv',20000,'other-scope',NULL);
CREATE TABLE resource_bindings(id text,tenant_id bigint,resource_id text,owner_type text,owner_id text);
INSERT INTO resource_bindings VALUES('binding-safe',10000,'res-safe','knowledge','doc-safe');
CREATE TABLE wiki_pages(id text,tenant_id bigint,knowledge_base_id text,slug text,content text,deleted_at timestamptz);
INSERT INTO wiki_pages VALUES('wiki-safe',10000,'kb-safe','index','old-content',NULL),('other-scope',20000,'kb-safe','secret','other-scope',NULL);
CREATE TABLE wiki_folders(id text,tenant_id bigint,knowledge_base_id text,deleted_at timestamptz);
CREATE TABLE wiki_page_issues(id text,tenant_id bigint,knowledge_base_id text,deleted_at timestamptz);
INSERT INTO wiki_page_issues VALUES('issue-safe',10000,'kb-safe',NULL);
CREATE TABLE wiki_log_entries(id bigint,tenant_id bigint,knowledge_base_id text);
INSERT INTO wiki_log_entries VALUES(1,10000,'kb-safe');
CREATE TABLE task_pending_ops(tenant_id bigint,task_type text,scope text,scope_id text);
CREATE TABLE task_dead_letters(tenant_id bigint,task_type text,scope text,scope_id text);
CREATE TABLE sessions(marker text); INSERT INTO sessions VALUES('private-session-marker');
`