From 07a9caf22b61586bd790bd8a78c08be66a5a5e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 2 Nov 2026 03:46:26 +0800 Subject: [PATCH] =?UTF-8?q?test(ds02):=20=E6=94=B6=E7=B4=A7=E6=98=A0?= =?UTF-8?q?=E5=B0=84/=E5=BC=95=E7=94=A8/=E5=93=A8=E5=85=B5=E4=B8=8E?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E8=BA=AB=E4=BB=BD=20oracle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:Astra 复验已独立复现旧 LocalIDs 循环的三种盲区——目标互换、错误来源集合、 不存在的目标都能绿;引用把 pre/next/parent 混在一个 OR 里,parent 分支没有覆盖; 成功路径没有序列非空比对;零 Embedding 只测了一个 client;旧侧对照用 rev-list 的 258 冒充 0.1.258(该工作树 identity 实际是 0.1.148);runner 在异常/超时路径不保证 清理,读回把任何非零退出都当"不存在"。 改了什么: - 抽出独立语义 mapping checker:三表 (table,column,source,target) 精确集合 + 真实行 对应;持久账本正例与语义副本变异走同一 checker,digest gate 与语义 gate 分开。 - 测试自带十表 scope 声明并与 schema 列集合对拍;逐种引用非空命中,软导航字段显式 列为范围外;外围 ES 完整 _source、res-other 字节各补负控制。 - 零 Embedding 改为发布段独立记账:ES + 对象存储两个可达 client,自定义 /v1/encode 负控制;范围只声明本测试可达 client,正式全链留给 DS03。 - 冻结输入 bundle(十表行/目标种子/序列/ES/对象/DDL,逐表 hash):旧 6fe98c9 与新源码 同一 bundle 文件;旧侧真实 23505,新侧发布成功 + 独立 oracle。 - 测试进程身份:runner 注入 test-only nonce/commit/树摘要,进程自报 executable hash。 - 历史兼容矩阵改同 catalog 初始化,三条序列显式存在并逐例 before/after。 - runner v2(证据根,不在本仓):失败关闭、finally 清理、所有权重核、keep-failure。 验证:make verify 16 项全绿;ds02_runner_v2 三次隔离运行(含负控制与 keep-failure) raw 见 handoffs/develop-release-contract/ds02-v2-*;旧侧探针 ds02-bundle-old2。 --- CHANGELOG.md | 70 +- .../releaseadapter/v071/ds02_bundle_test.go | 319 ++++++++ .../releaseadapter/v071/ds02_identity_test.go | 78 ++ .../v071/ds02_legacy_compat_test.go | 33 +- .../v071/ds02_mapping_oracle_test.go | 242 ++++++ .../v071/ds02_mapping_unit_test.go | 103 +++ .../releaseadapter/v071/ds02_oracle_test.go | 757 +++++++++++++++--- .../v071/library_writeset_integration_test.go | 13 +- scripts/test-scope.json | 2 + 9 files changed, 1477 insertions(+), 140 deletions(-) create mode 100644 internal/releaseadapter/v071/ds02_bundle_test.go create mode 100644 internal/releaseadapter/v071/ds02_identity_test.go create mode 100644 internal/releaseadapter/v071/ds02_mapping_oracle_test.go create mode 100644 internal/releaseadapter/v071/ds02_mapping_unit_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6551135..2213201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,72 @@ # Changelog +## 开发快照(DS02 oracle 独立性、映射语义与运行器失败关闭) + +### Added + +- 新增独立本地号映射 oracle:只接收测试声明的来源行、独立推导的期望目标行、真实目标行 + 与持久账本,按 `(table,column,source,target)` 精确集合比较;目标互换、不存在的目标、 + 被替换的来源、少项/多项/重复、错列都会被判红,而合法账本与真实行逐列相等才算绿。 +- 新增十表范围声明(测试自带 `tenant_id`/`knowledge_base_id` 规则并与真实 schema 列集合 + 对拍),不再复用产品的范围分类生成期望。 +- 新增逐种引用覆盖:`chunk.pre/next/parent_chunk_id`、`wiki_folders.parent_id`、 + `wiki_pages.source_refs/chunk_refs`、`wiki_page_issues.suspected_knowledge_ids`、 + `wiki_log_entries.pages_affected` 等各自 `violations=0` 且非空命中;没有外键契约的软 + 导航字段(`parent_slug`、`in_links/out_links`、`relation_chunks`)显式列为范围外。 +- 新增外围哨兵:`chunk-foreign` 文档完整 `_source` 逐字段比较、`res-other` 对象字节比较, + 两者各有"改坏必须红"的负控制。 +- 新增冻结输入 bundle(十表来源行、目标种子、序列种子、ES 文档、对象字节、完整 DDL, + 时间固定、逐表 hash):旧源码 `6fe98c9` 与当前源码读取同一份文件、同一 sha256。旧侧真实 + 复现 `wiki_log_entries_pkey / SQLSTATE 23505`,新侧同一输入发布成功并通过独立 oracle。 + 差异单列:6fe98c9 在 wiki 之前无法跨租户搬运资源,旧侧归档省略 `resources` / + `resource_bindings` 两表,其余八表与碰撞行逐字节相同;新侧按十表完整运行。 +- 新增测试进程身份断言:runner 注入 test-only nonce/commit/源码树摘要,测试进程自报 + `os.Executable` 的 sha256,与 runner 期望逐字符比较;构建与执行前后各算一次二进制哈希。 +- 新增隔离运行器 v2(`ds02_runner_v2.py`,v1 保留):超时/构建/准备异常都走 finally 清理, + 源码、HEAD、工作区、二进制哈希、bundle 字节、required 缺失/跳过任一变化即失败; + 清理前重核 namespace 前缀与容器 Id/label,读回区分"确实不存在"与认证/连接错误; + `--keep-failure` 保留现场并写 `resources-kept.json`。 + +### Changed + +- DS02 的零 Embedding 断言改为按"发布段"记账:ES 与对象存储两个可达 client 注入同一 + 记录器,按目的地址(路径含 embed/encode 或已知模型域名)识别向量化调用,并补了另一个 + client 走自定义 `/v1/encode` 的负控制;声明范围只到本测试的 client 清单,正式 + API→worker 全链仍留给 DS03。 +- 映射 RED 与摘要 gate 分开:语义变异只改内存账本副本(目标存储不动),归档字节变异只 + 证明 `LoadWriteSet` 摘要拒收,不再拿后者冒充映射断言。 +- 日志内容变异重命名为 `log-content`,不再当作 mapping RED。 +- 历史兼容矩阵改用同一份真实 v071 catalog 初始化,三条本地号序列显式断言存在并逐例比较 + before/after 摘要(不再有空集合"不变")。 +- 旧侧对照措辞修正为"精确旧 SHA 回归对照":不再用 `git rev-list --count` 的 258 当版本号; + 该工作树资源 identity 实际是 0.1.148,没有原发布产物就不做历史考古。 + +### Fixed + +- 修正旧 oracle 只检查"每表来源/目标数量与重复"导致的三种盲区(目标互换、错误来源集合、 + 不存在的目标号)。 +- 修正引用断言把 pre/next/parent 混在一个 OR 里、无法证明 parent 分支有覆盖的问题。 +- 修正"序列不变"在最小 schema 上没有序列可比的空覆盖。 +- 修正运行器把 namespace 解析失败静默跳过、删除前不重核所有权、读回把任何非零退出都当 + "不存在"、失败路径不保证清理的问题。 + ## 开发快照(整库发布的数据/证明 oracle 与历史兼容矩阵) ### Added - 新增 `TestDS02FullShapeOracleIsolated`:复用现有十表全形状夹具,用**独立 oracle** - 逐列比对目标行内容、全部父子/标签/资源/知识库/Wiki 引用、索引正文与向量、对象 - 字节,并断言发布过程中 Embedding 调用为 0。期望值由夹具声明推导(来源→目标范围 - 映射、资源定位符与 `location_hash` 在本测试里重算),不调用被测映射器生成。 -- 新增 Wiki 日志映射双射断言:来源日志数字 id 与目标本地号一一对应、无重复、内容 - 元组多重集相等。 -- 新增对照夹具探针(旧代码工作树):0.1.258(6fe98c9)在同一隔离库/桶/索引里以 - 相同输入复现 `wiki_log_entries_pkey` / SQLSTATE 23505;当前代码同一输入发布成功 - 并分配本地号,其它知识库的历史行不变。 + 逐列比对目标行内容、逐种引用(父子/标签/资源/知识库/Wiki,含 parent 分支与 + source_refs/chunk_refs/suspected ids)、索引正文与向量、目标及外围对象字节,并按发布段 + 断言 Embedding 调用为 0。期望值由夹具声明推导(来源→目标范围映射、资源定位符与 + `location_hash` 在本测试里重算),不调用被测映射器生成;`parent_slug` 与 in/out_links + 这类没有外键契约的软导航字段明确列为范围外。 +- 新增 Wiki 日志映射 oracle:来源日志数字 id 与目标本地号按独立期望精确配对,并用 + 真实目标行逐列对应;持久账本正例通过同一 checker,语义副本变异(互换/不存在/替换来源/ + 少项/多项/重复/错列)必须红。 +- 新增对照夹具探针(旧代码工作树):精确旧 SHA `6fe98c9` 在同一隔离库/桶/索引里以 + 同一份冻结 bundle 复现 `wiki_log_entries_pkey` / SQLSTATE 23505;当前代码同一输入发布 + 成功并分配本地号,其它知识库的历史行不变。这是**旧源码回归对照**,不是原 0.1.258 发布 + 产物;该工作树的资源身份实际是 0.1.148,`git rev-list --count` 的 258 只是自根提交数。 - 新增 `TestDS02LegacyCompatibilityMatrixIsolated`:旧 manifest、V1 台账、空作业/ 目标身份等缺证明组合必须在正式入口被拒绝,拒绝前后 PG 逐行 + 索引 + 对象版本 + 序列哈希不变;不做 V1→V2 迁移。 diff --git a/internal/releaseadapter/v071/ds02_bundle_test.go b/internal/releaseadapter/v071/ds02_bundle_test.go new file mode 100644 index 0000000..8bb31c3 --- /dev/null +++ b/internal/releaseadapter/v071/ds02_bundle_test.go @@ -0,0 +1,319 @@ +package v071 + +// DS-02 冻结输入 bundle:同一份文件同时给旧源码探针(6fe98c9 工作树)与当前源码 +// 使用。bundle 只含业务输入(十表来源行、目标种子行、序列种子、ES 文档、对象字节、 +// 固定时间与完整 DDL);两个隔离目标各自执行同一份 DDL、插入同一批行。 +// +// 唯一的环境替换是对象桶名:bundle 里写 %BUCKET%,两侧各自换成自己的桶。 +// 这不是输入差异,必须在报告里显式列出。 +// +// 导出(不连接任何服务): +// KG_DS02_BUNDLE_OUT=/path/dir go test -run TestDS02ExportFrozenBundle +// 新侧对照: +// KG_DS02_BUNDLE=/path/dir/ds02-frozen-bundle.json go test -run TestDS02FrozenBundleIsolated + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "knowledge-governance/internal/libraryrelease" +) + +type ds02BundleSeedRow struct { + Table string `json:"table"` + Row map[string]any `json:"row"` +} + +type ds02BundleSeedDoc struct { + ID string `json:"id"` + Tenant uint64 `json:"tenant"` + KB string `json:"kb"` + Knowledge string `json:"knowledge"` + Content string `json:"content"` +} + +type ds02FrozenBundle struct { + Bundle string `json:"bundle"` + Version int `json:"version"` + CreatedAt string `json:"created_at"` + Scope map[string]any `json:"scope"` + Sequences map[string][]any `json:"sequences"` + SourceRows map[string][]map[string]any `json:"source_rows"` + TargetRows []ds02BundleSeedRow `json:"target_rows"` + // OldAdapterRows 只给旧源码探针用:6fe98c9 在 resources 阶段按目标范围查资源行 + // (它在 database 阶段不重写租户),所以目标必须先有这两行才能走到 wiki。 + // 这是旧入口适配差异,不是业务输入;新侧不读这个字段。 + OldAdapterRows []ds02BundleSeedRow `json:"old_adapter_target_rows"` + SeedDocs []ds02BundleSeedDoc `json:"seed_docs"` + SeedObjects map[string]string `json:"seed_objects"` + Vectors map[string][]float64 `json:"vectors"` + DDL []string `json:"ddl"` +} + +// ds02BundleDDL 从保留的真实 v071 catalog 生成完整 DDL;旧/新隔离目标执行同一份。 +func ds02BundleDDL(t *testing.T) []string { + t.Helper() + catalog := r11CatalogData(t) + ddl := []string{ + `CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`, + `CREATE EXTENSION IF NOT EXISTS pg_trgm`, + // 预备份的 schema 契约从这张台账读版本;bundle 必须自带,不能靠夹具残留。 + `CREATE TABLE schema_migrations(version int,dirty bool)`, + `INSERT INTO schema_migrations VALUES(74,false)`, + // 旧源码的"库已结清"查询要读这两张任务表;同一份 DDL 两侧都建。 + `CREATE TABLE task_pending_ops(tenant_id bigint,task_type text,scope text,scope_id text)`, + `CREATE TABLE task_dead_letters(tenant_id bigint,task_type text,scope text,scope_id text)`, + `CREATE TABLE knowledge_bases(id text,tenant_id bigint,indexing_strategy jsonb,deleted_at timestamptz)`, + `INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`, + `CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`, + `INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + ds02BucketPlaceholder + `"}',now())`, + } + for _, sequence := range catalog.Sequences { + ddl = append(ddl, fmt.Sprintf("CREATE SEQUENCE %s START %d", sequence.Name, sequence.Start)) + } + for _, table := range productionLibraryTables { + defs := []string{} + for _, column := range catalog.Columns { + if column.Table != table { + continue + } + definition := pgxIdentifier(column.Name) + " " + column.Type + if column.Required { + definition += " NOT NULL" + } + if column.Default != "" { + definition += " DEFAULT " + column.Default + } + defs = append(defs, definition) + } + ddl = append(ddl, "CREATE TABLE "+table+"("+strings.Join(defs, ",")+")") + } + ddl = append(ddl, `ALTER SEQUENCE wiki_log_entries_id_seq OWNED BY wiki_log_entries.id`, + `CREATE TABLE resource_access_grants(id text,resource_id varchar(36))`) + constraints := append([]struct { + Table string `json:"table_name"` + Name string `json:"name"` + Definition string `json:"definition"` + }{}, catalog.Constraints...) + sort.SliceStable(constraints, func(i, j int) bool { + return !strings.HasPrefix(constraints[i].Definition, "FOREIGN KEY") && strings.HasPrefix(constraints[j].Definition, "FOREIGN KEY") + }) + names := map[string]bool{} + for _, constraint := range constraints { + ddl = append(ddl, "ALTER TABLE "+constraint.Table+" ADD CONSTRAINT "+constraint.Name+" "+constraint.Definition) + names[constraint.Name] = true + } + for _, index := range catalog.Indexes { + if names[index.Name] { + continue + } + ddl = append(ddl, index.Definition) + } + return ddl +} + +// pgxIdentifier 只处理本 catalog 里的普通小写列名。 +func pgxIdentifier(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +func ds02BundleSequences() map[string][]any { + return map[string][]any{ + "chunks_seq_id_seq": {int64(100000005), false}, + "knowledge_tags_seq_id_seq": {int64(50), false}, + "wiki_log_entries_id_seq": {int64(10), false}, + } +} + +func TestDS02ExportFrozenBundle(t *testing.T) { + root := os.Getenv("KG_DS02_BUNDLE_OUT") + if root == "" { + t.Skip("set KG_DS02_BUNDLE_OUT to write the frozen input bundle") + } + rows := ds02SourceRows() + // 逐表 hash,便于两侧各自核对同一份输入。 + perTable := map[string]string{} + for table, list := range rows { + raw, err := json.Marshal(list) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + perTable[table] = hex.EncodeToString(sum[:]) + } + bundle := ds02FrozenBundle{ + Bundle: "ds02-frozen-input", + Version: 1, + CreatedAt: "2026-11-02T00:00:00Z", + Scope: map[string]any{"source_tenant": wsSourceTenant, "source_kb": wsSourceKB, "target_tenant": wsTargetTenant, "target_kb": wsTargetKB, "other_kb": wsOtherKB, "foreign_tenant": 20000}, + Sequences: ds02BundleSequences(), + SourceRows: rows, + SeedObjects: func() map[string]string { + out := map[string]string{} + for id, body := range ds02Objects { + out[id] = body + } + for id, body := range ds02SeedObjects() { + out[id] = body + } + return out + }(), + Vectors: ds02Vectors, + DDL: ds02BundleDDL(t), + } + for _, row := range ds02SeedRows() { + bundle.TargetRows = append(bundle.TargetRows, ds02BundleSeedRow{Table: row.table, Row: row.row}) + } + for _, doc := range ds02SeedDocs() { + bundle.SeedDocs = append(bundle.SeedDocs, ds02BundleSeedDoc{ID: doc.id, Tenant: doc.tenant, KB: doc.kb, Knowledge: doc.knowledge, Content: doc.content}) + } + raw, err := json.MarshalIndent(bundle, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(root, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "ds02-frozen-bundle.json") + if err := os.WriteFile(path, append(raw, '\n'), 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(append(raw, '\n')) + hash := hex.EncodeToString(sum[:]) + if err := os.WriteFile(filepath.Join(root, "ds02-frozen-bundle.sha256"), []byte(hash+" ds02-frozen-bundle.json\n"), 0600); err != nil { + t.Fatal(err) + } + result := map[string]any{"bundle": path, "sha256": hash, "table_hashes": perTable, + "bytes": len(raw) + 1, "ddl_statements": len(bundle.DDL), "created_at": bundle.CreatedAt} + if run := os.Getenv("KG_R10_RUN"); run != "" { + r10Save(t, "bundle", result) + } + encoded, _ := json.MarshalIndent(result, "", " ") + t.Logf("%s", encoded) +} + +func ds02LoadBundle(t *testing.T) (ds02FrozenBundle, string) { + t.Helper() + path := os.Getenv("KG_DS02_BUNDLE") + if path == "" { + t.Skip("set KG_DS02_BUNDLE to run the frozen-bundle comparison") + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read frozen bundle: %v", err) + } + sum := sha256.Sum256(raw) + hash := hex.EncodeToString(sum[:]) + sidecar, err := os.ReadFile(strings.TrimSuffix(path, ".json") + ".sha256") + if err != nil { + t.Fatalf("read bundle sha256 sidecar: %v", err) + } + if !strings.HasPrefix(strings.TrimSpace(string(sidecar)), hash) { + t.Fatalf("bundle sha256 %s does not match sidecar %q", hash, strings.TrimSpace(string(sidecar))) + } + var bundle ds02FrozenBundle + if err := json.Unmarshal(raw, &bundle); err != nil { + t.Fatalf("parse frozen bundle: %v", err) + } + if bundle.Bundle != "ds02-frozen-input" || bundle.Version != 1 || len(bundle.DDL) == 0 { + t.Fatalf("unexpected frozen bundle header: %+v", bundle.Bundle) + } + return bundle, hash +} + +// TestDS02FrozenBundleIsolated 用与旧源码探针同一份 bundle 初始化新目标并发布, +// 再用独立 oracle 检查;bundle hash 写进 raw,供与旧侧结果对齐。 +func TestDS02FrozenBundleIsolated(t *testing.T) { + bundle, hash := ds02LoadBundle(t) + // 桶名在两个隔离目标里不同:fixture 在真正建 schema 时把 %BUCKET% 换成自己的桶。 + fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, ddl: bundle.DDL}) + for name, value := range bundle.Sequences { + last := int64(value[0].(float64)) + called := value[1].(bool) + r10Exec(t, fx, fmt.Sprintf("SELECT setval('%s',%d,%t)", name, last, called)) + } + for id, body := range bundle.SeedObjects { + if id != "res-other" { + continue + } + path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id) + if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil { + t.Fatal(err) + } + } + for _, row := range bundle.TargetRows { + if _, ok := ds02ScopeDecl[row.Table]; !ok { + t.Fatalf("bundle target row for undeclared table %s", row.Table) + } + r11Insert(t, fx, row.Table, ds02SeedRow(row.Row, fx.bucket)) + } + for _, doc := range bundle.SeedDocs { + fx.indexDoc(t, doc.ID, doc.Tenant, doc.KB, doc.Knowledge, doc.Content) + } + spec := ds02Spec{rows: map[string][]map[string]any{}, vectors: bundle.Vectors, objects: map[string]string{}} + for table, rows := range bundle.SourceRows { + for _, row := range rows { + spec.rows[table] = append(spec.rows[table], ds02Canon(t, fx, table, row)) + } + } + // 来源对象只取"来源资源行声明的那几个";res-other 是外围哨兵,不参与来源对象比对。 + for _, row := range bundle.SourceRows["resources"] { + id := fmt.Sprint(row["id"]) + body, ok := bundle.SeedObjects[id] + if !ok { + t.Fatalf("bundle has no bytes for source resource %s", id) + } + spec.objects[id] = body + } + preserved := map[string][]map[string]any{} + for _, table := range ds02TargetTables { + preserved[table] = ds02OtherRows(t, fx, table) + } + expectation := ds02Expected(t, fx, spec) + expected, paths := expectation.Rows, expectation.Paths + for _, table := range []string{"resources", "resource_bindings"} { + expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...) + } + expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"], + ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x + WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`, + wsTargetTenant, wsTargetKB)...) + snapshot := ds02Seal(t, fx, spec) + recorder, inventory := ds02AttachEgress(t, fx) + target := fx.target(t) + backups := fx.backups(t, target) + recorder.setPhase("publish") + out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil) + recorder.setPhase("oracle-read") + r10Save(t, "bundle-identity", map[string]any{"path": os.Getenv("KG_DS02_BUNDLE"), "sha256": hash, + "source_tenant": bundle.Scope["source_tenant"], "target_tenant": bundle.Scope["target_tenant"], + "created_at": bundle.CreatedAt, "adapter_note": "DDL 与行内容相同;对象桶名以 %BUCKET% 占位,两侧各自替换。"}) + r10Save(t, "outcome", out) + if out.State != "succeeded" { + t.Fatalf("new source must publish the frozen bundle: %+v", out) + } + if out.Backup.WriteSet == nil { + t.Fatal("write set missing") + } + ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest) + if err != nil { + t.Fatalf("load write set: %v", err) + } + columns := ds02LocalColumns(t, fx, spec, expectation) + mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs) + r10Save(t, "mapping-diffs", mappingDiffs) + if len(mappingDiffs) != 0 { + t.Fatalf("frozen-bundle publish fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n")) + } + diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory) + r10Save(t, "green-diffs", diffs) + if len(diffs) != 0 { + t.Fatalf("frozen-bundle independent oracle disagrees:\n%s", strings.Join(diffs, "\n")) + } + r10Save(t, "sequences-after", ds02SequenceSummary(t, fx)) +} diff --git a/internal/releaseadapter/v071/ds02_identity_test.go b/internal/releaseadapter/v071/ds02_identity_test.go new file mode 100644 index 0000000..2f09333 --- /dev/null +++ b/internal/releaseadapter/v071/ds02_identity_test.go @@ -0,0 +1,78 @@ +package v071 + +// DS-02 测试进程身份:把"runner 记录的那份源码构建的二进制"与"真正执行的进程" +// 绑定起来。runner 用 -ldflags 注入三个只在本测试包存在的值(run nonce、完整 +// commit、源码树摘要),并在启动前后各算一次二进制 sha256;这里在进程内读 +// os.Executable 的哈希,与 runner 通过环境变量声明的期望值逐字符比较。 +// +// 这些值不是产品 metadata:不参与发布、不改 internal/version。 + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "testing" +) + +// 由 runner 的 -ldflags -X 注入;未注入时保留显眼的值,绝不伪装成已知身份。 +var ( + ds02ProvenanceNonce = "unlinked" + ds02ProvenanceCommit = "unlinked" + ds02ProvenanceTree = "unlinked" +) + +type ds02ProcessIdentity struct { + Nonce string `json:"nonce"` + Commit string `json:"commit"` + SourceTree string `json:"source_tree"` + Executable string `json:"executable"` + BinarySHA256 string `json:"binary_sha256"` +} + +func TestDS02ProcessIdentity(t *testing.T) { + expectedNonce := os.Getenv("KG_DS02_EXPECT_NONCE") + if expectedNonce == "" { + t.Skip("requires runner-injected expectations (KG_DS02_EXPECT_NONCE)") + } + executable, err := os.Executable() + if err != nil { + t.Fatalf("resolve own executable: %v", err) + } + raw, err := os.ReadFile(executable) + if err != nil { + t.Fatalf("read own executable: %v", err) + } + sum := sha256.Sum256(raw) + identity := ds02ProcessIdentity{ + Nonce: ds02ProvenanceNonce, + Commit: ds02ProvenanceCommit, + SourceTree: ds02ProvenanceTree, + Executable: executable, + BinarySHA256: hex.EncodeToString(sum[:]), + } + encoded, err := json.Marshal(identity) + if err != nil { + t.Fatal(err) + } + // 单行输出给 runner 解析:证明是"这个进程自报",不是 runner 事后补写。 + t.Logf("DS02-IDENTITY %s", encoded) + cases := []struct { + name string + linked string + expected string + }{ + {"nonce", identity.Nonce, expectedNonce}, + {"commit", identity.Commit, os.Getenv("KG_DS02_EXPECT_COMMIT")}, + {"source tree", identity.SourceTree, os.Getenv("KG_DS02_EXPECT_TREE")}, + {"binary sha256", identity.BinarySHA256, os.Getenv("KG_DS02_EXPECT_BINARY_SHA256")}, + } + for _, item := range cases { + if item.expected == "" { + t.Fatalf("%s expectation missing from runner environment", item.name) + } + if item.linked != item.expected { + t.Fatalf("%s mismatch: process has %q, runner expected %q", item.name, item.linked, item.expected) + } + } +} \ No newline at end of file diff --git a/internal/releaseadapter/v071/ds02_legacy_compat_test.go b/internal/releaseadapter/v071/ds02_legacy_compat_test.go index 14dc767..4a079e5 100644 --- a/internal/releaseadapter/v071/ds02_legacy_compat_test.go +++ b/internal/releaseadapter/v071/ds02_legacy_compat_test.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -82,9 +83,36 @@ func ds02OldManifest(t *testing.T, fx *writeSetFixture) (string, string) { return id, hex.EncodeToString(fileSum[:]) } +// ds02LegacySeed 用与 full-shape 用例同一份真实 v071 catalog(r11Mirror)造目标行: +// 三条本地号序列必须真实存在,否则"序列不变"就是空覆盖。 +func ds02LegacySeed(t *testing.T, fx *writeSetFixture) { + t.Helper() + for _, row := range []struct { + table string + row map[string]any + }{ + {"knowledges", map[string]any{"id": "doc-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "type": "document", "title": "目标旧文档"}}, + {"chunks", map[string]any{"id": "chunk-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "knowledge_id": "doc-target", "content": "目标旧正文", "chunk_index": 0}}, + {"knowledge_tags", map[string]any{"id": "tag-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "name": "目标标签"}}, + {"knowledge_tag_relations", map[string]any{"knowledge_id": "doc-target", "tag_id": "tag-target"}}, + {"resources", map[string]any{"id": "res-target", "handle": wsHandleTarget, "tenant_id": wsTargetTenant, "provider": "minio", "storage_backend_id": "backend-target", "physical_path": wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-target"), "size": int64(8)}}, + {"resource_bindings", map[string]any{"id": "bind-target", "tenant_id": wsTargetTenant, "resource_id": "res-target", "owner_type": "knowledge", "owner_id": "doc-target", "relation": "attachment"}}, + {"wiki_folders", map[string]any{"id": "folder-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "name": "目标目录"}}, + {"wiki_pages", map[string]any{"id": "wiki-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "slug": "index", "content": "目标旧 wiki"}}, + {"wiki_page_issues", map[string]any{"id": "issue-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "slug": "index", "issue_type": "review"}}, + {"wiki_log_entries", map[string]any{"id": int64(5), "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "action": "ingest"}}, + } { + r11Insert(t, fx, row.table, row.row) + } +} + func TestDS02LegacyCompatibilityMatrixIsolated(t *testing.T) { fx := openWriteSetFixture(t) - seedTargetData(t, fx) + // 同 catalog 初始化:十表真实列/约束/三条序列都在,目标行不是最小 schema 的残影。 + r11Mirror(t, fx) + ds02LegacySeed(t, fx) + sequences := ds02SequenceSummary(t, fx) + r10Save(t, "sequences-before", sequences) target := fx.target(t) oldManifest, oldManifestHash := ds02OldManifest(t, fx) before := ds02UniverseHash(t, fx) @@ -169,6 +197,9 @@ func TestDS02LegacyCompatibilityMatrixIsolated(t *testing.T) { if after := ds02UniverseHash(t, fx); after != before { t.Fatalf("%s: rejected plan changed target data (%s -> %s)", scenario.name, before, after) } + if after := ds02SequenceSummary(t, fx); fmt.Sprint(after) != fmt.Sprint(sequences) { + t.Fatalf("%s: rejected plan changed local sequences (%v -> %v)", scenario.name, sequences, after) + } } r10Save(t, "universe-hash", before) } diff --git a/internal/releaseadapter/v071/ds02_mapping_oracle_test.go b/internal/releaseadapter/v071/ds02_mapping_oracle_test.go new file mode 100644 index 0000000..9173c85 --- /dev/null +++ b/internal/releaseadapter/v071/ds02_mapping_oracle_test.go @@ -0,0 +1,242 @@ +package v071 + +// DS-02 本地号映射 oracle。 +// +// 旧检查只数"每表来源/目标各几个、是否重复",因此三种语义错误能一路绿: +// 目标互换(1→11,2→10)、来源集合被替换(999→10)、目标根本不存在(1→999)。 +// 这里的检查器只接收四样东西: +// * 测试声明的来源行(数字 id 来自夹具声明,不由被测映射器产生); +// * 独立推导的期望目标行(来源 id 经过独立算出的分配顺序得到目标数字 id); +// * 真实目标行(按目标数字 id 从数据库读回); +// * 持久账本(WriteSet.LocalIDs)。 +// +// 它不调用任何产品映射函数,也不读写存储:把 (table,column,source,target) 当 +// 精确集合比较,并用 source 关联声明行、用 target 关联真实目标行。因此 +// "账本里的配对本身正确、真实行也正确" 和 "账本把配对写错" 是两件独立的事: +// 语义副本变异只改账本内存副本,不会碰 digest 校验;反过来,改归档字节让摘要 +// 拒绝也不等于映射语义检查红,两者必须分开报告。 + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + + "knowledge-governance/internal/libraryrelease" +) + +// ds02LocalColumn 是一张含本地号的表的独立期望与真实读回。 +type ds02LocalColumn struct { + Table string + Column string + // Sources 是夹具声明的来源数字 id(升序,即分配顺序)。 + Sources []int64 + // Expected 是 source→target 的独立期望(来源序列起点 + 声明顺序)。 + Expected map[int64]int64 + // ExpectedRows 是 source→独立推导的完整目标行(逐列,含真实目标数字 id)。 + ExpectedRows map[int64]map[string]any + // Actual 是真实目标行:target 数字 id→整行。 + Actual map[int64]map[string]any +} + +func ds02RowKey(row map[string]any) string { + raw, err := json.Marshal(row) + if err != nil { + return fmt.Sprintf("", err) + } + return string(raw) +} + +// ds02CheckLocalIDMapping 返回差异;空表示账本与独立期望一致,且每个配对都能 +// 落到真实目标行上。 +func ds02CheckLocalIDMapping(columns []ds02LocalColumn, ledger []libraryrelease.LocalIDMapping) []string { + var diffs []string + declared := map[string]*ds02LocalColumn{} + for index := range columns { + column := &columns[index] + if _, ok := declared[column.Table]; ok { + diffs = append(diffs, "duplicate declared local column for "+column.Table) + continue + } + declared[column.Table] = column + } + // 解析账本:逐条严格校验,任何解析失败都是差异,不静默跳过。 + type pair struct { + source int64 + target int64 + } + observed := map[string]map[int64]int64{} + seenSource := map[string]map[int64]bool{} + seenTarget := map[string]map[int64]bool{} + seenColumn := map[string]map[string]bool{} + for _, mapping := range ledger { + column, ok := declared[mapping.Table] + if !ok { + diffs = append(diffs, "unexpected ledger table "+mapping.Table) + continue + } + if mapping.Column != column.Column { + diffs = append(diffs, fmt.Sprintf("%s ledger column %q, want %q", mapping.Table, mapping.Column, column.Column)) + continue + } + source, err := strconv.ParseInt(mapping.Source, 10, 64) + if err != nil || source <= 0 { + diffs = append(diffs, fmt.Sprintf("%s ledger source %q is not a positive integer", mapping.Table, mapping.Source)) + continue + } + target, err := strconv.ParseInt(mapping.Target, 10, 64) + if err != nil || target <= 0 { + diffs = append(diffs, fmt.Sprintf("%s ledger target %q is not a positive integer", mapping.Table, mapping.Target)) + continue + } + if observed[mapping.Table] == nil { + observed[mapping.Table] = map[int64]int64{} + seenSource[mapping.Table] = map[int64]bool{} + seenTarget[mapping.Table] = map[int64]bool{} + seenColumn[mapping.Table] = map[string]bool{} + } + if seenSource[mapping.Table][source] { + diffs = append(diffs, fmt.Sprintf("%s duplicate ledger source %d", mapping.Table, source)) + } + if seenTarget[mapping.Table][target] { + diffs = append(diffs, fmt.Sprintf("%s duplicate ledger target %d", mapping.Table, target)) + } + key := strconv.FormatInt(source, 10) + "->" + strconv.FormatInt(target, 10) + if seenColumn[mapping.Table][key] { + diffs = append(diffs, fmt.Sprintf("%s duplicate ledger pair %s", mapping.Table, key)) + } + seenSource[mapping.Table][source] = true + seenTarget[mapping.Table][target] = true + seenColumn[mapping.Table][key] = true + observed[mapping.Table][source] = target + } + for _, column := range columns { + table := column.Table + got := observed[table] + // 1) 来源集合必须精确等于声明集合。 + var missing []int64 + expectedTargets := map[int64]bool{} + for _, source := range column.Sources { + want, ok := column.Expected[source] + if !ok { + diffs = append(diffs, fmt.Sprintf("%s source %d has no independent expectation", table, source)) + continue + } + expectedTargets[want] = true + actual, ok := got[source] + if !ok { + missing = append(missing, source) + continue + } + if actual != want { + diffs = append(diffs, fmt.Sprintf("%s wrong target pair: source %d mapped to %d, want %d", table, source, actual, want)) + } + // 3) 来源 id 关联声明行、target id 关联真实目标行:必须逐列相等。 + wantRow, ok := column.ExpectedRows[source] + if !ok { + diffs = append(diffs, fmt.Sprintf("%s source %d has no expected target row", table, source)) + continue + } + actualRow, ok := column.Actual[want] + if !ok { + diffs = append(diffs, fmt.Sprintf("%s target %d has no real target row", table, want)) + continue + } + if ds02RowKey(wantRow) != ds02RowKey(actualRow) { + diffs = append(diffs, fmt.Sprintf("%s source %d -> target %d row mismatch: want %s got %s", + table, source, want, ds02RowKey(wantRow), ds02RowKey(actualRow))) + } + } + var extraSources []int64 + for source := range got { + if _, ok := column.Expected[source]; !ok { + extraSources = append(extraSources, source) + } + } + if len(missing) > 0 { + sort.Slice(missing, func(i, j int) bool { return missing[i] < missing[j] }) + diffs = append(diffs, fmt.Sprintf("%s missing ledger source %v", table, missing)) + } + if len(extraSources) > 0 { + sort.Slice(extraSources, func(i, j int) bool { return extraSources[i] < extraSources[j] }) + diffs = append(diffs, fmt.Sprintf("%s unexpected ledger source %v", table, extraSources)) + } + // 2) target 集合必须精确等于独立预测集合,而且确实存在这些目标行。 + for _, target := range got { + if !expectedTargets[target] { + diffs = append(diffs, fmt.Sprintf("%s ledger target %d is not an independently predicted target", table, target)) + } + } + for target := range expectedTargets { + if _, ok := column.Actual[target]; !ok { + diffs = append(diffs, fmt.Sprintf("%s predicted target %d does not exist in target storage", table, target)) + } + } + if len(got) == 0 && len(column.Sources) > 0 { + diffs = append(diffs, fmt.Sprintf("%s has no ledger mapping rows", table)) + } + } + return diffs +} + +// ds02FlipMappingTargets 互换同一张表两条映射的目标,模拟"配对写反"。 +func ds02FlipMappingTargets(ledger []libraryrelease.LocalIDMapping, table string) []libraryrelease.LocalIDMapping { + out := append([]libraryrelease.LocalIDMapping(nil), ledger...) + first := -1 + for index := range out { + if out[index].Table != table { + continue + } + if first < 0 { + first = index + continue + } + out[first].Target, out[index].Target = out[index].Target, out[first].Target + return out + } + return out +} + +// ds02ReplaceMapping 把同一张表第一条映射的某个字段替换成新值。 +func ds02ReplaceMapping(ledger []libraryrelease.LocalIDMapping, table, field, value string) []libraryrelease.LocalIDMapping { + out := append([]libraryrelease.LocalIDMapping(nil), ledger...) + for index := range out { + if out[index].Table != table { + continue + } + switch field { + case "source": + out[index].Source = value + case "target": + out[index].Target = value + case "column": + out[index].Column = value + } + return out + } + return out +} + +// ds02DropMapping 删掉同一张表的最后一条映射(少项)。 +func ds02DropMapping(ledger []libraryrelease.LocalIDMapping, table string) []libraryrelease.LocalIDMapping { + out := append([]libraryrelease.LocalIDMapping(nil), ledger...) + for index := len(out) - 1; index >= 0; index-- { + if out[index].Table != table { + continue + } + return append(out[:index], out[index+1:]...) + } + return out +} + +// ds02DuplicateMapping 复制同一张表的第一条映射(多项/重复来源与目标)。 +func ds02DuplicateMapping(ledger []libraryrelease.LocalIDMapping, table string) []libraryrelease.LocalIDMapping { + out := append([]libraryrelease.LocalIDMapping(nil), ledger...) + for index := range out { + if out[index].Table != table { + continue + } + return append(out, out[index]) + } + return out +} diff --git a/internal/releaseadapter/v071/ds02_mapping_unit_test.go b/internal/releaseadapter/v071/ds02_mapping_unit_test.go new file mode 100644 index 0000000..df54474 --- /dev/null +++ b/internal/releaseadapter/v071/ds02_mapping_unit_test.go @@ -0,0 +1,103 @@ +package v071 + +// 映射 oracle 自身的负例矩阵:这里不连数据库,只用合成行证明检查器对 +// "目标互换 / 不存在目标 / 来源被替换 / 少项 / 多项 / 重复 / 错列" 会红, +// 且对合法账本为绿。真实账本正例与语义副本变异在 full-shape 用例里跑同一个 +// 检查器(那里才连接真实 PG/ES/MinIO)。 + +import ( + "strings" + "testing" + + "knowledge-governance/internal/libraryrelease" +) + +func ds02UnitColumns() []ds02LocalColumn { + rows := func(target int64, id string) map[string]any { + return map[string]any{"id": id, "seq_id": target, "tenant_id": 10001} + } + return []ds02LocalColumn{{ + Table: "chunks", + Column: "seq_id", + Sources: []int64{41, 42}, + Expected: map[int64]int64{ + 41: 100, + 42: 101, + }, + ExpectedRows: map[int64]map[string]any{ + 41: rows(100, "chunk-src"), + 42: rows(101, "chunk-src2"), + }, + Actual: map[int64]map[string]any{ + 100: rows(100, "chunk-src"), + 101: rows(101, "chunk-src2"), + }, + }} +} + +func ds02UnitLedger() []libraryrelease.LocalIDMapping { + return []libraryrelease.LocalIDMapping{ + {Table: "chunks", Column: "seq_id", Source: "41", Target: "100"}, + {Table: "chunks", Column: "seq_id", Source: "42", Target: "101"}, + } +} + +func TestDS02MappingOracleUnitCases(t *testing.T) { + valid := ds02CheckLocalIDMapping(ds02UnitColumns(), ds02UnitLedger()) + if len(valid) != 0 { + t.Fatalf("valid ledger must be green, got %v", valid) + } + cases := []struct { + name string + build func() []libraryrelease.LocalIDMapping + want string + }{ + {"swapped-targets", func() []libraryrelease.LocalIDMapping { + return ds02FlipMappingTargets(ds02UnitLedger(), "chunks") + }, "wrong target pair"}, + {"nonexistent-target", func() []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ds02UnitLedger(), "chunks", "target", "999") + }, "not an independently predicted target"}, + {"replaced-source", func() []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ds02UnitLedger(), "chunks", "source", "999") + }, "unexpected ledger source"}, + {"missing-entry", func() []libraryrelease.LocalIDMapping { + return ds02DropMapping(ds02UnitLedger(), "chunks") + }, "missing ledger source"}, + {"duplicate-pair", func() []libraryrelease.LocalIDMapping { + return ds02DuplicateMapping(ds02UnitLedger(), "chunks") + }, "duplicate ledger source"}, + {"wrong-column", func() []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ds02UnitLedger(), "chunks", "column", "id") + }, "ledger column"}, + {"garbled-source", func() []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ds02UnitLedger(), "chunks", "source", "41x") + }, "not a positive integer"}, + {"unexpected-table", func() []libraryrelease.LocalIDMapping { + ledger := ds02UnitLedger() + return append(ledger, libraryrelease.LocalIDMapping{Table: "knowledges", Column: "seq_id", Source: "1", Target: "2"}) + }, "unexpected ledger table"}, + } + for _, scenario := range cases { + diffs := ds02CheckLocalIDMapping(ds02UnitColumns(), scenario.build()) + joined := strings.Join(diffs, "\n") + if len(diffs) == 0 { + t.Fatalf("%s: checker stayed green", scenario.name) + } + if !strings.Contains(joined, scenario.want) { + t.Fatalf("%s: failed for an unrelated reason:\n%s", scenario.name, joined) + } + } +} + +// 真实目标行被改错一个字也必须红:配对正确但行内容不对不算通过。 +func TestDS02MappingOracleChecksTargetRowContent(t *testing.T) { + columns := ds02UnitColumns() + columns[0].Actual[101] = map[string]any{"id": "chunk-src2", "seq_id": 101, "tenant_id": 10001, "content": "tampered"} + columns[0].ExpectedRows[42] = map[string]any{"id": "chunk-src2", "seq_id": 101, "tenant_id": 10001, "content": "expected"} + diffs := ds02CheckLocalIDMapping(columns, ds02UnitLedger()) + joined := strings.Join(diffs, "\n") + if !strings.Contains(joined, "row mismatch") { + t.Fatalf("row content tamper must be red, got %v", diffs) + } +} diff --git a/internal/releaseadapter/v071/ds02_oracle_test.go b/internal/releaseadapter/v071/ds02_oracle_test.go index 836b29a..dc7905b 100644 --- a/internal/releaseadapter/v071/ds02_oracle_test.go +++ b/internal/releaseadapter/v071/ds02_oracle_test.go @@ -17,12 +17,18 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" + "net/url" + "os" "sort" "strconv" "strings" + "sync" "testing" "time" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" ) @@ -32,6 +38,7 @@ import ( var ds02Vectors = map[string][]float64{ "chunk-src": {0.5, 1.5}, "chunk-src2": {-2.25, 3.75}, + "chunk-src3": {1.0, -1.0}, } // ds02Objects 是夹具声明的资源字节(长度与 resources.size 一致)。 @@ -164,8 +171,9 @@ type ds02Spec struct { objects map[string]string } -func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec { - t.Helper() +// ds02SourceRows 是来源十表的纯声明(未按目标 schema 规范化);bundle 导出与 +// 真跑夹具共用,保证"同一输入"来自同一份声明。 +func ds02SourceRows() map[string][]map[string]any { tag := func(id string, seq int64) map[string]any { return map[string]any{"id": id, "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "name": "name-" + id, "color": "#112233", "sort_order": 3, "seq_id": seq, @@ -208,6 +216,12 @@ func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec { "start_at": 12, "end_at": 20, "pre_chunk_id": "chunk-src", "chunk_type": "text", "status": 1, "flags": 1, "seq_id": int64(42), "content_hash": strings.Repeat("4", 64), "created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"}, + // 第三块专门给 parent_chunk_id 分支一个非空命中(旧夹具全为 null)。 + {"id": "chunk-src3", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, + "knowledge_id": "doc-src", "content": "来源正文三", "chunk_index": 2, "is_enabled": true, + "start_at": 20, "end_at": 28, "parent_chunk_id": "chunk-src", "chunk_type": "text", + "status": 1, "flags": 1, "seq_id": int64(43), "content_hash": strings.Repeat("7", 64), + "created_at": "2026-01-05T00:00:00Z", "updated_at": "2026-01-06T00:00:00Z"}, }, "knowledge_tags": {tag("tag-src", 11), tag("tag-src2", 12)}, "knowledge_tag_relations": { @@ -279,6 +293,12 @@ func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec { "created_at": "2026-01-03T00:00:00Z"}, }, } + return rows +} + +func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec { + t.Helper() + rows := ds02SourceRows() for table, list := range rows { for i, row := range list { rows[table][i] = ds02Canon(t, fx, table, row) @@ -287,8 +307,17 @@ func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec { return ds02Spec{rows: rows, vectors: ds02Vectors, objects: ds02Objects} } +// ds02Expectation 是独立推导的期望:逐表逐列的行、资源定位符、本地号分配, +// 以及"来源数字 id → 期望目标行"的对应(mapping oracle 用它把配对落到真实行)。 +type ds02Expectation struct { + Rows map[string][]map[string]any + Paths map[string]string + Local map[string]map[int64]int64 + RowsBySource map[string]map[int64]map[string]any +} + // ds02Expected 从声明内容独立推导目标范围应有的十表逐列内容。 -func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) (map[string][]map[string]any, map[string]string) { +func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) ds02Expectation { t.Helper() targetBackend, targetProvider := "backend-target", "minio" locator := func(id string) string { @@ -323,25 +352,28 @@ func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) (map[string] } expected := map[string][]map[string]any{} paths := map[string]string{} + rowsBySource := map[string]map[int64]map[string]any{} for table, rows := range spec.rows { for _, declared := range rows { row := map[string]any{} for key, value := range declared { row[key] = value } - if tableHasTenant(table) { + rule := ds02ScopeDecl[table] + if rule.Tenant { row["tenant_id"] = json.Number(strconv.FormatUint(wsTargetTenant, 10)) } - if tablesWithKB[table] { + if rule.KnowledgeBase { row["knowledge_base_id"] = wsTargetKB } + var sourceKey int64 switch table { case "chunks", "knowledge_tags": - source, _ := strconv.ParseInt(fmt.Sprint(row["seq_id"]), 10, 64) - row["seq_id"] = json.Number(strconv.FormatInt(local[table][source], 10)) + sourceKey, _ = strconv.ParseInt(fmt.Sprint(row["seq_id"]), 10, 64) + row["seq_id"] = json.Number(strconv.FormatInt(local[table][sourceKey], 10)) case "wiki_log_entries": - source, _ := strconv.ParseInt(fmt.Sprint(row["id"]), 10, 64) - row["id"] = json.Number(strconv.FormatInt(local[table][source], 10)) + sourceKey, _ = strconv.ParseInt(fmt.Sprint(row["id"]), 10, 64) + row["id"] = json.Number(strconv.FormatInt(local[table][sourceKey], 10)) case "resources": id := fmt.Sprint(row["id"]) expectedPath := locator(id) @@ -352,23 +384,30 @@ func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) (map[string] row["storage_backend_id"] = targetBackend row["provider"] = targetProvider } - expected[table] = append(expected[table], ds02Canon(t, fx, table, row)) + canonical := ds02Canon(t, fx, table, row) + expected[table] = append(expected[table], canonical) + if _, ok := local[table]; ok { + if rowsBySource[table] == nil { + rowsBySource[table] = map[int64]map[string]any{} + } + rowsBySource[table][sourceKey] = canonical + } } } - return expected, paths + return ds02Expectation{Rows: expected, Paths: paths, Local: local, RowsBySource: rowsBySource} } -// ds02Seed 造目标/其它范围的真实行;目标范围里也放一行"会被整库替换"的数据。 -func ds02Seed(t *testing.T, fx *writeSetFixture) { - t.Helper() - put := func(id string, size int) { - path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id) - if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(strings.Repeat("z", size)), int64(size)); err != nil { - t.Fatal(err) - } - } - put("res-other", 8) - for _, row := range []struct { +// ds02BucketPlaceholder 让冻结 bundle 不绑定某个一次性桶名:两个隔离目标各自把 +// 环境桶替换进来,输入内容(行、id、字节)本身逐字节相同。 +const ds02BucketPlaceholder = "%BUCKET%" + +// ds02SeedRows 是"发布前目标实例本来就有的数据"的声明。bundle 导出与夹具插入共用 +// 这一份,避免两份行内容漂移。 +func ds02SeedRows() []struct { + table string + row map[string]any +} { + return []struct { table string row map[string]any }{ @@ -377,7 +416,7 @@ func ds02Seed(t *testing.T, fx *writeSetFixture) { {"chunks", map[string]any{"id": "chunk-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "knowledge_id": "doc-other", "content": "别的知识库正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(7)}}, {"knowledge_tags", map[string]any{"id": "tag-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库标签", "seq_id": int64(9)}}, {"knowledge_tag_relations", map[string]any{"knowledge_id": "doc-other", "tag_id": "tag-other"}}, - {"resources", map[string]any{"id": "res-other", "handle": wsHandleOther, "tenant_id": 10001, "provider": "minio", "physical_path": wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other"), "storage_backend_id": "backend-target", "size": int64(8)}}, + {"resources", map[string]any{"id": "res-other", "handle": wsHandleOther, "tenant_id": 10001, "provider": "minio", "physical_path": wsLocator("backend-target", "minio", ds02BucketPlaceholder, 10001, "res-other"), "storage_backend_id": "backend-target", "size": int64(8)}}, {"resource_bindings", map[string]any{"id": "bind-other", "resource_id": "res-other", "tenant_id": 10001, "owner_type": "knowledge", "owner_id": "doc-other", "relation": "attachment"}}, {"wiki_folders", map[string]any{"id": "folder-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库目录"}}, {"wiki_pages", map[string]any{"id": "wiki-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "slug": "index", "content": "别的知识库 wiki"}}, @@ -385,18 +424,78 @@ func ds02Seed(t *testing.T, fx *writeSetFixture) { // 这条是 B03 的碰撞输入:来源日志数字 id=1 与其它知识库已有的 id=1 同值。 {"wiki_log_entries", map[string]any{"id": int64(1), "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "action": "ingest"}}, // 目标范围自己的数据:整库发布必须把它们替换成来源内容。 - {"knowledges", map[string]any{"id": "doc-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "type": "document", "title": "目标旧文档", "source": "x"}}, + {"knowledges", map[string]any{"id": "doc-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "type": "document", "title": "目标旧文档", "source": "x", "parse_status": "completed", "pending_subtasks_count": 0}}, {"chunks", map[string]any{"id": "chunk-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "knowledge_id": "doc-target", "content": "目标旧正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(8)}}, {"wiki_pages", map[string]any{"id": "wiki-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "slug": "index", "content": "目标旧 wiki"}}, - } { - r11Insert(t, fx, row.table, row.row) } - fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "目标旧正文") - fx.indexDoc(t, "chunk-foreign", 20000, "kb-foreign", "doc-foreign", "外部正文") +} + +// ds02SeedDocs 是发布前就存在的索引文档(目标旧文档 + 外围租户文档)。 +func ds02SeedDocs() []struct { + id string + tenant uint64 + kb string + knowledge string + content string +} { + return []struct { + id string + tenant uint64 + kb string + knowledge string + content string + }{ + {"chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "目标旧正文"}, + {"chunk-foreign", 20000, "kb-foreign", "doc-foreign", "外部正文"}, + } +} + +// ds02SeedObjects 是发布前就存在的对象字节(外围对象 res-other 用于哨兵)。 +func ds02SeedObjects() map[string]string { + return map[string]string{"res-other": strings.Repeat("z", 8)} +} + +func ds02SubstituteBucket(value string, bucket string) string { + return strings.ReplaceAll(value, ds02BucketPlaceholder, bucket) +} + +func ds02Seed(t *testing.T, fx *writeSetFixture) { + t.Helper() + for id, body := range ds02SeedObjects() { + path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id) + if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil { + t.Fatal(err) + } + } + for _, row := range ds02SeedRows() { + r11Insert(t, fx, row.table, ds02SeedRow(row.row, fx.bucket)) + } + for _, doc := range ds02SeedDocs() { + fx.indexDoc(t, doc.id, doc.tenant, doc.kb, doc.knowledge, doc.content) + } +} + +// ds02SeedRow 返回替换了环境桶的副本,不改共享声明。 +func ds02SeedRow(row map[string]any, bucket string) map[string]any { + out := map[string]any{} + for key, value := range row { + if text, ok := value.(string); ok { + out[key] = ds02SubstituteBucket(text, bucket) + } else { + out[key] = value + } + } + return out } // ds02Seal 用声明内容封一份十表全非空快照(索引向量来自夹具声明)。 func ds02Seal(t *testing.T, fx *writeSetFixture, spec ds02Spec) string { + t.Helper() + return ds02SealScope(t, fx, spec, wsSourceTenant, wsSourceKB) +} + +// ds02SealScope 允许指定来源范围:冻结 bundle 用"同租户不同知识库"的对照场景。 +func ds02SealScope(t *testing.T, fx *writeSetFixture, spec ds02Spec, sourceTenant uint64, sourceKB string) string { t.Helper() archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024) if err != nil { @@ -445,7 +544,7 @@ func ds02Seal(t *testing.T, fx *writeSetFixture, spec ds02Spec) string { indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows)) parts = append(parts, indexPart) manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, - Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB, + Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB, CreatedAt: time.Now().UTC(), Parts: parts, Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", @@ -493,32 +592,212 @@ func ds02Docs(t *testing.T, fx *writeSetFixture) map[string]map[string]any { return out } -func ds02CountingClient(t *testing.T, fx *writeSetFixture) *[]string { +// ds02ScopeRule 是本测试自己声明的表范围规则,用来独立推导目标行该按什么范围读。 +// 刻意不读产品的 tableHasTenant/tablesWithKB:期望与查询共用产品的范围分类时, +// "两边一起错" 会被判绿。声明本身还要与真实 schema 的列集合对拍(ds02VerifyScopeDecl)。 +type ds02ScopeRule struct { + Tenant bool + KnowledgeBase bool +} + +var ds02ScopeDecl = map[string]ds02ScopeRule{ + "knowledges": {Tenant: true, KnowledgeBase: true}, + "chunks": {Tenant: true, KnowledgeBase: true}, + "knowledge_tags": {Tenant: true, KnowledgeBase: true}, + "knowledge_tag_relations": {Tenant: false, KnowledgeBase: false}, + "resources": {Tenant: true, KnowledgeBase: false}, + "resource_bindings": {Tenant: true, KnowledgeBase: false}, + "wiki_folders": {Tenant: true, KnowledgeBase: true}, + "wiki_pages": {Tenant: true, KnowledgeBase: true}, + "wiki_page_issues": {Tenant: true, KnowledgeBase: true}, + "wiki_log_entries": {Tenant: true, KnowledgeBase: true}, +} + +// ds02VerifyScopeDecl 把声明的表/列集合与真实 schema 对拍:表集合必须恰好是这十张, +// 每张表是否存在 tenant_id / knowledge_base_id 列必须与声明一致。 +func ds02VerifyScopeDecl(t *testing.T, fx *writeSetFixture) { t.Helper() - seen := &[]string{} - base := http.DefaultTransport - fx.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { - *seen = append(*seen, request.Method+" "+request.URL.String()) + declared := map[string]bool{} + for table := range ds02ScopeDecl { + declared[table] = true + } + actual := map[string]bool{} + for _, table := range ds02TargetTables { + actual[table] = true + } + if fmt.Sprint(declared) != fmt.Sprint(actual) { + t.Fatalf("scope declaration tables %v do not match target table set %v", declared, actual) + } + for _, table := range ds02TargetTables { + rule := ds02ScopeDecl[table] + rows := ds02Read(t, fx, `SELECT jsonb_build_object('c', column_name)::text AS row FROM information_schema.columns + WHERE table_schema='public' AND table_name=$1`, table) + hasTenant, hasKB := false, false + for _, row := range rows { + switch fmt.Sprint(row["c"]) { + case "tenant_id": + hasTenant = true + case "knowledge_base_id": + hasKB = true + } + } + if hasTenant != rule.Tenant || hasKB != rule.KnowledgeBase { + t.Fatalf("%s scope declaration (tenant=%v kb=%v) disagrees with schema (tenant=%v kb=%v)", + table, rule.Tenant, rule.KnowledgeBase, hasTenant, hasKB) + } + } + r10Save(t, "scope-declaration", ds02ScopeDecl) +} + +// ds02Egress 是一条被观测到的出站请求。phase 把"发布段"与"oracle 读段/夹具" +// 分开记账:只有发布段的 embedding 计数才能支撑零调用声明。 +type ds02Egress struct { + Phase string `json:"phase"` + Client string `json:"client"` + Method string `json:"method"` + URL string `json:"url"` +} + +type ds02EgressRecorder struct { + mu sync.Mutex + phase string + entries []ds02Egress +} + +func (r *ds02EgressRecorder) setPhase(phase string) { + r.mu.Lock() + r.phase = phase + r.mu.Unlock() +} + +func (r *ds02EgressRecorder) wrap(client string, base http.RoundTripper) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + return roundTripFunc(func(request *http.Request) (*http.Response, error) { + r.mu.Lock() + r.entries = append(r.entries, ds02Egress{Phase: r.phase, Client: client, Method: request.Method, URL: request.URL.String()}) + r.mu.Unlock() return base.RoundTrip(request) }) - t.Cleanup(func() { fx.client.Transport = nil }) - return seen +} + +func (r *ds02EgressRecorder) entriesFor(phase string) []ds02Egress { + r.mu.Lock() + defer r.mu.Unlock() + var out []ds02Egress + for _, entry := range r.entries { + if entry.Phase == phase { + out = append(out, entry) + } + } + return out +} + +// ds02IsEmbeddingTarget 按目的地址识别向量化调用,而不是只看品牌字符串: +// 任何路径里的 embed/embeddings/encode 段都算;已知模型服务域名也算。 +func ds02IsEmbeddingTarget(raw string) bool { + parsed, err := url.Parse(raw) + if err != nil { + return false + } + path := strings.ToLower(parsed.Path) + for _, marker := range []string{"embedding", "embeddings", "/embed", "/encode", "embeddings/create"} { + if strings.Contains(path, marker) { + return true + } + } + host := strings.ToLower(parsed.Hostname()) + for _, brand := range []string{"dashscope", "openai", "minimax", "siliconflow", "deepseek", "volces", "aliyuncs"} { + if strings.Contains(host, brand) { + return true + } + } + return false +} + +// ds02EgressDiffs 是纯函数:给定某一段的请求与声明的 client 清单,返回差异。 +func ds02EgressDiffs(see []ds02Egress, inventory []string) []string { + var diffs []string + seenClients := map[string]bool{} + for _, entry := range see { + seenClients[entry.Client] = true + if ds02IsEmbeddingTarget(entry.URL) { + diffs = append(diffs, fmt.Sprintf("embedding calls: want 0 got %s %s (%s)", entry.Method, entry.URL, entry.Client)) + } + } + for _, client := range inventory { + if !seenClients[client] { + diffs = append(diffs, "declared egress client never observed in publish: "+client) + } + } + for client := range seenClients { + known := false + for _, declared := range inventory { + if client == declared { + known = true + } + } + if !known { + diffs = append(diffs, "undeclared egress client observed in publish: "+client) + } + } + return diffs } type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } +// ds02RecordingHTTPClient 适配 AWS SDK 的 HTTPClient 接口(只有 Do)。 +type ds02RecordingHTTPClient struct { + recorder *ds02EgressRecorder + name string + next awss3.HTTPClient +} + +func (c *ds02RecordingHTTPClient) Do(request *http.Request) (*http.Response, error) { + c.recorder.mu.Lock() + c.recorder.entries = append(c.recorder.entries, ds02Egress{Phase: c.recorder.phase, Client: c.name, Method: request.Method, URL: request.URL.String()}) + c.recorder.mu.Unlock() + return c.next.Do(request) +} + +// ds02AttachEgress 把本测试全部可达 HTTP 构造器接到同一个记录器上,并返回可观测的 +// client 清单:ES(产品搜索/备份/独立读回共用 fx.client)与对象存储 SDK 客户端。 +// 正式装配里的 object gateway client 不在本测试路径上(本测试直接注入 S3 存储)。 +func ds02AttachEgress(t *testing.T, fx *writeSetFixture) (*ds02EgressRecorder, []string) { + t.Helper() + recorder := &ds02EgressRecorder{phase: "fixture"} + fx.client.Transport = recorder.wrap("elastic", http.DefaultTransport) + t.Cleanup(func() { + fx.client.Transport = nil + }) + options := fx.objects.s3.client.Options() + next := options.HTTPClient + if next == nil { + next = http.DefaultClient + } + options.HTTPClient = &ds02RecordingHTTPClient{recorder: recorder, name: "object-store", next: next} + fx.objects.s3.client = awss3.New(options) + return recorder, []string{"elastic", "object-store"} +} + // ds02Check 是 oracle 本体:返回逐项差异;空 = 全绿。 func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[string][]map[string]any, - paths map[string]string, preserved map[string][]map[string]any, requests *[]string) []string { + paths map[string]string, preserved map[string][]map[string]any, + recorder *ds02EgressRecorder, inventory []string) []string { t.Helper() var diffs []string scope := func(table string) (string, []any) { - if table == "knowledge_tag_relations" { - return "SELECT to_jsonb(x)::text FROM " + table + " AS x", nil + rule, ok := ds02ScopeDecl[table] + if !ok { + t.Fatalf("table %s missing from independent scope declaration", table) } - if tableHasTenant(table) && tablesWithKB[table] { + if !rule.Tenant { + return "SELECT to_jsonb(x)::text FROM " + table + " AS x ORDER BY 1", nil + } + if rule.KnowledgeBase { return "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY 1", []any{wsTargetTenant, wsTargetKB} } @@ -530,39 +809,68 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st diffs = append(diffs, diff) } // 目标范围之外的行必须一字不动(含其它知识库/租户以及它们的历史资源)。 - if table == "knowledge_tag_relations" { + if !ds02ScopeDecl[table].Tenant { continue } other := "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id<>$1" - if tableHasTenant(table) && tablesWithKB[table] { + if ds02ScopeDecl[table].KnowledgeBase { other = "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)" } if diff := ds02Compare(table+"(non-target)", preserved[table], ds02Read(t, fx, other, args...)); diff != "" { diffs = append(diffs, diff) } } - // 引用:每条关系都必须有真实目标行,而且断言确实覆盖到引用(匹配数 > 0)。 - relations := []struct { + // 引用清单:逐种关系分别断言 violations=0 **且** matched>0。范围外关系显式列出, + // 不把"七组 OR 查询"叫"全部引用"。 + type relation struct { name string sql string matched string - }{ + covered bool + reason string + } + relations := []relation{ {"chunk→knowledge", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`, - `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`}, - {"chunk→chunk links", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND ((c.pre_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id)) OR (c.next_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id)) OR (c.parent_chunk_id IS NOT NULL AND c.parent_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id)))`, - `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND (c.pre_chunk_id<>'' OR c.next_chunk_id<>'' OR coalesce(c.parent_chunk_id,'')<>'')`}, - {"relation→tag/knowledge", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) OR NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`, - `SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`}, - {"binding→resource/knowledge", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND (NOT EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id) OR (b.owner_type='knowledge' AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)))`, - `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`}, + `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`, true, ""}, + {"chunk.pre_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.pre_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id AND p.tenant_id=c.tenant_id AND p.knowledge_base_id=c.knowledge_base_id)`, + `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.pre_chunk_id<>'' AND EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id AND p.tenant_id=c.tenant_id AND p.knowledge_base_id=c.knowledge_base_id)`, true, ""}, + {"chunk.next_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.next_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id AND n.tenant_id=c.tenant_id AND n.knowledge_base_id=c.knowledge_base_id)`, + `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.next_chunk_id<>'' AND EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id AND n.tenant_id=c.tenant_id AND n.knowledge_base_id=c.knowledge_base_id)`, true, ""}, + {"chunk.parent_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND coalesce(c.parent_chunk_id,'')<>'' AND NOT EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id AND q.tenant_id=c.tenant_id AND q.knowledge_base_id=c.knowledge_base_id)`, + `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND coalesce(c.parent_chunk_id,'')<>'' AND EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id AND q.tenant_id=c.tenant_id AND q.knowledge_base_id=c.knowledge_base_id)`, true, ""}, + {"relation→tag", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id)`, + `SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id)`, true, ""}, + {"relation→knowledge", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`, + `SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`, true, ""}, + {"binding→resource", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND NOT EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`, + `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`, true, ""}, + {"binding→knowledge", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND b.owner_type='knowledge' AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)`, + `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND b.owner_type='knowledge' AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)`, true, ""}, {"wiki page→folder", `SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>'' AND NOT EXISTS(SELECT 1 FROM wiki_folders f WHERE f.id=p.folder_id)`, - `SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>''`}, + `SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>''`, true, ""}, + {"wiki folder→parent folder", `SELECT count(*) FROM wiki_folders f WHERE f.tenant_id=$1 AND f.knowledge_base_id=$2 AND f.parent_id<>'' AND NOT EXISTS(SELECT 1 FROM wiki_folders p WHERE p.id=f.parent_id AND p.tenant_id=f.tenant_id AND p.knowledge_base_id=f.knowledge_base_id)`, + `SELECT count(*) FROM wiki_folders f WHERE f.tenant_id=$1 AND f.knowledge_base_id=$2 AND f.parent_id<>''`, true, ""}, + {"wiki page.source_refs→knowledge", `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.source_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=split_part(ref,'|',1) AND k.tenant_id=p.tenant_id AND k.knowledge_base_id=p.knowledge_base_id)`, + `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.source_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=split_part(ref,'|',1) AND k.tenant_id=p.tenant_id AND k.knowledge_base_id=p.knowledge_base_id)`, true, ""}, + {"wiki page.chunk_refs→chunk", `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.chunk_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM chunks c WHERE c.id=ref AND c.tenant_id=p.tenant_id AND c.knowledge_base_id=p.knowledge_base_id)`, + `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.chunk_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM chunks c WHERE c.id=ref AND c.tenant_id=p.tenant_id AND c.knowledge_base_id=p.knowledge_base_id)`, true, ""}, + {"wiki issue.suspected_knowledge_ids→knowledge", `SELECT count(*) FROM wiki_page_issues i, jsonb_array_elements_text(coalesce(i.suspected_knowledge_ids,'[]'::jsonb)) ref WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=ref AND k.tenant_id=i.tenant_id AND k.knowledge_base_id=i.knowledge_base_id)`, + `SELECT count(*) FROM wiki_page_issues i, jsonb_array_elements_text(coalesce(i.suspected_knowledge_ids,'[]'::jsonb)) ref WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=ref AND k.tenant_id=i.tenant_id AND k.knowledge_base_id=i.knowledge_base_id)`, true, ""}, {"wiki issue→page", `SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=i.slug AND p.tenant_id=i.tenant_id AND p.knowledge_base_id=i.knowledge_base_id)`, - `SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2`}, - {"wiki log→knowledge/page", `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND (NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=l.knowledge_id) OR EXISTS(SELECT 1 FROM jsonb_array_elements(l.pages_affected) e WHERE NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)))`, - `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2`}, + `SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2`, true, ""}, + {"wiki log→knowledge", `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=l.knowledge_id)`, + `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2`, true, ""}, + {"wiki log.pages_affected→page", `SELECT count(*) FROM wiki_log_entries l, jsonb_array_elements(coalesce(l.pages_affected,'[]'::jsonb)) e WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)`, + `SELECT count(*) FROM wiki_log_entries l, jsonb_array_elements(coalesce(l.pages_affected,'[]'::jsonb)) e WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)`, true, ""}, + // 范围外(显式声明,不静默跳过):软导航字段没有外键契约,允许悬空。 + {"wiki page.parent_slug", "", "", false, "soft slug navigation, no FK; fixture keeps it empty"}, + {"wiki page.in_links/out_links", "", "", false, "soft slug navigation, no FK; dangling by design"}, + {"chunk.relation_chunks/indirect_relation_chunks", "", "", false, "jsonb relation hints, no FK contract"}, } for _, relation := range relations { + if !relation.covered { + continue + } var violations, matched int64 var args []any if strings.Contains(relation.sql, "$2") { @@ -583,6 +891,12 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st diffs = append(diffs, fmt.Sprintf("reference %s: oracle matched no rows", relation.name)) } } + inventoryRaw := make([]map[string]any, 0, len(relations)) + for _, item := range relations { + inventoryRaw = append(inventoryRaw, map[string]any{"name": item.name, "covered": item.covered, + "out_of_scope_reason": item.reason}) + } + r10Save(t, "reference-inventory", inventoryRaw) // 向量/正文:索引文档逐条与声明完全一致(多一条少一条都算差异)。 docs := ds02Docs(t, fx) if len(docs) != len(spec.rows["chunks"])+1 { // +1:目标范围之外的 chunk-foreign @@ -605,10 +919,14 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st } } } + // 外围索引文档:完整 _source 逐字段比,不只比 content。 + foreignWant := map[string]any{"tenant_id": json.Number("20000"), "knowledge_base_id": "kb-foreign", + "knowledge_id": "doc-foreign", "chunk_id": "chunk-foreign", "source_id": "chunk-foreign", + "content": "外部正文", "embedding": []any{0.1, 0.2}} if foreign, ok := docs["chunk-foreign"]; !ok { diffs = append(diffs, "foreign index doc disappeared") - } else if fmt.Sprint(foreign["content"]) != "外部正文" { - diffs = append(diffs, "foreign index doc changed") + } else if !ds02SameJSON(foreign, foreignWant) { + diffs = append(diffs, fmt.Sprintf("foreign index doc _source changed: want %s got %s", ds02RowKey(foreignWant), ds02RowKey(foreign))) } // 对象字节:按独立重算的定位符读回,逐字节比。 for id, want := range spec.objects { @@ -627,6 +945,20 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st diffs = append(diffs, fmt.Sprintf("object %s bytes: want %q got %q", id, want, got)) } } + // 外围对象:res-other 的字节也必须逐字节比(旧 oracle 只读 spec.objects)。 + otherPath := wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other") + reader, err := fx.objects.s3.Open(fx.ctx, "minio", otherPath) + if err != nil { + diffs = append(diffs, "sibling object res-other unreadable: "+err.Error()) + } else { + got, readErr := io.ReadAll(reader) + reader.Close() + if readErr != nil { + diffs = append(diffs, "sibling object res-other read: "+readErr.Error()) + } else if string(got) != strings.Repeat("z", 8) { + diffs = append(diffs, fmt.Sprintf("sibling object res-other bytes: want %q got %q", strings.Repeat("z", 8), got)) + } + } // Wiki 日志映射:内容元组多重集双射(本地号不参与身份,来源 id 只用于覆盖检查)。 wantTuples := []string{} for _, row := range spec.rows["wiki_log_entries"] { @@ -643,9 +975,6 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st wantTuples = append(wantTuples, string(raw)) } gotTuples := []string{} - for _, row := range expected["wiki_log_entries"] { - _ = row - } actualLogs := ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM wiki_log_entries AS x WHERE tenant_id=$1 AND knowledge_base_id=$2`, wsTargetTenant, wsTargetKB) for _, row := range actualLogs { copied := map[string]any{} @@ -663,29 +992,28 @@ func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[st if fmt.Sprint(wantTuples) != fmt.Sprint(gotTuples) { diffs = append(diffs, fmt.Sprintf("wiki log tuple bijection broken: want %v got %v", wantTuples, gotTuples)) } - // Embedding 调用必须为 0,且请求确实发生过(不是"一个请求都没有"的空绿)。 - embedding := 0 - for _, call := range *requests { - if strings.Contains(call, "embedding") || strings.Contains(call, "dashscope") || strings.Contains(call, "openai") { - embedding++ - } - } - if embedding != 0 { - diffs = append(diffs, fmt.Sprintf("embedding calls: want 0 got %d (%v)", embedding, *requests)) - } - if len(*requests) == 0 { - diffs = append(diffs, "no HTTP requests observed; zero-embedding oracle would be vacuous") + // Embedding:只对"发布段"记账;oracle 读段的请求不混进来。发布段必须有请求 + // (不是空绿),且 client 集合必须正是本测试声明的可达集合。 + publishEgress := recorder.entriesFor("publish") + diffs = append(diffs, ds02EgressDiffs(publishEgress, inventory)...) + if len(publishEgress) == 0 { + diffs = append(diffs, "no HTTP requests observed in publish phase; zero-embedding oracle would be vacuous") } return diffs } +func ds02SameJSON(left, right map[string]any) bool { + return ds02RowKey(left) == ds02RowKey(right) +} + // ds02OtherRows 读目标范围之外的行(发布前抓一次,发布后要求一字不差)。 func ds02OtherRows(t *testing.T, fx *writeSetFixture, table string) []map[string]any { t.Helper() - if table == "knowledge_tag_relations" { + rule := ds02ScopeDecl[table] + if !rule.Tenant { return nil } - if tableHasTenant(table) && tablesWithKB[table] { + if rule.KnowledgeBase { return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)", wsTargetTenant, wsTargetKB) } return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id<>$1", wsTargetTenant) @@ -696,6 +1024,65 @@ func ds02SameTenantRows(t *testing.T, fx *writeSetFixture, table string) []map[s return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id=$1", wsTargetTenant) } +// ds02SequenceSummary 记三条本地号序列的 (last_value,is_called):序列不存在直接失败, +// 不允许"空集合被当成不变"。 +func ds02SequenceSummary(t *testing.T, fx *writeSetFixture) map[string][]any { + t.Helper() + summary := map[string][]any{} + for _, name := range []string{"chunks_seq_id_seq", "knowledge_tags_seq_id_seq", "wiki_log_entries_id_seq"} { + var exists bool + if err := fx.pool.QueryRow(fx.ctx, "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("local sequence %s does not exist; sequence coverage would be empty", name) + } + var last int64 + var called bool + if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+name).Scan(&last, &called); err != nil { + t.Fatal(err) + } + summary[name] = []any{last, called} + } + return summary +} + +// ds02LocalColumns 把"声明来源 + 独立期望目标行 + 真实目标行"组成映射 oracle 的输入。 +func ds02LocalColumns(t *testing.T, fx *writeSetFixture, spec ds02Spec, expectation ds02Expectation) []ds02LocalColumn { + t.Helper() + var columns []ds02LocalColumn + for _, contract := range []struct{ table, column string }{ + {"chunks", "seq_id"}, {"knowledge_tags", "seq_id"}, {"wiki_log_entries", "id"}, + } { + sources := []int64{} + for _, row := range spec.rows[contract.table] { + raw := row["seq_id"] + if contract.table == "wiki_log_entries" { + raw = row["id"] + } + value, err := strconv.ParseInt(fmt.Sprint(raw), 10, 64) + if err != nil { + t.Fatalf("%s declared local id: %v", contract.table, err) + } + sources = append(sources, value) + } + sort.Slice(sources, func(i, j int) bool { return sources[i] < sources[j] }) + actual := map[int64]map[string]any{} + for _, row := range ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+contract.table+ + " AS x WHERE tenant_id=$1 AND knowledge_base_id=$2", wsTargetTenant, wsTargetKB) { + value, err := strconv.ParseInt(fmt.Sprint(row[contract.column]), 10, 64) + if err != nil { + t.Fatalf("%s actual local id: %v", contract.table, err) + } + actual[value] = row + } + columns = append(columns, ds02LocalColumn{Table: contract.table, Column: contract.column, + Sources: sources, Expected: expectation.Local[contract.table], + ExpectedRows: expectation.RowsBySource[contract.table], Actual: actual}) + } + return columns +} + func TestDS02FullShapeOracleIsolated(t *testing.T) { fx := openWriteSetFixture(t) // 用保留下来的真实 v071 catalog 重建十张表:逐列、外键、序列都是真实形状。 @@ -707,13 +1094,15 @@ func TestDS02FullShapeOracleIsolated(t *testing.T) { } { r10Exec(t, fx, sql) } + ds02VerifyScopeDecl(t, fx) ds02Seed(t, fx) preserved := map[string][]map[string]any{} for _, table := range ds02TargetTables { preserved[table] = ds02OtherRows(t, fx, table) } spec := ds02SourceSpec(t, fx) - expected, paths := ds02Expected(t, fx, spec) + expectation := ds02Expected(t, fx, spec) + expected, paths := expectation.Rows, expectation.Paths for _, table := range []string{"resources", "resource_bindings"} { expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...) } @@ -723,16 +1112,21 @@ func TestDS02FullShapeOracleIsolated(t *testing.T) { ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`, wsTargetTenant, wsTargetKB)...) + sequencesBefore := ds02SequenceSummary(t, fx) snapshot := ds02Seal(t, fx, spec) - requests := ds02CountingClient(t, fx) + recorder, inventory := ds02AttachEgress(t, fx) target := fx.target(t) backups := fx.backups(t, target) + recorder.setPhase("publish") out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil) + recorder.setPhase("oracle-read") r10Save(t, "outcome", out) if out.State != "succeeded" { t.Fatalf("full-shape publish must succeed: %+v", out) } - // 写集合里的本地号映射必须完整、双射,并且目标行真的用了这些号。 + sequencesAfter := ds02SequenceSummary(t, fx) + r10Save(t, "sequences", map[string]any{"before": sequencesBefore, "after": sequencesAfter}) + // 写集合里的本地号映射:独立语义 checker(不是"数个数/看双射"的循环)。 if out.Backup.WriteSet == nil { t.Fatal("write set missing") } @@ -740,42 +1134,154 @@ func TestDS02FullShapeOracleIsolated(t *testing.T) { if err != nil { t.Fatalf("load write set: %v", err) } - for _, table := range []string{"chunks", "knowledge_tags", "wiki_log_entries"} { - column := "seq_id" - if table == "wiki_log_entries" { - column = "id" + columns := ds02LocalColumns(t, fx, spec, expectation) + mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs) + r10Save(t, "mapping-diffs", mappingDiffs) + if len(mappingDiffs) != 0 { + t.Fatalf("persisted ledger fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n")) + } + // 语义副本变异:只改内存里的账本副本,目标存储与归档字节都不动。这个 RED + // 才是 mapping RED;改日志 action/正文只能证明内容 oracle 会红。 + for _, scenario := range []struct { + name string + build func([]libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping + want string + }{ + {"swapped-targets", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02FlipMappingTargets(ledger, "wiki_log_entries") + }, "wrong target pair"}, + {"nonexistent-target", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ledger, "wiki_log_entries", "target", "999") + }, "not an independently predicted target"}, + {"replaced-source", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ledger, "wiki_log_entries", "source", "999") + }, "unexpected ledger source"}, + {"missing-entry", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02DropMapping(ledger, "chunks") + }, "missing ledger source"}, + {"duplicate-entry", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02DuplicateMapping(ledger, "knowledge_tags") + }, "duplicate ledger source"}, + {"wrong-column", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping { + return ds02ReplaceMapping(ledger, "chunks", "column", "id") + }, "ledger column"}, + } { + red := ds02CheckLocalIDMapping(columns, scenario.build(ledger.LocalIDs)) + r10Save(t, "mapping-red-"+scenario.name, red) + joined := strings.Join(red, "\n") + if len(red) == 0 { + t.Fatalf("%s: semantic mapping mutation stayed green", scenario.name) } - sources := map[int64]bool{} - targets := map[int64]bool{} - for _, mapping := range ledger.LocalIDs { - if mapping.Table != table { - continue - } - if mapping.Column != column { - t.Fatalf("%s mapping column = %s", table, mapping.Column) - } - source, _ := strconv.ParseInt(mapping.Source, 10, 64) - targetID, _ := strconv.ParseInt(mapping.Target, 10, 64) - if sources[source] || targets[targetID] { - t.Fatalf("%s mapping not bijective: source=%d target=%d", table, source, targetID) - } - sources[source] = true - targets[targetID] = true - } - if len(sources) != len(spec.rows[table]) { - t.Fatalf("%s mapping covers %d source rows, want %d", table, len(sources), len(spec.rows[table])) + if !strings.Contains(joined, scenario.want) { + t.Fatalf("%s: mapping oracle failed for an unrelated reason:\n%s", scenario.name, joined) } } - diffs := ds02Check(t, fx, spec, expected, paths, preserved, requests) + // 归档字节变异 = checksum gate,不是 mapping RED:它只能证明摘要拒收, + // 不能替代上面那组语义 RED。 + ledgerPath := writeSetPath(fx.archiveRoot, out.Backup.WriteSet.Digest) + info, err := os.Stat(ledgerPath) + if err != nil { + t.Fatalf("stat persisted ledger: %v", err) + } + original, err := os.ReadFile(ledgerPath) + if err != nil { + t.Fatalf("read persisted ledger: %v", err) + } + // 台账落盘是只读的,变异前显式放开权限,结束后恢复原权限与原始字节。 + if err := os.Chmod(ledgerPath, 0600); err != nil { + t.Fatalf("unlock persisted ledger: %v", err) + } + // 只追加空白不会改变解析后的内容摘要(实现按内容摘要校验,不按字节); + // 这里改一个真实字段值,让内容摘要必然不符。 + var tampered map[string]any + if err := json.Unmarshal(original, &tampered); err != nil { + t.Fatalf("parse persisted ledger: %v", err) + } + if mappings, ok := tampered["local_ids"].([]any); ok && len(mappings) > 0 { + if first, ok := mappings[0].(map[string]any); ok { + first["target"] = "999999999" + } + } + reencoded, err := json.Marshal(tampered) + if err != nil { + t.Fatalf("encode tampered ledger: %v", err) + } + if err := os.WriteFile(ledgerPath, reencoded, 0600); err != nil { + t.Fatalf("tamper persisted ledger: %v", err) + } + _, digestErr := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest) + r10Save(t, "digest-gate", map[string]any{"error": fmt.Sprint(digestErr)}) + if digestErr == nil { + t.Fatal("tampered ledger bytes were accepted; digest gate is missing") + } + if err := os.WriteFile(ledgerPath, original, 0600); err != nil { + t.Fatalf("restore persisted ledger: %v", err) + } + if err := os.Chmod(ledgerPath, info.Mode().Perm()); err != nil { + t.Fatalf("restore ledger mode: %v", err) + } + if _, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest); err != nil { + t.Fatalf("restored ledger must load again: %v", err) + } + diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory) r10Save(t, "green-diffs", diffs) if len(diffs) != 0 { t.Fatalf("independent oracle disagrees on a correct publish:\n%s", strings.Join(diffs, "\n")) } - r10Save(t, "green-requests", *requests) + r10Save(t, "green-egress", map[string]any{"publish": recorder.entriesFor("publish"), + "oracle": recorder.entriesFor("oracle-read"), "inventory": inventory}) - // 变异自检:对真实目标改正文 / 向量 / 引用 / 日志映射,oracle 必须红; - // 改回后必须绿。这样每条断言都证明了自己能抓到它负责的东西。 + // 负控制:另一个 client 打到自定义 /v1/encode 路径必须被识别为 embedding, + // 而真正的发布段仍是 0。这证明分类器不靠品牌字符串。 + encoder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + defer encoder.Close() + custom := &http.Client{Transport: recorder.wrap("custom-encoder", http.DefaultTransport)} + recorder.setPhase("negative-control") + request, err := http.NewRequestWithContext(fx.ctx, "POST", encoder.URL+"/v1/encode", strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + response, err := custom.Do(request) + if err != nil { + t.Fatalf("negative control request: %v", err) + } + response.Body.Close() + negative := recorder.entriesFor("negative-control") + if len(ds02EgressDiffs(append(append([]ds02Egress{}, recorder.entriesFor("publish")...), negative...), inventory)) == 0 { + t.Fatal("custom /v1/encode path was not classified as an embedding call") + } + if len(ds02EgressDiffs(recorder.entriesFor("publish"), inventory)) != 0 { + t.Fatalf("publish phase embedding expectation broken: %v", ds02EgressDiffs(recorder.entriesFor("publish"), inventory)) + } + recorder.setPhase("oracle-read") + + // 变异自检:对真实目标改正文 / 向量 / 引用 / 日志内容 / 外围 ES 文档 / 外围对象, + // oracle 必须红;改回后必须绿。 content := fmt.Sprint(expected["chunks"][0]["content"]) + putDoc := func(id, body string) { + request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/"+id+"?refresh=true", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response, err := fx.client.Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode >= 300 { + t.Fatalf("mutate doc %s: %d", id, response.StatusCode) + } + } + putObject := func(path, body string) { + if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil { + t.Fatal(err) + } + } + foreignPath := wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other") + foreignDoc := `{"tenant_id":20000,"knowledge_base_id":"kb-foreign","knowledge_id":"doc-foreign","chunk_id":"chunk-foreign","source_id":"chunk-foreign","content":"外部正文","embedding":[0.1,0.2]}` + foreignDocTampered := `{"tenant_id":20000,"knowledge_base_id":"kb-foreign","knowledge_id":"doc-foreign","chunk_id":"chunk-foreign","source_id":"chunk-foreign","content":"外部正文篡改","embedding":[0.1,0.2]}` + chunkDoc := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[0.5,1.5]}` + chunkDocTampered := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[9.5,9.5]}` mutations := []struct { name string marker string @@ -790,44 +1296,35 @@ func TestDS02FullShapeOracleIsolated(t *testing.T) { } }}, {"vector", "index doc chunk-src.embedding", func() { - body := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[9.5,9.5]}` - request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/chunk-src?refresh=true", strings.NewReader(body)) - request.Header.Set("Content-Type", "application/json") - response, err := fx.client.Do(request) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode >= 300 { - t.Fatalf("mutate vector: %d", response.StatusCode) - } + putDoc("chunk-src", chunkDocTampered) }, func() { - body := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[0.5,1.5]}` - request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/chunk-src?refresh=true", strings.NewReader(body)) - request.Header.Set("Content-Type", "application/json") - response, err := fx.client.Do(request) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode >= 300 { - t.Fatalf("restore vector: %d", response.StatusCode) - } + putDoc("chunk-src", chunkDoc) }}, {"reference", "reference relation", func() { r10Exec(t, fx, `UPDATE knowledge_tag_relations SET tag_id='missing-tag' WHERE knowledge_id='doc-src' AND tag_id='tag-src'`) }, func() { r10Exec(t, fx, `UPDATE knowledge_tag_relations SET tag_id='tag-src' WHERE knowledge_id='doc-src' AND tag_id='missing-tag'`) }}, - {"mapping", "wiki log", func() { + // 这是日志内容变异,不是本地号映射变异:映射 RED 只在账本语义副本上做。 + {"log-content", "wiki log", func() { r10Exec(t, fx, `UPDATE wiki_log_entries SET action='tampered' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id='doc-src'`) }, func() { r10Exec(t, fx, `UPDATE wiki_log_entries SET action='ingest' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id='doc-src'`) }}, + {"foreign-index-source", "foreign index doc _source changed", func() { + putDoc("chunk-foreign", foreignDocTampered) + }, func() { + putDoc("chunk-foreign", foreignDoc) + }}, + {"foreign-object-bytes", "res-other bytes", func() { + putObject(foreignPath, strings.Repeat("z", 7)+"!") + }, func() { + putObject(foreignPath, strings.Repeat("z", 8)) + }}, } for _, mutation := range mutations { mutation.mutate() - red := ds02Check(t, fx, spec, expected, paths, preserved, requests) + red := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory) r10Save(t, "red-"+mutation.name, red) if len(red) == 0 { t.Fatalf("%s mutation: oracle stayed green", mutation.name) @@ -837,7 +1334,7 @@ func TestDS02FullShapeOracleIsolated(t *testing.T) { t.Fatalf("%s mutation: oracle failed for an unrelated reason:\n%s", mutation.name, joined) } mutation.restore() - green := ds02Check(t, fx, spec, expected, paths, preserved, requests) + green := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory) if len(green) != 0 { t.Fatalf("%s mutation not restored cleanly:\n%s", mutation.name, strings.Join(green, "\n")) } diff --git a/internal/releaseadapter/v071/library_writeset_integration_test.go b/internal/releaseadapter/v071/library_writeset_integration_test.go index 8cdeeb2..05f4636 100644 --- a/internal/releaseadapter/v071/library_writeset_integration_test.go +++ b/internal/releaseadapter/v071/library_writeset_integration_test.go @@ -117,6 +117,8 @@ type writeSetFixtureOptions struct { reset bool archiveRoot string index string + // ddl 非空时替换最小夹具 schema:DS-02 冻结 bundle 用同一份 DDL 初始化两个隔离目标。 + ddl []string } func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture { @@ -159,6 +161,15 @@ func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writ t.Fatalf("reset isolated schema: %v", err) } } + // 冻结 bundle 自带完整 DDL 时,不叠加最小夹具(否则两个目标的 schema 又不同)。 + if len(options.ddl) > 0 { + for _, statement := range options.ddl { + statement = strings.ReplaceAll(statement, "%BUCKET%", wsBucket) + if _, err := pool.Exec(ctx, statement); err != nil { + t.Fatalf("bundle schema: %v\n%s", err, statement) + } + } + } schemaStatements := []string{ libraryTestSchema, // 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据, @@ -177,7 +188,7 @@ func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writ `INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`, `INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`, } - if options.reset { + if options.reset && len(options.ddl) == 0 { for _, statement := range schemaStatements { if _, err := pool.Exec(ctx, statement); err != nil { t.Fatalf("fixture schema: %v", err) diff --git a/scripts/test-scope.json b/scripts/test-scope.json index 539ab38..7878b29 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -11,6 +11,8 @@ "optional": [ {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestDS02FullShapeOracleIsolated","reason":"需显式一次性本机PG/ES/MinIO:十表逐列/引用/向量/对象字节独立oracle、零Embedding计数与正文/向量/引用/映射变异自检;普通CI跳过不是通过"}, {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestDS02LegacyCompatibilityMatrixIsolated","reason":"需显式一次性本机PG/ES/MinIO:旧manifest/V1/空作业或目标身份的正式入口拒绝且全数据hash不变;普通CI跳过不是通过"}, + {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestDS02FrozenBundleIsolated","reason":"需 KG_DS02_BUNDLE 指向的冻结输入 bundle 加一次性本机PG/ES/MinIO:与旧SHA同一bundle hash的新侧发布与独立oracle;普通CI没有bundle,跳过不是通过"}, + {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestDS02ProcessIdentity","reason":"需隔离运行器注入 test-only nonce/commit/源码树与二进制哈希期望;普通CI没有注入,跳过不是通过"}, {"package":"knowledge-governance/cmd/worker","test":"TestR13AnchorWriteFailureBlocksTargetIsolated","reason":"需显式一次性本机治理PG:真实约束拒绝写锚点,验证目标零写入且不留假基线;普通CI跳过不是通过"}, {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOraclePropagatesDriverFailureIsolated","reason":"需显式一次性本机PG:用真实 statement timeout 让校验查询中途失败,断言不是\"无孤儿\";普通CI跳过不是通过"}, {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestR13OrphanOracleRejectsMalformedAndUnknownDocumentsIsolated","reason":"需显式一次性本机PG/ES/MinIO:孤儿校验的缺失/未知 knowledge_id 分支;普通CI跳过不是通过"},