0d9091e464
build / build (push) Successful in 1m41s
空 mode 计划不映射;索引 source_id 按语义映射;relation 按字段表解析。 隔离回归按持久映射 ID 断言,DS02 故障注入真实命中。
1383 lines
63 KiB
Go
1383 lines
63 KiB
Go
package v071
|
||
|
||
// DS-02:整库发布的独立 oracle。
|
||
//
|
||
// 这里不调用被测的映射/改写函数来生成期望值:期望由夹具自己声明的十张表逐列内容
|
||
// 推导(来源范围 → 目标范围的映射规则、资源定位符、location_hash 都在本文件里
|
||
// 用字面规则重算),本地号(chunks.seq_id / knowledge_tags.seq_id /
|
||
// wiki_log_entries.id)由测试**发布前**读到的序列状态加声明顺序算出来。
|
||
// 正文、引用、向量、对象字节、Wiki 日志映射分别有独立断言;断言本身再用
|
||
// "真实改写目标存储 → 必须红 → 改回 → 必须绿" 证明不是空跑。
|
||
|
||
import (
|
||
"bytes"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"net/url"
|
||
"os"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// ds02Vectors 是夹具声明的来源向量(逐分块不同)。索引部件由它生成,
|
||
// 不是由被测适配器生成。
|
||
var ds02Vectors = map[string][]float64{
|
||
"chunk-src": {0.5, 1.5},
|
||
"chunk-src2": {-2.25, 3.75},
|
||
"chunk-src3": {1.0, -1.0},
|
||
}
|
||
|
||
// ds02Objects 是夹具声明的资源字节(长度与 resources.size 一致)。
|
||
var ds02Objects = map[string]string{
|
||
"res-src": "abc12345",
|
||
"res-src2": "wxyz6789",
|
||
}
|
||
|
||
var ds02TargetTables = []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations",
|
||
"resources", "resource_bindings", "wiki_folders", "wiki_pages", "wiki_page_issues", "wiki_log_entries"}
|
||
|
||
func ds02Canon(t *testing.T, fx *writeSetFixture, table string, row map[string]any) map[string]any {
|
||
t.Helper()
|
||
raw, err := json.Marshal(row)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var canonical []byte
|
||
if err := fx.pool.QueryRow(fx.ctx,
|
||
"SELECT to_jsonb(jsonb_populate_record(NULL::"+table+",$1::jsonb))", raw).Scan(&canonical); err != nil {
|
||
t.Fatalf("canonicalize %s: %v", table, err)
|
||
}
|
||
dec := json.NewDecoder(bytes.NewReader(canonical))
|
||
dec.UseNumber()
|
||
var out map[string]any
|
||
if err := dec.Decode(&out); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func ds02Read(t *testing.T, fx *writeSetFixture, query string, args ...any) []map[string]any {
|
||
t.Helper()
|
||
rows, err := fx.pool.Query(fx.ctx, query, args...)
|
||
if err != nil {
|
||
t.Fatalf("oracle query: %v", err)
|
||
}
|
||
defer rows.Close()
|
||
var out []map[string]any
|
||
for rows.Next() {
|
||
var text string
|
||
if err := rows.Scan(&text); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
dec := json.NewDecoder(strings.NewReader(text))
|
||
dec.UseNumber()
|
||
var row map[string]any
|
||
if err := dec.Decode(&row); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
out = append(out, row)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func ds02Normalize(rows []map[string]any) []string {
|
||
out := make([]string, 0, len(rows))
|
||
for _, row := range rows {
|
||
raw, _ := json.Marshal(row)
|
||
out = append(out, string(raw))
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// ds02Compare 返回人可读的差异;空串表示逐列内容完全一致(含行数)。
|
||
func ds02Compare(table string, want, got []map[string]any) string {
|
||
w, g := ds02Normalize(want), ds02Normalize(got)
|
||
if len(w) == len(g) {
|
||
same := true
|
||
for i := range w {
|
||
if w[i] != g[i] {
|
||
same = false
|
||
break
|
||
}
|
||
}
|
||
if same {
|
||
return ""
|
||
}
|
||
}
|
||
seen := map[string]bool{}
|
||
for _, row := range g {
|
||
seen[row] = true
|
||
}
|
||
var missing, extra []string
|
||
for _, row := range w {
|
||
if !seen[row] {
|
||
missing = append(missing, row)
|
||
}
|
||
}
|
||
present := map[string]bool{}
|
||
for _, row := range w {
|
||
present[row] = true
|
||
}
|
||
for _, row := range g {
|
||
if !present[row] {
|
||
extra = append(extra, row)
|
||
}
|
||
}
|
||
if len(missing) > 6 {
|
||
missing = missing[:6]
|
||
}
|
||
if len(extra) > 6 {
|
||
extra = extra[:6]
|
||
}
|
||
return fmt.Sprintf("%s: want %d rows, got %d; missing=%v extra=%v", table, len(w), len(g), missing, extra)
|
||
}
|
||
|
||
// ds02SequenceNext 是本地号分配的独立期望:发布前序列状态 + 是否已调用。
|
||
func ds02SequenceNext(t *testing.T, fx *writeSetFixture, sequence string) int64 {
|
||
t.Helper()
|
||
var last int64
|
||
var called bool
|
||
if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+sequence).Scan(&last, &called); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if called {
|
||
return last + 1
|
||
}
|
||
return last
|
||
}
|
||
|
||
// ds02Spec 是夹具声明的来源内容(十表逐列 + 向量 + 对象字节)。
|
||
type ds02Spec struct {
|
||
rows map[string][]map[string]any
|
||
vectors map[string][]float64
|
||
objects map[string]string
|
||
}
|
||
|
||
// ds02SourceRows 是来源十表的纯声明(未按目标 schema 规范化);bundle 导出与
|
||
// 真跑夹具共用,保证"同一输入"来自同一份声明。
|
||
func ds02SourceRows() map[string][]map[string]any {
|
||
tag := func(id string, seq int64) map[string]any {
|
||
return map[string]any{"id": id, "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"name": "name-" + id, "color": "#112233", "sort_order": 3, "seq_id": seq,
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"}
|
||
}
|
||
sourcePath := func(id string) string {
|
||
return wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, id)
|
||
}
|
||
sourceHash := func(id string) string {
|
||
sum := sha256.Sum256([]byte(sourcePath(id)))
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
rows := map[string][]map[string]any{
|
||
"knowledges": {
|
||
{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "type": "document",
|
||
"title": "来源文档一", "description": "desc-1", "source": "resource://" + wsHandleSource,
|
||
"parse_status": "completed", "enable_status": "enabled", "embedding_model_id": "source-model",
|
||
"file_name": "one.md", "file_type": "md", "file_size": int64(8), "file_path": "source/one.md",
|
||
"file_hash": strings.Repeat("1", 64), "storage_size": int64(8), "metadata": map[string]any{"k": "v1"},
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z",
|
||
"processed_at": "2026-01-03T00:00:00Z", "summary_status": "none", "channel": "web",
|
||
"pending_subtasks_count": 0},
|
||
{"id": "doc-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "type": "document",
|
||
"title": "来源文档二", "description": "desc-2", "source": "upload",
|
||
"parse_status": "completed", "enable_status": "enabled", "embedding_model_id": "source-model",
|
||
"file_name": "two.md", "file_type": "md", "file_size": int64(4), "file_path": "source/two.md",
|
||
"file_hash": strings.Repeat("2", 64), "storage_size": int64(4), "metadata": map[string]any{"k": "v2"},
|
||
"created_at": "2026-01-04T00:00:00Z", "updated_at": "2026-01-05T00:00:00Z",
|
||
"summary_status": "none", "channel": "web", "pending_subtasks_count": 1},
|
||
},
|
||
"chunks": {
|
||
{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"knowledge_id": "doc-src", "content": "来源正文一 resource://" + wsHandleSource,
|
||
"chunk_index": 0, "is_enabled": true, "start_at": 0, "end_at": 12,
|
||
"next_chunk_id": "chunk-src2", "chunk_type": "text", "status": 1, "flags": 1,
|
||
"seq_id": int64(41), "content_hash": strings.Repeat("3", 64),
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
||
{"id": "chunk-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"knowledge_id": "doc-src", "content": "来源正文二", "chunk_index": 1, "is_enabled": true,
|
||
"start_at": 12, "end_at": 20, "pre_chunk_id": "chunk-src", "chunk_type": "text",
|
||
"status": 1, "flags": 1, "seq_id": int64(42), "content_hash": strings.Repeat("4", 64),
|
||
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
||
// 第三块专门给 parent_chunk_id 分支一个非空命中(旧夹具全为 null)。
|
||
{"id": "chunk-src3", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"knowledge_id": "doc-src", "content": "来源正文三", "chunk_index": 2, "is_enabled": true,
|
||
"start_at": 20, "end_at": 28, "parent_chunk_id": "chunk-src", "chunk_type": "text",
|
||
"status": 1, "flags": 1, "seq_id": int64(43), "content_hash": strings.Repeat("7", 64),
|
||
"created_at": "2026-01-05T00:00:00Z", "updated_at": "2026-01-06T00:00:00Z"},
|
||
},
|
||
"knowledge_tags": {tag("tag-src", 11), tag("tag-src2", 12)},
|
||
"knowledge_tag_relations": {
|
||
{"knowledge_id": "doc-src", "tag_id": "tag-src", "created_at": "2026-01-01T00:00:00Z"},
|
||
{"knowledge_id": "doc-src2", "tag_id": "tag-src2", "created_at": "2026-01-03T00:00:00Z"},
|
||
},
|
||
"resources": {
|
||
{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
|
||
"storage_backend_id": "backend-source", "provider": "oss", "physical_path": sourcePath("res-src"),
|
||
"location_hash": sourceHash("res-src"), "kind": "file", "mime_type": "image/png",
|
||
"original_name": "a.png", "size": int64(8), "content_hash": strings.Repeat("5", 64),
|
||
"lifecycle": "persistent", "state": "active",
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
||
{"id": "res-src2", "handle": strings.Repeat("g", 22), "tenant_id": wsSourceTenant,
|
||
"storage_backend_id": "backend-source", "provider": "oss", "physical_path": sourcePath("res-src2"),
|
||
"location_hash": sourceHash("res-src2"), "kind": "file", "mime_type": "image/png",
|
||
"original_name": "b.png", "size": int64(8), "content_hash": strings.Repeat("6", 64),
|
||
"lifecycle": "persistent", "state": "active",
|
||
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
||
},
|
||
"resource_bindings": {
|
||
{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
|
||
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment",
|
||
"created_at": "2026-01-01T00:00:00Z"},
|
||
{"id": "bind-src2", "resource_id": "res-src2", "tenant_id": wsSourceTenant,
|
||
"owner_type": "knowledge", "owner_id": "doc-src2", "relation": "attachment",
|
||
"created_at": "2026-01-03T00:00:00Z"},
|
||
},
|
||
"wiki_folders": {
|
||
{"id": "folder-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"parent_id": "", "name": "根目录", "path": "root", "depth": 0, "sort_order": 1,
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
||
{"id": "folder-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"parent_id": "folder-src", "name": "子目录", "path": "root/sub", "depth": 1, "sort_order": 2,
|
||
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
||
},
|
||
"wiki_pages": {
|
||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index",
|
||
"title": "首页", "page_type": "summary", "status": "published", "content": "来源 wiki 首页",
|
||
"summary": "摘要一", "parent_slug": "", "folder_id": "folder-src", "category_path": []any{"root"},
|
||
"wiki_path": "index", "depth": 0, "sort_order": 1,
|
||
"source_refs": []any{"doc-src|来源文档一"},
|
||
"chunk_refs": []any{"chunk-src"}, "in_links": []any{}, "out_links": []any{"guide"},
|
||
"page_metadata": map[string]any{"lang": "zh"}, "aliases": []any{"home"}, "version": 1,
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
||
{"id": "wiki-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "guide",
|
||
"title": "指南", "page_type": "summary", "status": "published", "content": "来源 wiki 指南",
|
||
"summary": "摘要二", "parent_slug": "", "folder_id": "folder-src2", "category_path": []any{"root", "sub"},
|
||
"wiki_path": "root/sub/guide", "depth": 1, "sort_order": 2,
|
||
"source_refs": []any{"doc-src2|来源文档二"},
|
||
"chunk_refs": []any{"chunk-src2"}, "in_links": []any{"guide"}, "out_links": []any{},
|
||
"page_metadata": map[string]any{"lang": "zh"}, "aliases": []any{}, "version": 1,
|
||
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
||
},
|
||
"wiki_page_issues": {
|
||
{"id": "issue-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index",
|
||
"issue_type": "review", "description": "需要复核", "suspected_knowledge_ids": []any{"doc-src"},
|
||
"status": "pending", "reported_by": "fixture",
|
||
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
||
},
|
||
"wiki_log_entries": {
|
||
{"id": int64(1), "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "action": "ingest",
|
||
"knowledge_id": "doc-src", "doc_title": "来源文档一", "summary": "首次入库",
|
||
"pages_affected": []any{map[string]any{"slug": "index", "title": "首页"}},
|
||
"created_at": "2026-01-01T00:00:00Z"},
|
||
{"id": int64(2), "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "action": "update",
|
||
"knowledge_id": "doc-src2", "doc_title": "来源文档二", "summary": "更新",
|
||
"pages_affected": []any{map[string]any{"slug": "guide", "title": "指南"}},
|
||
"created_at": "2026-01-03T00:00:00Z"},
|
||
},
|
||
}
|
||
return rows
|
||
}
|
||
|
||
func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec {
|
||
t.Helper()
|
||
rows := ds02SourceRows()
|
||
for table, list := range rows {
|
||
for i, row := range list {
|
||
rows[table][i] = ds02Canon(t, fx, table, row)
|
||
}
|
||
}
|
||
return ds02Spec{rows: rows, vectors: ds02Vectors, objects: ds02Objects}
|
||
}
|
||
|
||
// ds02Expectation 是独立推导的期望:逐表逐列的行、资源定位符、本地号分配,
|
||
// 以及"来源数字 id → 期望目标行"的对应(mapping oracle 用它把配对落到真实行)。
|
||
type ds02Expectation struct {
|
||
Rows map[string][]map[string]any
|
||
Paths map[string]string
|
||
Local map[string]map[int64]int64
|
||
RowsBySource map[string]map[int64]map[string]any
|
||
}
|
||
|
||
// ds02Expected 从声明内容独立推导目标范围应有的十表逐列内容。
|
||
func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) ds02Expectation {
|
||
t.Helper()
|
||
targetBackend, targetProvider := "backend-target", "minio"
|
||
locator := func(id string) string {
|
||
// 独立重算定位符:storage://<backend>/<provider>://<bucket>/<tenant>/<object>
|
||
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", targetBackend, targetProvider, fx.bucket, wsTargetTenant, id)
|
||
}
|
||
next := map[string]int64{
|
||
"chunks": ds02SequenceNext(t, fx, "chunks_seq_id_seq"),
|
||
"knowledge_tags": ds02SequenceNext(t, fx, "knowledge_tags_seq_id_seq"),
|
||
"wiki_log_entries": ds02SequenceNext(t, fx, "wiki_log_entries_id_seq"),
|
||
}
|
||
local := map[string]map[int64]int64{"chunks": {}, "knowledge_tags": {}, "wiki_log_entries": {}}
|
||
for _, table := range []string{"chunks", "knowledge_tags", "wiki_log_entries"} {
|
||
sources := []int64{}
|
||
for _, row := range spec.rows[table] {
|
||
raw := row["seq_id"]
|
||
if table == "wiki_log_entries" {
|
||
raw = row["id"]
|
||
}
|
||
value, err := strconv.ParseInt(fmt.Sprint(raw), 10, 64)
|
||
if err != nil {
|
||
t.Fatalf("%s local id: %v", table, err)
|
||
}
|
||
sources = append(sources, value)
|
||
}
|
||
sort.Slice(sources, func(i, j int) bool { return sources[i] < sources[j] })
|
||
value := next[table]
|
||
for _, source := range sources {
|
||
local[table][source] = value
|
||
value++
|
||
}
|
||
}
|
||
expected := map[string][]map[string]any{}
|
||
paths := map[string]string{}
|
||
rowsBySource := map[string]map[int64]map[string]any{}
|
||
for table, rows := range spec.rows {
|
||
for _, declared := range rows {
|
||
row := map[string]any{}
|
||
for key, value := range declared {
|
||
row[key] = value
|
||
}
|
||
rule := ds02ScopeDecl[table]
|
||
if rule.Tenant {
|
||
row["tenant_id"] = json.Number(strconv.FormatUint(wsTargetTenant, 10))
|
||
}
|
||
if rule.KnowledgeBase {
|
||
row["knowledge_base_id"] = wsTargetKB
|
||
}
|
||
var sourceKey int64
|
||
switch table {
|
||
case "chunks", "knowledge_tags":
|
||
sourceKey, _ = strconv.ParseInt(fmt.Sprint(row["seq_id"]), 10, 64)
|
||
row["seq_id"] = json.Number(strconv.FormatInt(local[table][sourceKey], 10))
|
||
case "wiki_log_entries":
|
||
sourceKey, _ = strconv.ParseInt(fmt.Sprint(row["id"]), 10, 64)
|
||
row["id"] = json.Number(strconv.FormatInt(local[table][sourceKey], 10))
|
||
case "resources":
|
||
id := fmt.Sprint(row["id"])
|
||
expectedPath := locator(id)
|
||
paths[id] = expectedPath
|
||
row["physical_path"] = expectedPath
|
||
sum := sha256.Sum256([]byte(expectedPath))
|
||
row["location_hash"] = hex.EncodeToString(sum[:])
|
||
row["storage_backend_id"] = targetBackend
|
||
row["provider"] = targetProvider
|
||
}
|
||
canonical := ds02Canon(t, fx, table, row)
|
||
expected[table] = append(expected[table], canonical)
|
||
if _, ok := local[table]; ok {
|
||
if rowsBySource[table] == nil {
|
||
rowsBySource[table] = map[int64]map[string]any{}
|
||
}
|
||
rowsBySource[table][sourceKey] = canonical
|
||
}
|
||
}
|
||
}
|
||
return ds02Expectation{Rows: expected, Paths: paths, Local: local, RowsBySource: rowsBySource}
|
||
}
|
||
|
||
// ds02BucketPlaceholder 让冻结 bundle 不绑定某个一次性桶名:两个隔离目标各自把
|
||
// 环境桶替换进来,输入内容(行、id、字节)本身逐字节相同。
|
||
const ds02BucketPlaceholder = "%BUCKET%"
|
||
|
||
// ds02SeedRows 是"发布前目标实例本来就有的数据"的声明。bundle 导出与夹具插入共用
|
||
// 这一份,避免两份行内容漂移。
|
||
func ds02SeedRows() []struct {
|
||
table string
|
||
row map[string]any
|
||
} {
|
||
return []struct {
|
||
table string
|
||
row map[string]any
|
||
}{
|
||
{"knowledges", map[string]any{"id": "doc-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "type": "document", "title": "别的知识库文档", "source": "x"}},
|
||
{"knowledges", map[string]any{"id": "doc-foreign", "tenant_id": 20000, "knowledge_base_id": "kb-foreign", "type": "document", "title": "外部租户文档", "source": "x"}},
|
||
{"chunks", map[string]any{"id": "chunk-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "knowledge_id": "doc-other", "content": "别的知识库正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(7)}},
|
||
{"knowledge_tags", map[string]any{"id": "tag-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库标签", "seq_id": int64(9)}},
|
||
{"knowledge_tag_relations", map[string]any{"knowledge_id": "doc-other", "tag_id": "tag-other"}},
|
||
{"resources", map[string]any{"id": "res-other", "handle": wsHandleOther, "tenant_id": 10001, "provider": "minio", "physical_path": wsLocator("backend-target", "minio", ds02BucketPlaceholder, 10001, "res-other"), "storage_backend_id": "backend-target", "size": int64(8)}},
|
||
{"resource_bindings", map[string]any{"id": "bind-other", "resource_id": "res-other", "tenant_id": 10001, "owner_type": "knowledge", "owner_id": "doc-other", "relation": "attachment"}},
|
||
{"wiki_folders", map[string]any{"id": "folder-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库目录"}},
|
||
{"wiki_pages", map[string]any{"id": "wiki-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "slug": "index", "content": "别的知识库 wiki"}},
|
||
{"wiki_page_issues", map[string]any{"id": "issue-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "slug": "index", "issue_type": "review", "reported_by": "fixture", "description": "x"}},
|
||
// 这条是 B03 的碰撞输入:来源日志数字 id=1 与其它知识库已有的 id=1 同值。
|
||
{"wiki_log_entries", map[string]any{"id": int64(1), "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "action": "ingest"}},
|
||
// 目标范围自己的数据:整库发布必须把它们替换成来源内容。
|
||
{"knowledges", map[string]any{"id": "doc-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "type": "document", "title": "目标旧文档", "source": "x", "parse_status": "completed", "pending_subtasks_count": 0}},
|
||
{"chunks", map[string]any{"id": "chunk-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "knowledge_id": "doc-target", "content": "目标旧正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(8)}},
|
||
{"wiki_pages", map[string]any{"id": "wiki-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "slug": "index", "content": "目标旧 wiki"}},
|
||
}
|
||
}
|
||
|
||
// ds02SeedDocs 是发布前就存在的索引文档(目标旧文档 + 外围租户文档)。
|
||
func ds02SeedDocs() []struct {
|
||
id string
|
||
tenant uint64
|
||
kb string
|
||
knowledge string
|
||
content string
|
||
} {
|
||
return []struct {
|
||
id string
|
||
tenant uint64
|
||
kb string
|
||
knowledge string
|
||
content string
|
||
}{
|
||
{"chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "目标旧正文"},
|
||
{"chunk-foreign", 20000, "kb-foreign", "doc-foreign", "外部正文"},
|
||
}
|
||
}
|
||
|
||
// ds02SeedObjects 是发布前就存在的对象字节(外围对象 res-other 用于哨兵)。
|
||
func ds02SeedObjects() map[string]string {
|
||
return map[string]string{"res-other": strings.Repeat("z", 8)}
|
||
}
|
||
|
||
func ds02SubstituteBucket(value string, bucket string) string {
|
||
return strings.ReplaceAll(value, ds02BucketPlaceholder, bucket)
|
||
}
|
||
|
||
func ds02Seed(t *testing.T, fx *writeSetFixture) {
|
||
t.Helper()
|
||
for id, body := range ds02SeedObjects() {
|
||
path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id)
|
||
if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
for _, row := range ds02SeedRows() {
|
||
r11Insert(t, fx, row.table, ds02SeedRow(row.row, fx.bucket))
|
||
}
|
||
for _, doc := range ds02SeedDocs() {
|
||
fx.indexDoc(t, doc.id, doc.tenant, doc.kb, doc.knowledge, doc.content)
|
||
}
|
||
}
|
||
|
||
// ds02SeedRow 返回替换了环境桶的副本,不改共享声明。
|
||
func ds02SeedRow(row map[string]any, bucket string) map[string]any {
|
||
out := map[string]any{}
|
||
for key, value := range row {
|
||
if text, ok := value.(string); ok {
|
||
out[key] = ds02SubstituteBucket(text, bucket)
|
||
} else {
|
||
out[key] = value
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ds02Seal 用声明内容封一份十表全非空快照(索引向量来自夹具声明)。
|
||
func ds02Seal(t *testing.T, fx *writeSetFixture, spec ds02Spec) string {
|
||
t.Helper()
|
||
return ds02SealScope(t, fx, spec, wsSourceTenant, wsSourceKB)
|
||
}
|
||
|
||
// ds02SealScope 允许指定来源范围:冻结 bundle 用"同租户不同知识库"的对照场景。
|
||
func ds02SealScope(t *testing.T, fx *writeSetFixture, spec ds02Spec, sourceTenant uint64, sourceKB string) string {
|
||
t.Helper()
|
||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer archive.Close()
|
||
encode := func(value any) []byte {
|
||
raw, err := json.Marshal(value)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return raw
|
||
}
|
||
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
||
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
||
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
||
parts := []librarysnapshot.Part{}
|
||
for _, table := range ds02TargetTables {
|
||
part, err := archive.Put(bytes.NewReader(encode(spec.rows[table])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = kinds[table], table, int64(len(spec.rows[table]))
|
||
parts = append(parts, part)
|
||
}
|
||
for _, id := range []string{"res-src", "res-src2"} {
|
||
part, err := archive.Put(bytes.NewReader([]byte(spec.objects[id])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = "resource", id, 1
|
||
parts = append(parts, part)
|
||
}
|
||
indexRows := []map[string]any{}
|
||
for _, chunk := range spec.rows["chunks"] {
|
||
id := fmt.Sprint(chunk["id"])
|
||
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
||
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||
"knowledge_id": chunk["knowledge_id"], "chunk_id": id, "source_id": id,
|
||
"content": chunk["content"], "embedding": spec.vectors[id]}})
|
||
}
|
||
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
||
parts = append(parts, indexPart)
|
||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
|
||
CreatedAt: time.Now().UTC(), Parts: parts,
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
||
id, err := archive.Seal(manifest)
|
||
if err != nil {
|
||
t.Fatalf("seal ds02 snapshot: %v", err)
|
||
}
|
||
return id
|
||
}
|
||
|
||
// ds02Docs 读回索引里的全部文档(独立查询,不经过被测 SearchExporter)。
|
||
func ds02Docs(t *testing.T, fx *writeSetFixture) map[string]map[string]any {
|
||
t.Helper()
|
||
body := `{"size":1000,"query":{"match_all":{}}}`
|
||
request, err := http.NewRequestWithContext(fx.ctx, "POST", fx.searchBase+"/"+fx.index+"/_search", strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := fx.client.Do(request)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer response.Body.Close()
|
||
if response.StatusCode != 200 {
|
||
raw, _ := io.ReadAll(response.Body)
|
||
t.Fatalf("search %d: %s", response.StatusCode, raw)
|
||
}
|
||
var payload struct {
|
||
Hits struct {
|
||
Hits []struct {
|
||
ID string `json:"_id"`
|
||
Source map[string]any `json:"_source"`
|
||
} `json:"hits"`
|
||
} `json:"hits"`
|
||
}
|
||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
out := map[string]map[string]any{}
|
||
for _, hit := range payload.Hits.Hits {
|
||
out[hit.ID] = hit.Source
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ds02ScopeRule 是本测试自己声明的表范围规则,用来独立推导目标行该按什么范围读。
|
||
// 刻意不读产品的 tableHasTenant/tablesWithKB:期望与查询共用产品的范围分类时,
|
||
// "两边一起错" 会被判绿。声明本身还要与真实 schema 的列集合对拍(ds02VerifyScopeDecl)。
|
||
type ds02ScopeRule struct {
|
||
Tenant bool
|
||
KnowledgeBase bool
|
||
}
|
||
|
||
var ds02ScopeDecl = map[string]ds02ScopeRule{
|
||
"knowledges": {Tenant: true, KnowledgeBase: true},
|
||
"chunks": {Tenant: true, KnowledgeBase: true},
|
||
"knowledge_tags": {Tenant: true, KnowledgeBase: true},
|
||
"knowledge_tag_relations": {Tenant: false, KnowledgeBase: false},
|
||
"resources": {Tenant: true, KnowledgeBase: false},
|
||
"resource_bindings": {Tenant: true, KnowledgeBase: false},
|
||
"wiki_folders": {Tenant: true, KnowledgeBase: true},
|
||
"wiki_pages": {Tenant: true, KnowledgeBase: true},
|
||
"wiki_page_issues": {Tenant: true, KnowledgeBase: true},
|
||
"wiki_log_entries": {Tenant: true, KnowledgeBase: true},
|
||
}
|
||
|
||
// ds02VerifyScopeDecl 把声明的表/列集合与真实 schema 对拍:表集合必须恰好是这十张,
|
||
// 每张表是否存在 tenant_id / knowledge_base_id 列必须与声明一致。
|
||
func ds02VerifyScopeDecl(t *testing.T, fx *writeSetFixture) {
|
||
t.Helper()
|
||
declared := map[string]bool{}
|
||
for table := range ds02ScopeDecl {
|
||
declared[table] = true
|
||
}
|
||
actual := map[string]bool{}
|
||
for _, table := range ds02TargetTables {
|
||
actual[table] = true
|
||
}
|
||
if fmt.Sprint(declared) != fmt.Sprint(actual) {
|
||
t.Fatalf("scope declaration tables %v do not match target table set %v", declared, actual)
|
||
}
|
||
for _, table := range ds02TargetTables {
|
||
rule := ds02ScopeDecl[table]
|
||
rows := ds02Read(t, fx, `SELECT jsonb_build_object('c', column_name)::text AS row FROM information_schema.columns
|
||
WHERE table_schema='public' AND table_name=$1`, table)
|
||
hasTenant, hasKB := false, false
|
||
for _, row := range rows {
|
||
switch fmt.Sprint(row["c"]) {
|
||
case "tenant_id":
|
||
hasTenant = true
|
||
case "knowledge_base_id":
|
||
hasKB = true
|
||
}
|
||
}
|
||
if hasTenant != rule.Tenant || hasKB != rule.KnowledgeBase {
|
||
t.Fatalf("%s scope declaration (tenant=%v kb=%v) disagrees with schema (tenant=%v kb=%v)",
|
||
table, rule.Tenant, rule.KnowledgeBase, hasTenant, hasKB)
|
||
}
|
||
}
|
||
r10Save(t, "scope-declaration", ds02ScopeDecl)
|
||
}
|
||
|
||
// ds02Egress 是一条被观测到的出站请求。phase 把"发布段"与"oracle 读段/夹具"
|
||
// 分开记账:只有发布段的 embedding 计数才能支撑零调用声明。
|
||
type ds02Egress struct {
|
||
Phase string `json:"phase"`
|
||
Client string `json:"client"`
|
||
Method string `json:"method"`
|
||
URL string `json:"url"`
|
||
}
|
||
|
||
type ds02EgressRecorder struct {
|
||
mu sync.Mutex
|
||
phase string
|
||
entries []ds02Egress
|
||
}
|
||
|
||
func (r *ds02EgressRecorder) setPhase(phase string) {
|
||
r.mu.Lock()
|
||
r.phase = phase
|
||
r.mu.Unlock()
|
||
}
|
||
|
||
func (r *ds02EgressRecorder) wrap(client string, base http.RoundTripper) http.RoundTripper {
|
||
if base == nil {
|
||
base = http.DefaultTransport
|
||
}
|
||
return roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||
r.mu.Lock()
|
||
r.entries = append(r.entries, ds02Egress{Phase: r.phase, Client: client, Method: request.Method, URL: request.URL.String()})
|
||
r.mu.Unlock()
|
||
return base.RoundTrip(request)
|
||
})
|
||
}
|
||
|
||
func (r *ds02EgressRecorder) entriesFor(phase string) []ds02Egress {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
var out []ds02Egress
|
||
for _, entry := range r.entries {
|
||
if entry.Phase == phase {
|
||
out = append(out, entry)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ds02IsEmbeddingTarget 按目的地址识别向量化调用,而不是只看品牌字符串:
|
||
// 任何路径里的 embed/embeddings/encode 段都算;已知模型服务域名也算。
|
||
func ds02IsEmbeddingTarget(raw string) bool {
|
||
parsed, err := url.Parse(raw)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
path := strings.ToLower(parsed.Path)
|
||
for _, marker := range []string{"embedding", "embeddings", "/embed", "/encode", "embeddings/create"} {
|
||
if strings.Contains(path, marker) {
|
||
return true
|
||
}
|
||
}
|
||
host := strings.ToLower(parsed.Hostname())
|
||
for _, brand := range []string{"dashscope", "openai", "minimax", "siliconflow", "deepseek", "volces", "aliyuncs"} {
|
||
if strings.Contains(host, brand) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ds02EgressDiffs 是纯函数:给定某一段的请求与声明的 client 清单,返回差异。
|
||
func ds02EgressDiffs(see []ds02Egress, inventory []string) []string {
|
||
var diffs []string
|
||
seenClients := map[string]bool{}
|
||
for _, entry := range see {
|
||
seenClients[entry.Client] = true
|
||
if ds02IsEmbeddingTarget(entry.URL) {
|
||
diffs = append(diffs, fmt.Sprintf("embedding calls: want 0 got %s %s (%s)", entry.Method, entry.URL, entry.Client))
|
||
}
|
||
}
|
||
for _, client := range inventory {
|
||
if !seenClients[client] {
|
||
diffs = append(diffs, "declared egress client never observed in publish: "+client)
|
||
}
|
||
}
|
||
for client := range seenClients {
|
||
known := false
|
||
for _, declared := range inventory {
|
||
if client == declared {
|
||
known = true
|
||
}
|
||
}
|
||
if !known {
|
||
diffs = append(diffs, "undeclared egress client observed in publish: "+client)
|
||
}
|
||
}
|
||
return diffs
|
||
}
|
||
|
||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||
|
||
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
|
||
|
||
// ds02RecordingHTTPClient 适配 AWS SDK 的 HTTPClient 接口(只有 Do)。
|
||
type ds02RecordingHTTPClient struct {
|
||
recorder *ds02EgressRecorder
|
||
name string
|
||
next awss3.HTTPClient
|
||
}
|
||
|
||
func (c *ds02RecordingHTTPClient) Do(request *http.Request) (*http.Response, error) {
|
||
c.recorder.mu.Lock()
|
||
c.recorder.entries = append(c.recorder.entries, ds02Egress{Phase: c.recorder.phase, Client: c.name, Method: request.Method, URL: request.URL.String()})
|
||
c.recorder.mu.Unlock()
|
||
return c.next.Do(request)
|
||
}
|
||
|
||
// ds02AttachEgress 把本测试全部可达 HTTP 构造器接到同一个记录器上,并返回可观测的
|
||
// client 清单:ES(产品搜索/备份/独立读回共用 fx.client)与对象存储 SDK 客户端。
|
||
// 正式装配里的 object gateway client 不在本测试路径上(本测试直接注入 S3 存储)。
|
||
func ds02AttachEgress(t *testing.T, fx *writeSetFixture) (*ds02EgressRecorder, []string) {
|
||
t.Helper()
|
||
recorder := &ds02EgressRecorder{phase: "fixture"}
|
||
fx.client.Transport = recorder.wrap("elastic", http.DefaultTransport)
|
||
t.Cleanup(func() {
|
||
fx.client.Transport = nil
|
||
})
|
||
options := fx.objects.s3.client.Options()
|
||
next := options.HTTPClient
|
||
if next == nil {
|
||
next = http.DefaultClient
|
||
}
|
||
options.HTTPClient = &ds02RecordingHTTPClient{recorder: recorder, name: "object-store", next: next}
|
||
fx.objects.s3.client = awss3.New(options)
|
||
return recorder, []string{"elastic", "object-store"}
|
||
}
|
||
|
||
// ds02Check 是 oracle 本体:返回逐项差异;空 = 全绿。
|
||
func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[string][]map[string]any,
|
||
paths map[string]string, preserved map[string][]map[string]any,
|
||
recorder *ds02EgressRecorder, inventory []string) []string {
|
||
t.Helper()
|
||
var diffs []string
|
||
scope := func(table string) (string, []any) {
|
||
rule, ok := ds02ScopeDecl[table]
|
||
if !ok {
|
||
t.Fatalf("table %s missing from independent scope declaration", table)
|
||
}
|
||
if !rule.Tenant {
|
||
return "SELECT to_jsonb(x)::text FROM " + table + " AS x ORDER BY 1", nil
|
||
}
|
||
if rule.KnowledgeBase {
|
||
return "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY 1",
|
||
[]any{wsTargetTenant, wsTargetKB}
|
||
}
|
||
return "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id=$1 ORDER BY 1", []any{wsTargetTenant}
|
||
}
|
||
for _, table := range ds02TargetTables {
|
||
query, args := scope(table)
|
||
if diff := ds02Compare(table, expected[table], ds02Read(t, fx, query, args...)); diff != "" {
|
||
diffs = append(diffs, diff)
|
||
}
|
||
// 目标范围之外的行必须一字不动(含其它知识库/租户以及它们的历史资源)。
|
||
if !ds02ScopeDecl[table].Tenant {
|
||
continue
|
||
}
|
||
other := "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id<>$1"
|
||
if ds02ScopeDecl[table].KnowledgeBase {
|
||
other = "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)"
|
||
}
|
||
if diff := ds02Compare(table+"(non-target)", preserved[table], ds02Read(t, fx, other, args...)); diff != "" {
|
||
diffs = append(diffs, diff)
|
||
}
|
||
}
|
||
// 引用清单:逐种关系分别断言 violations=0 **且** matched>0。范围外关系显式列出,
|
||
// 不把"七组 OR 查询"叫"全部引用"。
|
||
type relation struct {
|
||
name string
|
||
sql string
|
||
matched string
|
||
covered bool
|
||
reason string
|
||
}
|
||
relations := []relation{
|
||
{"chunk→knowledge", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`,
|
||
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`, true, ""},
|
||
{"chunk.pre_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.pre_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id AND p.tenant_id=c.tenant_id AND p.knowledge_base_id=c.knowledge_base_id)`,
|
||
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.pre_chunk_id<>'' AND EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id AND p.tenant_id=c.tenant_id AND p.knowledge_base_id=c.knowledge_base_id)`, true, ""},
|
||
{"chunk.next_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.next_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id AND n.tenant_id=c.tenant_id AND n.knowledge_base_id=c.knowledge_base_id)`,
|
||
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.next_chunk_id<>'' AND EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id AND n.tenant_id=c.tenant_id AND n.knowledge_base_id=c.knowledge_base_id)`, true, ""},
|
||
{"chunk.parent_chunk_id→chunk", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND coalesce(c.parent_chunk_id,'')<>'' AND NOT EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id AND q.tenant_id=c.tenant_id AND q.knowledge_base_id=c.knowledge_base_id)`,
|
||
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND coalesce(c.parent_chunk_id,'')<>'' AND EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id AND q.tenant_id=c.tenant_id AND q.knowledge_base_id=c.knowledge_base_id)`, true, ""},
|
||
{"relation→tag", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id)`,
|
||
`SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id)`, true, ""},
|
||
{"relation→knowledge", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`,
|
||
`SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`, true, ""},
|
||
{"binding→resource", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND NOT EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`,
|
||
`SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`, true, ""},
|
||
{"binding→knowledge", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND b.owner_type='knowledge' AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)`,
|
||
`SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND b.owner_type='knowledge' AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)`, true, ""},
|
||
{"wiki page→folder", `SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>'' AND NOT EXISTS(SELECT 1 FROM wiki_folders f WHERE f.id=p.folder_id)`,
|
||
`SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>''`, true, ""},
|
||
{"wiki folder→parent folder", `SELECT count(*) FROM wiki_folders f WHERE f.tenant_id=$1 AND f.knowledge_base_id=$2 AND f.parent_id<>'' AND NOT EXISTS(SELECT 1 FROM wiki_folders p WHERE p.id=f.parent_id AND p.tenant_id=f.tenant_id AND p.knowledge_base_id=f.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_folders f WHERE f.tenant_id=$1 AND f.knowledge_base_id=$2 AND f.parent_id<>''`, true, ""},
|
||
{"wiki page.source_refs→knowledge", `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.source_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=split_part(ref,'|',1) AND k.tenant_id=p.tenant_id AND k.knowledge_base_id=p.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.source_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=split_part(ref,'|',1) AND k.tenant_id=p.tenant_id AND k.knowledge_base_id=p.knowledge_base_id)`, true, ""},
|
||
{"wiki page.chunk_refs→chunk", `SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.chunk_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM chunks c WHERE c.id=ref AND c.tenant_id=p.tenant_id AND c.knowledge_base_id=p.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_pages p, jsonb_array_elements_text(coalesce(p.chunk_refs,'[]'::jsonb)) ref WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM chunks c WHERE c.id=ref AND c.tenant_id=p.tenant_id AND c.knowledge_base_id=p.knowledge_base_id)`, true, ""},
|
||
{"wiki issue.suspected_knowledge_ids→knowledge", `SELECT count(*) FROM wiki_page_issues i, jsonb_array_elements_text(coalesce(i.suspected_knowledge_ids,'[]'::jsonb)) ref WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=ref AND k.tenant_id=i.tenant_id AND k.knowledge_base_id=i.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_page_issues i, jsonb_array_elements_text(coalesce(i.suspected_knowledge_ids,'[]'::jsonb)) ref WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=ref AND k.tenant_id=i.tenant_id AND k.knowledge_base_id=i.knowledge_base_id)`, true, ""},
|
||
{"wiki issue→page", `SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=i.slug AND p.tenant_id=i.tenant_id AND p.knowledge_base_id=i.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2`, true, ""},
|
||
{"wiki log→knowledge", `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=l.knowledge_id)`,
|
||
`SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2`, true, ""},
|
||
{"wiki log.pages_affected→page", `SELECT count(*) FROM wiki_log_entries l, jsonb_array_elements(coalesce(l.pages_affected,'[]'::jsonb)) e WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)`,
|
||
`SELECT count(*) FROM wiki_log_entries l, jsonb_array_elements(coalesce(l.pages_affected,'[]'::jsonb)) e WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)`, true, ""},
|
||
// 范围外(显式声明,不静默跳过):软导航字段没有外键契约,允许悬空。
|
||
{"wiki page.parent_slug", "", "", false, "soft slug navigation, no FK; fixture keeps it empty"},
|
||
{"wiki page.in_links/out_links", "", "", false, "soft slug navigation, no FK; dangling by design"},
|
||
{"chunk.relation_chunks/indirect_relation_chunks", "", "", false, "jsonb relation hints, no FK contract"},
|
||
}
|
||
for _, relation := range relations {
|
||
if !relation.covered {
|
||
continue
|
||
}
|
||
var violations, matched int64
|
||
var args []any
|
||
if strings.Contains(relation.sql, "$2") {
|
||
args = []any{wsTargetTenant, wsTargetKB}
|
||
} else if strings.Contains(relation.sql, "$1") {
|
||
args = []any{wsTargetTenant}
|
||
}
|
||
if err := fx.pool.QueryRow(fx.ctx, relation.sql, args...).Scan(&violations); err != nil {
|
||
t.Fatalf("reference %s: %v", relation.name, err)
|
||
}
|
||
if err := fx.pool.QueryRow(fx.ctx, relation.matched, args...).Scan(&matched); err != nil {
|
||
t.Fatalf("reference coverage %s: %v", relation.name, err)
|
||
}
|
||
if violations != 0 {
|
||
diffs = append(diffs, fmt.Sprintf("reference %s: %d violations", relation.name, violations))
|
||
}
|
||
if matched == 0 {
|
||
diffs = append(diffs, fmt.Sprintf("reference %s: oracle matched no rows", relation.name))
|
||
}
|
||
}
|
||
inventoryRaw := make([]map[string]any, 0, len(relations))
|
||
for _, item := range relations {
|
||
inventoryRaw = append(inventoryRaw, map[string]any{"name": item.name, "covered": item.covered,
|
||
"out_of_scope_reason": item.reason})
|
||
}
|
||
r10Save(t, "reference-inventory", inventoryRaw)
|
||
// 向量/正文:索引文档逐条与声明完全一致(多一条少一条都算差异)。
|
||
docs := ds02Docs(t, fx)
|
||
if len(docs) != len(spec.rows["chunks"])+1 { // +1:目标范围之外的 chunk-foreign
|
||
diffs = append(diffs, fmt.Sprintf("index docs: want %d got %d", len(spec.rows["chunks"])+1, len(docs)))
|
||
}
|
||
for _, chunk := range spec.rows["chunks"] {
|
||
sourceID := fmt.Sprint(chunk["id"])
|
||
id := expectedIndexID(sourceID)
|
||
doc, ok := docs[id]
|
||
if !ok {
|
||
diffs = append(diffs, "index doc missing: "+id)
|
||
continue
|
||
}
|
||
source := doc
|
||
want := map[string]any{"tenant_id": json.Number(strconv.FormatUint(wsTargetTenant, 10)),
|
||
"knowledge_base_id": wsTargetKB,
|
||
"knowledge_id": expectedTargetID(wsTargetTenant, wsTargetKB, "knowledges", fmt.Sprint(chunk["knowledge_id"])),
|
||
"chunk_id": expectedTargetID(wsTargetTenant, wsTargetKB, "chunks", sourceID),
|
||
"source_id": expectedTargetID(wsTargetTenant, wsTargetKB, "chunks", sourceID),
|
||
"content": chunk["content"], "embedding": spec.vectors[sourceID]}
|
||
for key, value := range want {
|
||
if fmt.Sprint(source[key]) != fmt.Sprint(value) {
|
||
diffs = append(diffs, fmt.Sprintf("index doc %s.%s: want %v got %v", id, key, value, source[key]))
|
||
}
|
||
}
|
||
}
|
||
// 外围索引文档:完整 _source 逐字段比,不只比 content。
|
||
foreignWant := map[string]any{"tenant_id": json.Number("20000"), "knowledge_base_id": "kb-foreign",
|
||
"knowledge_id": "doc-foreign", "chunk_id": "chunk-foreign", "source_id": "chunk-foreign",
|
||
"content": "外部正文", "embedding": []any{0.1, 0.2}}
|
||
if foreign, ok := docs["chunk-foreign"]; !ok {
|
||
diffs = append(diffs, "foreign index doc disappeared")
|
||
} else if !ds02SameJSON(foreign, foreignWant) {
|
||
diffs = append(diffs, fmt.Sprintf("foreign index doc _source changed: want %s got %s", ds02RowKey(foreignWant), ds02RowKey(foreign)))
|
||
}
|
||
// 对象字节:按独立重算的定位符读回,逐字节比。
|
||
for id, want := range spec.objects {
|
||
reader, err := fx.objects.s3.Open(fx.ctx, "minio", paths[id])
|
||
if err != nil {
|
||
diffs = append(diffs, fmt.Sprintf("object %s: %v", id, err))
|
||
continue
|
||
}
|
||
got, err := io.ReadAll(reader)
|
||
reader.Close()
|
||
if err != nil {
|
||
diffs = append(diffs, fmt.Sprintf("object %s read: %v", id, err))
|
||
continue
|
||
}
|
||
if string(got) != want {
|
||
diffs = append(diffs, fmt.Sprintf("object %s bytes: want %q got %q", id, want, got))
|
||
}
|
||
}
|
||
// 外围对象:res-other 的字节也必须逐字节比(旧 oracle 只读 spec.objects)。
|
||
otherPath := wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other")
|
||
reader, err := fx.objects.s3.Open(fx.ctx, "minio", otherPath)
|
||
if err != nil {
|
||
diffs = append(diffs, "sibling object res-other unreadable: "+err.Error())
|
||
} else {
|
||
got, readErr := io.ReadAll(reader)
|
||
reader.Close()
|
||
if readErr != nil {
|
||
diffs = append(diffs, "sibling object res-other read: "+readErr.Error())
|
||
} else if string(got) != strings.Repeat("z", 8) {
|
||
diffs = append(diffs, fmt.Sprintf("sibling object res-other bytes: want %q got %q", strings.Repeat("z", 8), got))
|
||
}
|
||
}
|
||
// Wiki 日志映射:内容元组多重集双射(本地号不参与身份,来源 id 只用于覆盖检查)。
|
||
wantTuples := []string{}
|
||
for _, row := range spec.rows["wiki_log_entries"] {
|
||
copied := map[string]any{}
|
||
for key, value := range row {
|
||
if key == "id" {
|
||
continue
|
||
}
|
||
copied[key] = value
|
||
}
|
||
copied["tenant_id"] = json.Number(strconv.FormatUint(wsTargetTenant, 10))
|
||
copied["knowledge_base_id"] = wsTargetKB
|
||
if knowledge, ok := copied["knowledge_id"].(string); ok && knowledge != "" {
|
||
copied["knowledge_id"] = expectedTargetID(wsTargetTenant, wsTargetKB, "knowledges", knowledge)
|
||
}
|
||
raw, _ := json.Marshal(copied)
|
||
wantTuples = append(wantTuples, string(raw))
|
||
}
|
||
gotTuples := []string{}
|
||
actualLogs := ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM wiki_log_entries AS x WHERE tenant_id=$1 AND knowledge_base_id=$2`, wsTargetTenant, wsTargetKB)
|
||
for _, row := range actualLogs {
|
||
copied := map[string]any{}
|
||
for key, value := range row {
|
||
if key == "id" {
|
||
continue
|
||
}
|
||
copied[key] = value
|
||
}
|
||
raw, _ := json.Marshal(copied)
|
||
gotTuples = append(gotTuples, string(raw))
|
||
}
|
||
sort.Strings(wantTuples)
|
||
sort.Strings(gotTuples)
|
||
if fmt.Sprint(wantTuples) != fmt.Sprint(gotTuples) {
|
||
diffs = append(diffs, fmt.Sprintf("wiki log tuple bijection broken: want %v got %v", wantTuples, gotTuples))
|
||
}
|
||
// Embedding:只对"发布段"记账;oracle 读段的请求不混进来。发布段必须有请求
|
||
// (不是空绿),且 client 集合必须正是本测试声明的可达集合。
|
||
publishEgress := recorder.entriesFor("publish")
|
||
diffs = append(diffs, ds02EgressDiffs(publishEgress, inventory)...)
|
||
if len(publishEgress) == 0 {
|
||
diffs = append(diffs, "no HTTP requests observed in publish phase; zero-embedding oracle would be vacuous")
|
||
}
|
||
return diffs
|
||
}
|
||
|
||
func ds02SameJSON(left, right map[string]any) bool {
|
||
return ds02RowKey(left) == ds02RowKey(right)
|
||
}
|
||
|
||
// ds02OtherRows 读目标范围之外的行(发布前抓一次,发布后要求一字不差)。
|
||
func ds02OtherRows(t *testing.T, fx *writeSetFixture, table string) []map[string]any {
|
||
t.Helper()
|
||
rule := ds02ScopeDecl[table]
|
||
if !rule.Tenant {
|
||
return nil
|
||
}
|
||
if rule.KnowledgeBase {
|
||
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)", wsTargetTenant, wsTargetKB)
|
||
}
|
||
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id<>$1", wsTargetTenant)
|
||
}
|
||
|
||
func ds02SameTenantRows(t *testing.T, fx *writeSetFixture, table string) []map[string]any {
|
||
t.Helper()
|
||
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id=$1", wsTargetTenant)
|
||
}
|
||
|
||
// ds02SequenceSummary 记三条本地号序列的 (last_value,is_called):序列不存在直接失败,
|
||
// 不允许"空集合被当成不变"。
|
||
func ds02SequenceSummary(t *testing.T, fx *writeSetFixture) map[string][]any {
|
||
t.Helper()
|
||
summary := map[string][]any{}
|
||
for _, name := range []string{"chunks_seq_id_seq", "knowledge_tags_seq_id_seq", "wiki_log_entries_id_seq"} {
|
||
var exists bool
|
||
if err := fx.pool.QueryRow(fx.ctx, "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !exists {
|
||
t.Fatalf("local sequence %s does not exist; sequence coverage would be empty", name)
|
||
}
|
||
var last int64
|
||
var called bool
|
||
if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+name).Scan(&last, &called); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
summary[name] = []any{last, called}
|
||
}
|
||
return summary
|
||
}
|
||
|
||
// ds02LocalColumns 把"声明来源 + 独立期望目标行 + 真实目标行"组成映射 oracle 的输入。
|
||
func ds02LocalColumns(t *testing.T, fx *writeSetFixture, spec ds02Spec, expectation ds02Expectation) []ds02LocalColumn {
|
||
t.Helper()
|
||
var columns []ds02LocalColumn
|
||
for _, contract := range []struct{ table, column string }{
|
||
{"chunks", "seq_id"}, {"knowledge_tags", "seq_id"}, {"wiki_log_entries", "id"},
|
||
} {
|
||
sources := []int64{}
|
||
for _, row := range spec.rows[contract.table] {
|
||
raw := row["seq_id"]
|
||
if contract.table == "wiki_log_entries" {
|
||
raw = row["id"]
|
||
}
|
||
value, err := strconv.ParseInt(fmt.Sprint(raw), 10, 64)
|
||
if err != nil {
|
||
t.Fatalf("%s declared local id: %v", contract.table, err)
|
||
}
|
||
sources = append(sources, value)
|
||
}
|
||
sort.Slice(sources, func(i, j int) bool { return sources[i] < sources[j] })
|
||
actual := map[int64]map[string]any{}
|
||
for _, row := range ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+contract.table+
|
||
" AS x WHERE tenant_id=$1 AND knowledge_base_id=$2", wsTargetTenant, wsTargetKB) {
|
||
value, err := strconv.ParseInt(fmt.Sprint(row[contract.column]), 10, 64)
|
||
if err != nil {
|
||
t.Fatalf("%s actual local id: %v", contract.table, err)
|
||
}
|
||
actual[value] = row
|
||
}
|
||
columns = append(columns, ds02LocalColumn{Table: contract.table, Column: contract.column,
|
||
Sources: sources, Expected: expectation.Local[contract.table],
|
||
ExpectedRows: expectation.RowsBySource[contract.table], Actual: actual})
|
||
}
|
||
return columns
|
||
}
|
||
|
||
func TestDS02FullShapeOracleIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
// 用保留下来的真实 v071 catalog 重建十张表:逐列、外键、序列都是真实形状。
|
||
r11Mirror(t, fx)
|
||
for _, sql := range []string{
|
||
`SELECT setval('chunks_seq_id_seq',100000005,false)`,
|
||
`SELECT setval('knowledge_tags_seq_id_seq',50,false)`,
|
||
`SELECT setval('wiki_log_entries_id_seq',10,false)`,
|
||
} {
|
||
r10Exec(t, fx, sql)
|
||
}
|
||
ds02VerifyScopeDecl(t, fx)
|
||
ds02Seed(t, fx)
|
||
preserved := map[string][]map[string]any{}
|
||
for _, table := range ds02TargetTables {
|
||
preserved[table] = ds02OtherRows(t, fx, table)
|
||
}
|
||
spec := ds02SourceSpec(t, fx)
|
||
expectation := ds02Expected(t, fx, spec)
|
||
expected, paths := expectation.Rows, expectation.Paths
|
||
// 独立 oracle:只对"来源快照将写进目标范围"的期望行按 scoped-sha256-v1
|
||
// 推导目标身份;随后追加的同租户/外库保留行仍是原身份,不能映射。
|
||
for table := range expected {
|
||
for i, row := range expected[table] {
|
||
expected[table][i] = expectedTargetRow(table, row)
|
||
}
|
||
}
|
||
for table, bySeq := range expectation.RowsBySource {
|
||
for seq, row := range bySeq {
|
||
bySeq[seq] = expectedTargetRow(table, row)
|
||
}
|
||
}
|
||
for _, table := range []string{"resources", "resource_bindings"} {
|
||
expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...)
|
||
}
|
||
// knowledge_tag_relations 没有 tenant/kb:产品按父文档归属删除,非目标范围的
|
||
// 关系行必须原样保留,所以它们也在期望里。
|
||
expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"],
|
||
ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x
|
||
WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`,
|
||
wsTargetTenant, wsTargetKB)...)
|
||
|
||
sequencesBefore := ds02SequenceSummary(t, fx)
|
||
snapshot := ds02Seal(t, fx, spec)
|
||
recorder, inventory := ds02AttachEgress(t, fx)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
recorder.setPhase("publish")
|
||
out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil)
|
||
recorder.setPhase("oracle-read")
|
||
r10Save(t, "outcome", out)
|
||
if out.State != "succeeded" {
|
||
t.Fatalf("full-shape publish must succeed: %+v", out)
|
||
}
|
||
sequencesAfter := ds02SequenceSummary(t, fx)
|
||
r10Save(t, "sequences", map[string]any{"before": sequencesBefore, "after": sequencesAfter})
|
||
// 写集合里的本地号映射:独立语义 checker(不是"数个数/看双射"的循环)。
|
||
if out.Backup.WriteSet == nil {
|
||
t.Fatal("write set missing")
|
||
}
|
||
ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
||
if err != nil {
|
||
t.Fatalf("load write set: %v", err)
|
||
}
|
||
columns := ds02LocalColumns(t, fx, spec, expectation)
|
||
mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs)
|
||
r10Save(t, "mapping-diffs", mappingDiffs)
|
||
if len(mappingDiffs) != 0 {
|
||
t.Fatalf("persisted ledger fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n"))
|
||
}
|
||
// 语义副本变异:只改内存里的账本副本,目标存储与归档字节都不动。这个 RED
|
||
// 才是 mapping RED;改日志 action/正文只能证明内容 oracle 会红。
|
||
for _, scenario := range []struct {
|
||
name string
|
||
build func([]libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping
|
||
want string
|
||
}{
|
||
{"swapped-targets", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02FlipMappingTargets(ledger, "wiki_log_entries")
|
||
}, "wrong target pair"},
|
||
{"nonexistent-target", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02ReplaceMapping(ledger, "wiki_log_entries", "target", "999")
|
||
}, "not an independently predicted target"},
|
||
{"replaced-source", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02ReplaceMapping(ledger, "wiki_log_entries", "source", "999")
|
||
}, "unexpected ledger source"},
|
||
{"missing-entry", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02DropMapping(ledger, "chunks")
|
||
}, "missing ledger source"},
|
||
{"duplicate-entry", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02DuplicateMapping(ledger, "knowledge_tags")
|
||
}, "duplicate ledger source"},
|
||
{"wrong-column", func(ledger []libraryrelease.LocalIDMapping) []libraryrelease.LocalIDMapping {
|
||
return ds02ReplaceMapping(ledger, "chunks", "column", "id")
|
||
}, "ledger column"},
|
||
} {
|
||
red := ds02CheckLocalIDMapping(columns, scenario.build(ledger.LocalIDs))
|
||
r10Save(t, "mapping-red-"+scenario.name, red)
|
||
joined := strings.Join(red, "\n")
|
||
if len(red) == 0 {
|
||
t.Fatalf("%s: semantic mapping mutation stayed green", scenario.name)
|
||
}
|
||
if !strings.Contains(joined, scenario.want) {
|
||
t.Fatalf("%s: mapping oracle failed for an unrelated reason:\n%s", scenario.name, joined)
|
||
}
|
||
}
|
||
// 归档字节变异 = checksum gate,不是 mapping RED:它只能证明摘要拒收,
|
||
// 不能替代上面那组语义 RED。
|
||
ledgerPath := writeSetPath(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
||
info, err := os.Stat(ledgerPath)
|
||
if err != nil {
|
||
t.Fatalf("stat persisted ledger: %v", err)
|
||
}
|
||
original, err := os.ReadFile(ledgerPath)
|
||
if err != nil {
|
||
t.Fatalf("read persisted ledger: %v", err)
|
||
}
|
||
// 台账落盘是只读的,变异前显式放开权限,结束后恢复原权限与原始字节。
|
||
if err := os.Chmod(ledgerPath, 0600); err != nil {
|
||
t.Fatalf("unlock persisted ledger: %v", err)
|
||
}
|
||
// 只追加空白不会改变解析后的内容摘要(实现按内容摘要校验,不按字节);
|
||
// 这里改一个真实字段值,让内容摘要必然不符。
|
||
var tampered map[string]any
|
||
if err := json.Unmarshal(original, &tampered); err != nil {
|
||
t.Fatalf("parse persisted ledger: %v", err)
|
||
}
|
||
if mappings, ok := tampered["local_ids"].([]any); ok && len(mappings) > 0 {
|
||
if first, ok := mappings[0].(map[string]any); ok {
|
||
first["target"] = "999999999"
|
||
}
|
||
}
|
||
reencoded, err := json.Marshal(tampered)
|
||
if err != nil {
|
||
t.Fatalf("encode tampered ledger: %v", err)
|
||
}
|
||
if err := os.WriteFile(ledgerPath, reencoded, 0600); err != nil {
|
||
t.Fatalf("tamper persisted ledger: %v", err)
|
||
}
|
||
_, digestErr := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
||
r10Save(t, "digest-gate", map[string]any{"error": fmt.Sprint(digestErr)})
|
||
if digestErr == nil {
|
||
t.Fatal("tampered ledger bytes were accepted; digest gate is missing")
|
||
}
|
||
if err := os.WriteFile(ledgerPath, original, 0600); err != nil {
|
||
t.Fatalf("restore persisted ledger: %v", err)
|
||
}
|
||
if err := os.Chmod(ledgerPath, info.Mode().Perm()); err != nil {
|
||
t.Fatalf("restore ledger mode: %v", err)
|
||
}
|
||
if _, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest); err != nil {
|
||
t.Fatalf("restored ledger must load again: %v", err)
|
||
}
|
||
diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory)
|
||
r10Save(t, "green-diffs", diffs)
|
||
if len(diffs) != 0 {
|
||
t.Fatalf("independent oracle disagrees on a correct publish:\n%s", strings.Join(diffs, "\n"))
|
||
}
|
||
r10Save(t, "green-egress", map[string]any{"publish": recorder.entriesFor("publish"),
|
||
"oracle": recorder.entriesFor("oracle-read"), "inventory": inventory})
|
||
|
||
// 负控制:另一个 client 打到自定义 /v1/encode 路径必须被识别为 embedding,
|
||
// 而真正的发布段仍是 0。这证明分类器不靠品牌字符串。
|
||
encoder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.WriteHeader(200)
|
||
}))
|
||
defer encoder.Close()
|
||
custom := &http.Client{Transport: recorder.wrap("custom-encoder", http.DefaultTransport)}
|
||
recorder.setPhase("negative-control")
|
||
request, err := http.NewRequestWithContext(fx.ctx, "POST", encoder.URL+"/v1/encode", strings.NewReader(`{}`))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
response, err := custom.Do(request)
|
||
if err != nil {
|
||
t.Fatalf("negative control request: %v", err)
|
||
}
|
||
response.Body.Close()
|
||
negative := recorder.entriesFor("negative-control")
|
||
negativeDiffs := ds02EgressDiffs(append(append([]ds02Egress{}, recorder.entriesFor("publish")...), negative...), inventory)
|
||
r10Save(t, "negative-control", map[string]any{"entries": negative, "diffs_if_in_publish": negativeDiffs})
|
||
if len(negativeDiffs) == 0 {
|
||
t.Fatal("custom /v1/encode path was not classified as an embedding call")
|
||
}
|
||
if len(ds02EgressDiffs(recorder.entriesFor("publish"), inventory)) != 0 {
|
||
t.Fatalf("publish phase embedding expectation broken: %v", ds02EgressDiffs(recorder.entriesFor("publish"), inventory))
|
||
}
|
||
recorder.setPhase("oracle-read")
|
||
|
||
// 变异自检:对真实目标改正文 / 向量 / 引用 / 日志内容 / 外围 ES 文档 / 外围对象,
|
||
// oracle 必须红;改回后必须绿。
|
||
content := fmt.Sprint(expected["chunks"][0]["content"])
|
||
putDoc := func(id, body string) {
|
||
request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/"+id+"?refresh=true", strings.NewReader(body))
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := fx.client.Do(request)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
response.Body.Close()
|
||
if response.StatusCode >= 300 {
|
||
t.Fatalf("mutate doc %s: %d", id, response.StatusCode)
|
||
}
|
||
}
|
||
putObject := func(path, body string) {
|
||
if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
foreignPath := wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other")
|
||
foreignDoc := `{"tenant_id":20000,"knowledge_base_id":"kb-foreign","knowledge_id":"doc-foreign","chunk_id":"chunk-foreign","source_id":"chunk-foreign","content":"外部正文","embedding":[0.1,0.2]}`
|
||
foreignDocTampered := `{"tenant_id":20000,"knowledge_base_id":"kb-foreign","knowledge_id":"doc-foreign","chunk_id":"chunk-foreign","source_id":"chunk-foreign","content":"外部正文篡改","embedding":[0.1,0.2]}`
|
||
mappedDoc := expectedTargetID(wsTargetTenant, wsTargetKB, "knowledges", "doc-src")
|
||
mappedTag := expectedTargetID(wsTargetTenant, wsTargetKB, "knowledge_tags", "tag-src")
|
||
mappedChunk := expectedTargetID(wsTargetTenant, wsTargetKB, "chunks", "chunk-src")
|
||
mappedIndex := expectedIndexID("chunk-src")
|
||
chunkJSON := func(embedding string) string {
|
||
return fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":%s}`,
|
||
wsTargetTenant, wsTargetKB, mappedDoc, mappedChunk, mappedChunk, content, embedding)
|
||
}
|
||
chunkDoc, chunkDocTampered := chunkJSON("[0.5,1.5]"), chunkJSON("[9.5,9.5]")
|
||
var originalChunk string
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT content FROM chunks WHERE id=$1`, mappedChunk).Scan(&originalChunk); err != nil {
|
||
t.Fatalf("read published chunk content: %v", err)
|
||
}
|
||
execOne := func(sql string, args ...any) {
|
||
tag, err := fx.pool.Exec(fx.ctx, sql, args...)
|
||
if err != nil {
|
||
t.Fatalf("mutation sql: %v", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
t.Fatalf("mutation sql affected %d rows, want 1", tag.RowsAffected())
|
||
}
|
||
}
|
||
mutations := []struct {
|
||
name string
|
||
marker string
|
||
mutate func()
|
||
restore func()
|
||
}{
|
||
{"content", "chunks:", func() {
|
||
execOne(`UPDATE chunks SET content=content||'!MUT' WHERE id=$1`, mappedChunk)
|
||
}, func() {
|
||
execOne(`UPDATE chunks SET content=$1 WHERE id=$2`, originalChunk, mappedChunk)
|
||
}},
|
||
{"vector", "index doc " + expectedIndexID("chunk-src") + ".embedding", func() {
|
||
putDoc(mappedIndex, chunkDocTampered)
|
||
}, func() {
|
||
putDoc(mappedIndex, chunkDoc)
|
||
}},
|
||
{"reference", "reference relation", func() {
|
||
execOne(`UPDATE knowledge_tag_relations SET tag_id='missing-tag' WHERE knowledge_id=$1 AND tag_id=$2`, mappedDoc, mappedTag)
|
||
}, func() {
|
||
execOne(`UPDATE knowledge_tag_relations SET tag_id=$1 WHERE knowledge_id=$2 AND tag_id='missing-tag'`, mappedTag, mappedDoc)
|
||
}},
|
||
// 这是日志内容变异,不是本地号映射变异:映射 RED 只在账本语义副本上做。
|
||
{"log-content", "wiki log", func() {
|
||
execOne(`UPDATE wiki_log_entries SET action='tampered' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id=$1`, mappedDoc)
|
||
}, func() {
|
||
execOne(`UPDATE wiki_log_entries SET action='ingest' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id=$1`, mappedDoc)
|
||
}},
|
||
{"foreign-index-source", "foreign index doc _source changed", func() {
|
||
putDoc("chunk-foreign", foreignDocTampered)
|
||
}, func() {
|
||
putDoc("chunk-foreign", foreignDoc)
|
||
}},
|
||
{"foreign-object-bytes", "res-other bytes", func() {
|
||
putObject(foreignPath, strings.Repeat("z", 7)+"!")
|
||
}, func() {
|
||
putObject(foreignPath, strings.Repeat("z", 8))
|
||
}},
|
||
}
|
||
for _, mutation := range mutations {
|
||
mutation.mutate()
|
||
red := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory)
|
||
r10Save(t, "red-"+mutation.name, red)
|
||
if len(red) == 0 {
|
||
t.Fatalf("%s mutation: oracle stayed green", mutation.name)
|
||
}
|
||
joined := strings.Join(red, "\n")
|
||
if !strings.Contains(joined, mutation.marker) {
|
||
t.Fatalf("%s mutation: oracle failed for an unrelated reason:\n%s", mutation.name, joined)
|
||
}
|
||
mutation.restore()
|
||
green := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory)
|
||
if len(green) != 0 {
|
||
t.Fatalf("%s mutation not restored cleanly:\n%s", mutation.name, strings.Join(green, "\n"))
|
||
}
|
||
}
|
||
}
|