|
|
|
@@ -0,0 +1,845 @@
|
|
|
|
|
package v071
|
|
|
|
|
|
|
|
|
|
// DS-02:整库发布的独立 oracle。
|
|
|
|
|
//
|
|
|
|
|
// 这里不调用被测的映射/改写函数来生成期望值:期望由夹具自己声明的十张表逐列内容
|
|
|
|
|
// 推导(来源范围 → 目标范围的映射规则、资源定位符、location_hash 都在本文件里
|
|
|
|
|
// 用字面规则重算),本地号(chunks.seq_id / knowledge_tags.seq_id /
|
|
|
|
|
// wiki_log_entries.id)由测试**发布前**读到的序列状态加声明顺序算出来。
|
|
|
|
|
// 正文、引用、向量、对象字节、Wiki 日志映射分别有独立断言;断言本身再用
|
|
|
|
|
// "真实改写目标存储 → 必须红 → 改回 → 必须绿" 证明不是空跑。
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"crypto/sha256"
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"io"
|
|
|
|
|
"net/http"
|
|
|
|
|
"sort"
|
|
|
|
|
"strconv"
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"knowledge-governance/internal/libraryrelease"
|
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// ds02Vectors 是夹具声明的来源向量(逐分块不同)。索引部件由它生成,
|
|
|
|
|
// 不是由被测适配器生成。
|
|
|
|
|
var ds02Vectors = map[string][]float64{
|
|
|
|
|
"chunk-src": {0.5, 1.5},
|
|
|
|
|
"chunk-src2": {-2.25, 3.75},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Objects 是夹具声明的资源字节(长度与 resources.size 一致)。
|
|
|
|
|
var ds02Objects = map[string]string{
|
|
|
|
|
"res-src": "abc12345",
|
|
|
|
|
"res-src2": "wxyz6789",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var ds02TargetTables = []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations",
|
|
|
|
|
"resources", "resource_bindings", "wiki_folders", "wiki_pages", "wiki_page_issues", "wiki_log_entries"}
|
|
|
|
|
|
|
|
|
|
func ds02Canon(t *testing.T, fx *writeSetFixture, table string, row map[string]any) map[string]any {
|
|
|
|
|
t.Helper()
|
|
|
|
|
raw, err := json.Marshal(row)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var canonical []byte
|
|
|
|
|
if err := fx.pool.QueryRow(fx.ctx,
|
|
|
|
|
"SELECT to_jsonb(jsonb_populate_record(NULL::"+table+",$1::jsonb))", raw).Scan(&canonical); err != nil {
|
|
|
|
|
t.Fatalf("canonicalize %s: %v", table, err)
|
|
|
|
|
}
|
|
|
|
|
dec := json.NewDecoder(bytes.NewReader(canonical))
|
|
|
|
|
dec.UseNumber()
|
|
|
|
|
var out map[string]any
|
|
|
|
|
if err := dec.Decode(&out); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ds02Read(t *testing.T, fx *writeSetFixture, query string, args ...any) []map[string]any {
|
|
|
|
|
t.Helper()
|
|
|
|
|
rows, err := fx.pool.Query(fx.ctx, query, args...)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("oracle query: %v", err)
|
|
|
|
|
}
|
|
|
|
|
defer rows.Close()
|
|
|
|
|
var out []map[string]any
|
|
|
|
|
for rows.Next() {
|
|
|
|
|
var text string
|
|
|
|
|
if err := rows.Scan(&text); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
dec := json.NewDecoder(strings.NewReader(text))
|
|
|
|
|
dec.UseNumber()
|
|
|
|
|
var row map[string]any
|
|
|
|
|
if err := dec.Decode(&row); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
out = append(out, row)
|
|
|
|
|
}
|
|
|
|
|
if err := rows.Err(); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ds02Normalize(rows []map[string]any) []string {
|
|
|
|
|
out := make([]string, 0, len(rows))
|
|
|
|
|
for _, row := range rows {
|
|
|
|
|
raw, _ := json.Marshal(row)
|
|
|
|
|
out = append(out, string(raw))
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(out)
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Compare 返回人可读的差异;空串表示逐列内容完全一致(含行数)。
|
|
|
|
|
func ds02Compare(table string, want, got []map[string]any) string {
|
|
|
|
|
w, g := ds02Normalize(want), ds02Normalize(got)
|
|
|
|
|
if len(w) == len(g) {
|
|
|
|
|
same := true
|
|
|
|
|
for i := range w {
|
|
|
|
|
if w[i] != g[i] {
|
|
|
|
|
same = false
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if same {
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
seen := map[string]bool{}
|
|
|
|
|
for _, row := range g {
|
|
|
|
|
seen[row] = true
|
|
|
|
|
}
|
|
|
|
|
var missing, extra []string
|
|
|
|
|
for _, row := range w {
|
|
|
|
|
if !seen[row] {
|
|
|
|
|
missing = append(missing, row)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
present := map[string]bool{}
|
|
|
|
|
for _, row := range w {
|
|
|
|
|
present[row] = true
|
|
|
|
|
}
|
|
|
|
|
for _, row := range g {
|
|
|
|
|
if !present[row] {
|
|
|
|
|
extra = append(extra, row)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if len(missing) > 6 {
|
|
|
|
|
missing = missing[:6]
|
|
|
|
|
}
|
|
|
|
|
if len(extra) > 6 {
|
|
|
|
|
extra = extra[:6]
|
|
|
|
|
}
|
|
|
|
|
return fmt.Sprintf("%s: want %d rows, got %d; missing=%v extra=%v", table, len(w), len(g), missing, extra)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02SequenceNext 是本地号分配的独立期望:发布前序列状态 + 是否已调用。
|
|
|
|
|
func ds02SequenceNext(t *testing.T, fx *writeSetFixture, sequence string) int64 {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var last int64
|
|
|
|
|
var called bool
|
|
|
|
|
if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+sequence).Scan(&last, &called); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if called {
|
|
|
|
|
return last + 1
|
|
|
|
|
}
|
|
|
|
|
return last
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Spec 是夹具声明的来源内容(十表逐列 + 向量 + 对象字节)。
|
|
|
|
|
type ds02Spec struct {
|
|
|
|
|
rows map[string][]map[string]any
|
|
|
|
|
vectors map[string][]float64
|
|
|
|
|
objects map[string]string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ds02SourceSpec(t *testing.T, fx *writeSetFixture) ds02Spec {
|
|
|
|
|
t.Helper()
|
|
|
|
|
tag := func(id string, seq int64) map[string]any {
|
|
|
|
|
return map[string]any{"id": id, "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"name": "name-" + id, "color": "#112233", "sort_order": 3, "seq_id": seq,
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"}
|
|
|
|
|
}
|
|
|
|
|
sourcePath := func(id string) string {
|
|
|
|
|
return wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, id)
|
|
|
|
|
}
|
|
|
|
|
sourceHash := func(id string) string {
|
|
|
|
|
sum := sha256.Sum256([]byte(sourcePath(id)))
|
|
|
|
|
return hex.EncodeToString(sum[:])
|
|
|
|
|
}
|
|
|
|
|
rows := map[string][]map[string]any{
|
|
|
|
|
"knowledges": {
|
|
|
|
|
{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "type": "document",
|
|
|
|
|
"title": "来源文档一", "description": "desc-1", "source": "resource://" + wsHandleSource,
|
|
|
|
|
"parse_status": "completed", "enable_status": "enabled", "embedding_model_id": "source-model",
|
|
|
|
|
"file_name": "one.md", "file_type": "md", "file_size": int64(8), "file_path": "source/one.md",
|
|
|
|
|
"file_hash": strings.Repeat("1", 64), "storage_size": int64(8), "metadata": map[string]any{"k": "v1"},
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z",
|
|
|
|
|
"processed_at": "2026-01-03T00:00:00Z", "summary_status": "none", "channel": "web",
|
|
|
|
|
"pending_subtasks_count": 0},
|
|
|
|
|
{"id": "doc-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "type": "document",
|
|
|
|
|
"title": "来源文档二", "description": "desc-2", "source": "upload",
|
|
|
|
|
"parse_status": "completed", "enable_status": "enabled", "embedding_model_id": "source-model",
|
|
|
|
|
"file_name": "two.md", "file_type": "md", "file_size": int64(4), "file_path": "source/two.md",
|
|
|
|
|
"file_hash": strings.Repeat("2", 64), "storage_size": int64(4), "metadata": map[string]any{"k": "v2"},
|
|
|
|
|
"created_at": "2026-01-04T00:00:00Z", "updated_at": "2026-01-05T00:00:00Z",
|
|
|
|
|
"summary_status": "none", "channel": "web", "pending_subtasks_count": 1},
|
|
|
|
|
},
|
|
|
|
|
"chunks": {
|
|
|
|
|
{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"knowledge_id": "doc-src", "content": "来源正文一 resource://" + wsHandleSource,
|
|
|
|
|
"chunk_index": 0, "is_enabled": true, "start_at": 0, "end_at": 12,
|
|
|
|
|
"next_chunk_id": "chunk-src2", "chunk_type": "text", "status": 1, "flags": 1,
|
|
|
|
|
"seq_id": int64(41), "content_hash": strings.Repeat("3", 64),
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
|
|
|
|
{"id": "chunk-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"knowledge_id": "doc-src", "content": "来源正文二", "chunk_index": 1, "is_enabled": true,
|
|
|
|
|
"start_at": 12, "end_at": 20, "pre_chunk_id": "chunk-src", "chunk_type": "text",
|
|
|
|
|
"status": 1, "flags": 1, "seq_id": int64(42), "content_hash": strings.Repeat("4", 64),
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"knowledge_tags": {tag("tag-src", 11), tag("tag-src2", 12)},
|
|
|
|
|
"knowledge_tag_relations": {
|
|
|
|
|
{"knowledge_id": "doc-src", "tag_id": "tag-src", "created_at": "2026-01-01T00:00:00Z"},
|
|
|
|
|
{"knowledge_id": "doc-src2", "tag_id": "tag-src2", "created_at": "2026-01-03T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"resources": {
|
|
|
|
|
{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
|
|
|
|
|
"storage_backend_id": "backend-source", "provider": "oss", "physical_path": sourcePath("res-src"),
|
|
|
|
|
"location_hash": sourceHash("res-src"), "kind": "file", "mime_type": "image/png",
|
|
|
|
|
"original_name": "a.png", "size": int64(8), "content_hash": strings.Repeat("5", 64),
|
|
|
|
|
"lifecycle": "persistent", "state": "active",
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
|
|
|
|
{"id": "res-src2", "handle": strings.Repeat("g", 22), "tenant_id": wsSourceTenant,
|
|
|
|
|
"storage_backend_id": "backend-source", "provider": "oss", "physical_path": sourcePath("res-src2"),
|
|
|
|
|
"location_hash": sourceHash("res-src2"), "kind": "file", "mime_type": "image/png",
|
|
|
|
|
"original_name": "b.png", "size": int64(8), "content_hash": strings.Repeat("6", 64),
|
|
|
|
|
"lifecycle": "persistent", "state": "active",
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"resource_bindings": {
|
|
|
|
|
{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
|
|
|
|
|
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment",
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z"},
|
|
|
|
|
{"id": "bind-src2", "resource_id": "res-src2", "tenant_id": wsSourceTenant,
|
|
|
|
|
"owner_type": "knowledge", "owner_id": "doc-src2", "relation": "attachment",
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"wiki_folders": {
|
|
|
|
|
{"id": "folder-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"parent_id": "", "name": "根目录", "path": "root", "depth": 0, "sort_order": 1,
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
|
|
|
|
{"id": "folder-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"parent_id": "folder-src", "name": "子目录", "path": "root/sub", "depth": 1, "sort_order": 2,
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"wiki_pages": {
|
|
|
|
|
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index",
|
|
|
|
|
"title": "首页", "page_type": "summary", "status": "published", "content": "来源 wiki 首页",
|
|
|
|
|
"summary": "摘要一", "parent_slug": "", "folder_id": "folder-src", "category_path": []any{"root"},
|
|
|
|
|
"wiki_path": "index", "depth": 0, "sort_order": 1,
|
|
|
|
|
"source_refs": []any{"doc-src|来源文档一"},
|
|
|
|
|
"chunk_refs": []any{"chunk-src"}, "in_links": []any{}, "out_links": []any{"guide"},
|
|
|
|
|
"page_metadata": map[string]any{"lang": "zh"}, "aliases": []any{"home"}, "version": 1,
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
|
|
|
|
{"id": "wiki-src2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "guide",
|
|
|
|
|
"title": "指南", "page_type": "summary", "status": "published", "content": "来源 wiki 指南",
|
|
|
|
|
"summary": "摘要二", "parent_slug": "", "folder_id": "folder-src2", "category_path": []any{"root", "sub"},
|
|
|
|
|
"wiki_path": "root/sub/guide", "depth": 1, "sort_order": 2,
|
|
|
|
|
"source_refs": []any{"doc-src2|来源文档二"},
|
|
|
|
|
"chunk_refs": []any{"chunk-src2"}, "in_links": []any{"guide"}, "out_links": []any{},
|
|
|
|
|
"page_metadata": map[string]any{"lang": "zh"}, "aliases": []any{}, "version": 1,
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-04T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"wiki_page_issues": {
|
|
|
|
|
{"id": "issue-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index",
|
|
|
|
|
"issue_type": "review", "description": "需要复核", "suspected_knowledge_ids": []any{"doc-src"},
|
|
|
|
|
"status": "pending", "reported_by": "fixture",
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
"wiki_log_entries": {
|
|
|
|
|
{"id": int64(1), "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "action": "ingest",
|
|
|
|
|
"knowledge_id": "doc-src", "doc_title": "来源文档一", "summary": "首次入库",
|
|
|
|
|
"pages_affected": []any{map[string]any{"slug": "index", "title": "首页"}},
|
|
|
|
|
"created_at": "2026-01-01T00:00:00Z"},
|
|
|
|
|
{"id": int64(2), "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "action": "update",
|
|
|
|
|
"knowledge_id": "doc-src2", "doc_title": "来源文档二", "summary": "更新",
|
|
|
|
|
"pages_affected": []any{map[string]any{"slug": "guide", "title": "指南"}},
|
|
|
|
|
"created_at": "2026-01-03T00:00:00Z"},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
for table, list := range rows {
|
|
|
|
|
for i, row := range list {
|
|
|
|
|
rows[table][i] = ds02Canon(t, fx, table, row)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ds02Spec{rows: rows, vectors: ds02Vectors, objects: ds02Objects}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Expected 从声明内容独立推导目标范围应有的十表逐列内容。
|
|
|
|
|
func ds02Expected(t *testing.T, fx *writeSetFixture, spec ds02Spec) (map[string][]map[string]any, map[string]string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
targetBackend, targetProvider := "backend-target", "minio"
|
|
|
|
|
locator := func(id string) string {
|
|
|
|
|
// 独立重算定位符:storage://<backend>/<provider>://<bucket>/<tenant>/<object>
|
|
|
|
|
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", targetBackend, targetProvider, fx.bucket, wsTargetTenant, id)
|
|
|
|
|
}
|
|
|
|
|
next := map[string]int64{
|
|
|
|
|
"chunks": ds02SequenceNext(t, fx, "chunks_seq_id_seq"),
|
|
|
|
|
"knowledge_tags": ds02SequenceNext(t, fx, "knowledge_tags_seq_id_seq"),
|
|
|
|
|
"wiki_log_entries": ds02SequenceNext(t, fx, "wiki_log_entries_id_seq"),
|
|
|
|
|
}
|
|
|
|
|
local := map[string]map[int64]int64{"chunks": {}, "knowledge_tags": {}, "wiki_log_entries": {}}
|
|
|
|
|
for _, table := range []string{"chunks", "knowledge_tags", "wiki_log_entries"} {
|
|
|
|
|
sources := []int64{}
|
|
|
|
|
for _, row := range spec.rows[table] {
|
|
|
|
|
raw := row["seq_id"]
|
|
|
|
|
if table == "wiki_log_entries" {
|
|
|
|
|
raw = row["id"]
|
|
|
|
|
}
|
|
|
|
|
value, err := strconv.ParseInt(fmt.Sprint(raw), 10, 64)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("%s local id: %v", table, err)
|
|
|
|
|
}
|
|
|
|
|
sources = append(sources, value)
|
|
|
|
|
}
|
|
|
|
|
sort.Slice(sources, func(i, j int) bool { return sources[i] < sources[j] })
|
|
|
|
|
value := next[table]
|
|
|
|
|
for _, source := range sources {
|
|
|
|
|
local[table][source] = value
|
|
|
|
|
value++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
expected := map[string][]map[string]any{}
|
|
|
|
|
paths := map[string]string{}
|
|
|
|
|
for table, rows := range spec.rows {
|
|
|
|
|
for _, declared := range rows {
|
|
|
|
|
row := map[string]any{}
|
|
|
|
|
for key, value := range declared {
|
|
|
|
|
row[key] = value
|
|
|
|
|
}
|
|
|
|
|
if tableHasTenant(table) {
|
|
|
|
|
row["tenant_id"] = json.Number(strconv.FormatUint(wsTargetTenant, 10))
|
|
|
|
|
}
|
|
|
|
|
if tablesWithKB[table] {
|
|
|
|
|
row["knowledge_base_id"] = wsTargetKB
|
|
|
|
|
}
|
|
|
|
|
switch table {
|
|
|
|
|
case "chunks", "knowledge_tags":
|
|
|
|
|
source, _ := strconv.ParseInt(fmt.Sprint(row["seq_id"]), 10, 64)
|
|
|
|
|
row["seq_id"] = json.Number(strconv.FormatInt(local[table][source], 10))
|
|
|
|
|
case "wiki_log_entries":
|
|
|
|
|
source, _ := strconv.ParseInt(fmt.Sprint(row["id"]), 10, 64)
|
|
|
|
|
row["id"] = json.Number(strconv.FormatInt(local[table][source], 10))
|
|
|
|
|
case "resources":
|
|
|
|
|
id := fmt.Sprint(row["id"])
|
|
|
|
|
expectedPath := locator(id)
|
|
|
|
|
paths[id] = expectedPath
|
|
|
|
|
row["physical_path"] = expectedPath
|
|
|
|
|
sum := sha256.Sum256([]byte(expectedPath))
|
|
|
|
|
row["location_hash"] = hex.EncodeToString(sum[:])
|
|
|
|
|
row["storage_backend_id"] = targetBackend
|
|
|
|
|
row["provider"] = targetProvider
|
|
|
|
|
}
|
|
|
|
|
expected[table] = append(expected[table], ds02Canon(t, fx, table, row))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return expected, paths
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Seed 造目标/其它范围的真实行;目标范围里也放一行"会被整库替换"的数据。
|
|
|
|
|
func ds02Seed(t *testing.T, fx *writeSetFixture) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
put := func(id string, size int) {
|
|
|
|
|
path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id)
|
|
|
|
|
if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(strings.Repeat("z", size)), int64(size)); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
put("res-other", 8)
|
|
|
|
|
for _, row := range []struct {
|
|
|
|
|
table string
|
|
|
|
|
row map[string]any
|
|
|
|
|
}{
|
|
|
|
|
{"knowledges", map[string]any{"id": "doc-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "type": "document", "title": "别的知识库文档", "source": "x"}},
|
|
|
|
|
{"knowledges", map[string]any{"id": "doc-foreign", "tenant_id": 20000, "knowledge_base_id": "kb-foreign", "type": "document", "title": "外部租户文档", "source": "x"}},
|
|
|
|
|
{"chunks", map[string]any{"id": "chunk-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "knowledge_id": "doc-other", "content": "别的知识库正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(7)}},
|
|
|
|
|
{"knowledge_tags", map[string]any{"id": "tag-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库标签", "seq_id": int64(9)}},
|
|
|
|
|
{"knowledge_tag_relations", map[string]any{"knowledge_id": "doc-other", "tag_id": "tag-other"}},
|
|
|
|
|
{"resources", map[string]any{"id": "res-other", "handle": wsHandleOther, "tenant_id": 10001, "provider": "minio", "physical_path": wsLocator("backend-target", "minio", fx.bucket, 10001, "res-other"), "storage_backend_id": "backend-target", "size": int64(8)}},
|
|
|
|
|
{"resource_bindings", map[string]any{"id": "bind-other", "resource_id": "res-other", "tenant_id": 10001, "owner_type": "knowledge", "owner_id": "doc-other", "relation": "attachment"}},
|
|
|
|
|
{"wiki_folders", map[string]any{"id": "folder-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "name": "别的知识库目录"}},
|
|
|
|
|
{"wiki_pages", map[string]any{"id": "wiki-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "slug": "index", "content": "别的知识库 wiki"}},
|
|
|
|
|
{"wiki_page_issues", map[string]any{"id": "issue-other", "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "slug": "index", "issue_type": "review", "reported_by": "fixture", "description": "x"}},
|
|
|
|
|
// 这条是 B03 的碰撞输入:来源日志数字 id=1 与其它知识库已有的 id=1 同值。
|
|
|
|
|
{"wiki_log_entries", map[string]any{"id": int64(1), "tenant_id": 10001, "knowledge_base_id": wsOtherKB, "action": "ingest"}},
|
|
|
|
|
// 目标范围自己的数据:整库发布必须把它们替换成来源内容。
|
|
|
|
|
{"knowledges", map[string]any{"id": "doc-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "type": "document", "title": "目标旧文档", "source": "x"}},
|
|
|
|
|
{"chunks", map[string]any{"id": "chunk-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "knowledge_id": "doc-target", "content": "目标旧正文", "chunk_index": 0, "start_at": 0, "end_at": 1, "seq_id": int64(8)}},
|
|
|
|
|
{"wiki_pages", map[string]any{"id": "wiki-target", "tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB, "slug": "index", "content": "目标旧 wiki"}},
|
|
|
|
|
} {
|
|
|
|
|
r11Insert(t, fx, row.table, row.row)
|
|
|
|
|
}
|
|
|
|
|
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "目标旧正文")
|
|
|
|
|
fx.indexDoc(t, "chunk-foreign", 20000, "kb-foreign", "doc-foreign", "外部正文")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Seal 用声明内容封一份十表全非空快照(索引向量来自夹具声明)。
|
|
|
|
|
func ds02Seal(t *testing.T, fx *writeSetFixture, spec ds02Spec) string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
defer archive.Close()
|
|
|
|
|
encode := func(value any) []byte {
|
|
|
|
|
raw, err := json.Marshal(value)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return raw
|
|
|
|
|
}
|
|
|
|
|
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
|
|
|
|
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
|
|
|
|
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
|
|
|
|
parts := []librarysnapshot.Part{}
|
|
|
|
|
for _, table := range ds02TargetTables {
|
|
|
|
|
part, err := archive.Put(bytes.NewReader(encode(spec.rows[table])))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
part.Kind, part.Name, part.Count = kinds[table], table, int64(len(spec.rows[table]))
|
|
|
|
|
parts = append(parts, part)
|
|
|
|
|
}
|
|
|
|
|
for _, id := range []string{"res-src", "res-src2"} {
|
|
|
|
|
part, err := archive.Put(bytes.NewReader([]byte(spec.objects[id])))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
part.Kind, part.Name, part.Count = "resource", id, 1
|
|
|
|
|
parts = append(parts, part)
|
|
|
|
|
}
|
|
|
|
|
indexRows := []map[string]any{}
|
|
|
|
|
for _, chunk := range spec.rows["chunks"] {
|
|
|
|
|
id := fmt.Sprint(chunk["id"])
|
|
|
|
|
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
|
|
|
|
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
|
|
|
|
"knowledge_id": chunk["knowledge_id"], "chunk_id": id, "source_id": id,
|
|
|
|
|
"content": chunk["content"], "embedding": spec.vectors[id]}})
|
|
|
|
|
}
|
|
|
|
|
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
|
|
|
|
parts = append(parts, indexPart)
|
|
|
|
|
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
|
|
|
|
Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB,
|
|
|
|
|
CreatedAt: time.Now().UTC(), Parts: parts,
|
|
|
|
|
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
|
|
|
|
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
|
|
|
|
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
|
|
|
|
id, err := archive.Seal(manifest)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("seal ds02 snapshot: %v", err)
|
|
|
|
|
}
|
|
|
|
|
return id
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02Docs 读回索引里的全部文档(独立查询,不经过被测 SearchExporter)。
|
|
|
|
|
func ds02Docs(t *testing.T, fx *writeSetFixture) map[string]map[string]any {
|
|
|
|
|
t.Helper()
|
|
|
|
|
body := `{"size":1000,"query":{"match_all":{}}}`
|
|
|
|
|
request, err := http.NewRequestWithContext(fx.ctx, "POST", fx.searchBase+"/"+fx.index+"/_search", strings.NewReader(body))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request.Header.Set("Content-Type", "application/json")
|
|
|
|
|
response, err := fx.client.Do(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
defer response.Body.Close()
|
|
|
|
|
if response.StatusCode != 200 {
|
|
|
|
|
raw, _ := io.ReadAll(response.Body)
|
|
|
|
|
t.Fatalf("search %d: %s", response.StatusCode, raw)
|
|
|
|
|
}
|
|
|
|
|
var payload struct {
|
|
|
|
|
Hits struct {
|
|
|
|
|
Hits []struct {
|
|
|
|
|
ID string `json:"_id"`
|
|
|
|
|
Source map[string]any `json:"_source"`
|
|
|
|
|
} `json:"hits"`
|
|
|
|
|
} `json:"hits"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
out := map[string]map[string]any{}
|
|
|
|
|
for _, hit := range payload.Hits.Hits {
|
|
|
|
|
out[hit.ID] = hit.Source
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ds02CountingClient(t *testing.T, fx *writeSetFixture) *[]string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
seen := &[]string{}
|
|
|
|
|
base := http.DefaultTransport
|
|
|
|
|
fx.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
|
|
|
|
*seen = append(*seen, request.Method+" "+request.URL.String())
|
|
|
|
|
return base.RoundTrip(request)
|
|
|
|
|
})
|
|
|
|
|
t.Cleanup(func() { fx.client.Transport = nil })
|
|
|
|
|
return seen
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
|
|
|
|
|
|
|
|
|
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
|
|
|
|
|
|
|
|
|
|
// ds02Check 是 oracle 本体:返回逐项差异;空 = 全绿。
|
|
|
|
|
func ds02Check(t *testing.T, fx *writeSetFixture, spec ds02Spec, expected map[string][]map[string]any,
|
|
|
|
|
paths map[string]string, preserved map[string][]map[string]any, requests *[]string) []string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var diffs []string
|
|
|
|
|
scope := func(table string) (string, []any) {
|
|
|
|
|
if table == "knowledge_tag_relations" {
|
|
|
|
|
return "SELECT to_jsonb(x)::text FROM " + table + " AS x", nil
|
|
|
|
|
}
|
|
|
|
|
if tableHasTenant(table) && tablesWithKB[table] {
|
|
|
|
|
return "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id=$1 AND knowledge_base_id=$2 ORDER BY 1",
|
|
|
|
|
[]any{wsTargetTenant, wsTargetKB}
|
|
|
|
|
}
|
|
|
|
|
return "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id=$1 ORDER BY 1", []any{wsTargetTenant}
|
|
|
|
|
}
|
|
|
|
|
for _, table := range ds02TargetTables {
|
|
|
|
|
query, args := scope(table)
|
|
|
|
|
if diff := ds02Compare(table, expected[table], ds02Read(t, fx, query, args...)); diff != "" {
|
|
|
|
|
diffs = append(diffs, diff)
|
|
|
|
|
}
|
|
|
|
|
// 目标范围之外的行必须一字不动(含其它知识库/租户以及它们的历史资源)。
|
|
|
|
|
if table == "knowledge_tag_relations" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
other := "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE tenant_id<>$1"
|
|
|
|
|
if tableHasTenant(table) && tablesWithKB[table] {
|
|
|
|
|
other = "SELECT to_jsonb(x)::text FROM " + table + " AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)"
|
|
|
|
|
}
|
|
|
|
|
if diff := ds02Compare(table+"(non-target)", preserved[table], ds02Read(t, fx, other, args...)); diff != "" {
|
|
|
|
|
diffs = append(diffs, diff)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// 引用:每条关系都必须有真实目标行,而且断言确实覆盖到引用(匹配数 > 0)。
|
|
|
|
|
relations := []struct {
|
|
|
|
|
name string
|
|
|
|
|
sql string
|
|
|
|
|
matched string
|
|
|
|
|
}{
|
|
|
|
|
{"chunk→knowledge", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`,
|
|
|
|
|
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id)`},
|
|
|
|
|
{"chunk→chunk links", `SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND ((c.pre_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks p WHERE p.id=c.pre_chunk_id)) OR (c.next_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks n WHERE n.id=c.next_chunk_id)) OR (c.parent_chunk_id IS NOT NULL AND c.parent_chunk_id<>'' AND NOT EXISTS(SELECT 1 FROM chunks q WHERE q.id=c.parent_chunk_id)))`,
|
|
|
|
|
`SELECT count(*) FROM chunks c WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND (c.pre_chunk_id<>'' OR c.next_chunk_id<>'' OR coalesce(c.parent_chunk_id,'')<>'')`},
|
|
|
|
|
{"relation→tag/knowledge", `SELECT count(*) FROM knowledge_tag_relations r WHERE NOT EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) OR NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`,
|
|
|
|
|
`SELECT count(*) FROM knowledge_tag_relations r WHERE EXISTS(SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=r.knowledge_id)`},
|
|
|
|
|
{"binding→resource/knowledge", `SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND (NOT EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id) OR (b.owner_type='knowledge' AND NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id)))`,
|
|
|
|
|
`SELECT count(*) FROM resource_bindings b WHERE b.tenant_id=$1 AND EXISTS(SELECT 1 FROM resources r WHERE r.id=b.resource_id)`},
|
|
|
|
|
{"wiki page→folder", `SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>'' AND NOT EXISTS(SELECT 1 FROM wiki_folders f WHERE f.id=p.folder_id)`,
|
|
|
|
|
`SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=$1 AND p.knowledge_base_id=$2 AND p.folder_id<>''`},
|
|
|
|
|
{"wiki issue→page", `SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2 AND NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=i.slug AND p.tenant_id=i.tenant_id AND p.knowledge_base_id=i.knowledge_base_id)`,
|
|
|
|
|
`SELECT count(*) FROM wiki_page_issues i WHERE i.tenant_id=$1 AND i.knowledge_base_id=$2`},
|
|
|
|
|
{"wiki log→knowledge/page", `SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2 AND (NOT EXISTS(SELECT 1 FROM knowledges k WHERE k.id=l.knowledge_id) OR EXISTS(SELECT 1 FROM jsonb_array_elements(l.pages_affected) e WHERE NOT EXISTS(SELECT 1 FROM wiki_pages p WHERE p.slug=e->>'slug' AND p.tenant_id=l.tenant_id AND p.knowledge_base_id=l.knowledge_base_id)))`,
|
|
|
|
|
`SELECT count(*) FROM wiki_log_entries l WHERE l.tenant_id=$1 AND l.knowledge_base_id=$2`},
|
|
|
|
|
}
|
|
|
|
|
for _, relation := range relations {
|
|
|
|
|
var violations, matched int64
|
|
|
|
|
var args []any
|
|
|
|
|
if strings.Contains(relation.sql, "$2") {
|
|
|
|
|
args = []any{wsTargetTenant, wsTargetKB}
|
|
|
|
|
} else if strings.Contains(relation.sql, "$1") {
|
|
|
|
|
args = []any{wsTargetTenant}
|
|
|
|
|
}
|
|
|
|
|
if err := fx.pool.QueryRow(fx.ctx, relation.sql, args...).Scan(&violations); err != nil {
|
|
|
|
|
t.Fatalf("reference %s: %v", relation.name, err)
|
|
|
|
|
}
|
|
|
|
|
if err := fx.pool.QueryRow(fx.ctx, relation.matched, args...).Scan(&matched); err != nil {
|
|
|
|
|
t.Fatalf("reference coverage %s: %v", relation.name, err)
|
|
|
|
|
}
|
|
|
|
|
if violations != 0 {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("reference %s: %d violations", relation.name, violations))
|
|
|
|
|
}
|
|
|
|
|
if matched == 0 {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("reference %s: oracle matched no rows", relation.name))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// 向量/正文:索引文档逐条与声明完全一致(多一条少一条都算差异)。
|
|
|
|
|
docs := ds02Docs(t, fx)
|
|
|
|
|
if len(docs) != len(spec.rows["chunks"])+1 { // +1:目标范围之外的 chunk-foreign
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("index docs: want %d got %d", len(spec.rows["chunks"])+1, len(docs)))
|
|
|
|
|
}
|
|
|
|
|
for _, chunk := range spec.rows["chunks"] {
|
|
|
|
|
id := fmt.Sprint(chunk["id"])
|
|
|
|
|
doc, ok := docs[id]
|
|
|
|
|
if !ok {
|
|
|
|
|
diffs = append(diffs, "index doc missing: "+id)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
source := doc
|
|
|
|
|
want := map[string]any{"tenant_id": json.Number(strconv.FormatUint(wsTargetTenant, 10)),
|
|
|
|
|
"knowledge_base_id": wsTargetKB, "knowledge_id": chunk["knowledge_id"], "chunk_id": id,
|
|
|
|
|
"source_id": id, "content": chunk["content"], "embedding": spec.vectors[id]}
|
|
|
|
|
for key, value := range want {
|
|
|
|
|
if fmt.Sprint(source[key]) != fmt.Sprint(value) {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("index doc %s.%s: want %v got %v", id, key, value, source[key]))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if foreign, ok := docs["chunk-foreign"]; !ok {
|
|
|
|
|
diffs = append(diffs, "foreign index doc disappeared")
|
|
|
|
|
} else if fmt.Sprint(foreign["content"]) != "外部正文" {
|
|
|
|
|
diffs = append(diffs, "foreign index doc changed")
|
|
|
|
|
}
|
|
|
|
|
// 对象字节:按独立重算的定位符读回,逐字节比。
|
|
|
|
|
for id, want := range spec.objects {
|
|
|
|
|
reader, err := fx.objects.s3.Open(fx.ctx, "minio", paths[id])
|
|
|
|
|
if err != nil {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("object %s: %v", id, err))
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
got, err := io.ReadAll(reader)
|
|
|
|
|
reader.Close()
|
|
|
|
|
if err != nil {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("object %s read: %v", id, err))
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if string(got) != want {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("object %s bytes: want %q got %q", id, want, got))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Wiki 日志映射:内容元组多重集双射(本地号不参与身份,来源 id 只用于覆盖检查)。
|
|
|
|
|
wantTuples := []string{}
|
|
|
|
|
for _, row := range spec.rows["wiki_log_entries"] {
|
|
|
|
|
copied := map[string]any{}
|
|
|
|
|
for key, value := range row {
|
|
|
|
|
if key == "id" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
copied[key] = value
|
|
|
|
|
}
|
|
|
|
|
copied["tenant_id"] = json.Number(strconv.FormatUint(wsTargetTenant, 10))
|
|
|
|
|
copied["knowledge_base_id"] = wsTargetKB
|
|
|
|
|
raw, _ := json.Marshal(copied)
|
|
|
|
|
wantTuples = append(wantTuples, string(raw))
|
|
|
|
|
}
|
|
|
|
|
gotTuples := []string{}
|
|
|
|
|
for _, row := range expected["wiki_log_entries"] {
|
|
|
|
|
_ = row
|
|
|
|
|
}
|
|
|
|
|
actualLogs := ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM wiki_log_entries AS x WHERE tenant_id=$1 AND knowledge_base_id=$2`, wsTargetTenant, wsTargetKB)
|
|
|
|
|
for _, row := range actualLogs {
|
|
|
|
|
copied := map[string]any{}
|
|
|
|
|
for key, value := range row {
|
|
|
|
|
if key == "id" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
copied[key] = value
|
|
|
|
|
}
|
|
|
|
|
raw, _ := json.Marshal(copied)
|
|
|
|
|
gotTuples = append(gotTuples, string(raw))
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(wantTuples)
|
|
|
|
|
sort.Strings(gotTuples)
|
|
|
|
|
if fmt.Sprint(wantTuples) != fmt.Sprint(gotTuples) {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("wiki log tuple bijection broken: want %v got %v", wantTuples, gotTuples))
|
|
|
|
|
}
|
|
|
|
|
// Embedding 调用必须为 0,且请求确实发生过(不是"一个请求都没有"的空绿)。
|
|
|
|
|
embedding := 0
|
|
|
|
|
for _, call := range *requests {
|
|
|
|
|
if strings.Contains(call, "embedding") || strings.Contains(call, "dashscope") || strings.Contains(call, "openai") {
|
|
|
|
|
embedding++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if embedding != 0 {
|
|
|
|
|
diffs = append(diffs, fmt.Sprintf("embedding calls: want 0 got %d (%v)", embedding, *requests))
|
|
|
|
|
}
|
|
|
|
|
if len(*requests) == 0 {
|
|
|
|
|
diffs = append(diffs, "no HTTP requests observed; zero-embedding oracle would be vacuous")
|
|
|
|
|
}
|
|
|
|
|
return diffs
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ds02OtherRows 读目标范围之外的行(发布前抓一次,发布后要求一字不差)。
|
|
|
|
|
func ds02OtherRows(t *testing.T, fx *writeSetFixture, table string) []map[string]any {
|
|
|
|
|
t.Helper()
|
|
|
|
|
if table == "knowledge_tag_relations" {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
if tableHasTenant(table) && tablesWithKB[table] {
|
|
|
|
|
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE NOT (tenant_id=$1 AND knowledge_base_id=$2)", wsTargetTenant, wsTargetKB)
|
|
|
|
|
}
|
|
|
|
|
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id<>$1", wsTargetTenant)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ds02SameTenantRows(t *testing.T, fx *writeSetFixture, table string) []map[string]any {
|
|
|
|
|
t.Helper()
|
|
|
|
|
return ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM "+table+" AS x WHERE tenant_id=$1", wsTargetTenant)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestDS02FullShapeOracleIsolated(t *testing.T) {
|
|
|
|
|
fx := openWriteSetFixture(t)
|
|
|
|
|
// 用保留下来的真实 v071 catalog 重建十张表:逐列、外键、序列都是真实形状。
|
|
|
|
|
r11Mirror(t, fx)
|
|
|
|
|
for _, sql := range []string{
|
|
|
|
|
`SELECT setval('chunks_seq_id_seq',100000005,false)`,
|
|
|
|
|
`SELECT setval('knowledge_tags_seq_id_seq',50,false)`,
|
|
|
|
|
`SELECT setval('wiki_log_entries_id_seq',10,false)`,
|
|
|
|
|
} {
|
|
|
|
|
r10Exec(t, fx, sql)
|
|
|
|
|
}
|
|
|
|
|
ds02Seed(t, fx)
|
|
|
|
|
preserved := map[string][]map[string]any{}
|
|
|
|
|
for _, table := range ds02TargetTables {
|
|
|
|
|
preserved[table] = ds02OtherRows(t, fx, table)
|
|
|
|
|
}
|
|
|
|
|
spec := ds02SourceSpec(t, fx)
|
|
|
|
|
expected, paths := ds02Expected(t, fx, spec)
|
|
|
|
|
for _, table := range []string{"resources", "resource_bindings"} {
|
|
|
|
|
expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...)
|
|
|
|
|
}
|
|
|
|
|
// knowledge_tag_relations 没有 tenant/kb:产品按父文档归属删除,非目标范围的
|
|
|
|
|
// 关系行必须原样保留,所以它们也在期望里。
|
|
|
|
|
expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"],
|
|
|
|
|
ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x
|
|
|
|
|
WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`,
|
|
|
|
|
wsTargetTenant, wsTargetKB)...)
|
|
|
|
|
snapshot := ds02Seal(t, fx, spec)
|
|
|
|
|
requests := ds02CountingClient(t, fx)
|
|
|
|
|
target := fx.target(t)
|
|
|
|
|
backups := fx.backups(t, target)
|
|
|
|
|
out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil)
|
|
|
|
|
r10Save(t, "outcome", out)
|
|
|
|
|
if out.State != "succeeded" {
|
|
|
|
|
t.Fatalf("full-shape publish must succeed: %+v", out)
|
|
|
|
|
}
|
|
|
|
|
// 写集合里的本地号映射必须完整、双射,并且目标行真的用了这些号。
|
|
|
|
|
if out.Backup.WriteSet == nil {
|
|
|
|
|
t.Fatal("write set missing")
|
|
|
|
|
}
|
|
|
|
|
ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("load write set: %v", err)
|
|
|
|
|
}
|
|
|
|
|
for _, table := range []string{"chunks", "knowledge_tags", "wiki_log_entries"} {
|
|
|
|
|
column := "seq_id"
|
|
|
|
|
if table == "wiki_log_entries" {
|
|
|
|
|
column = "id"
|
|
|
|
|
}
|
|
|
|
|
sources := map[int64]bool{}
|
|
|
|
|
targets := map[int64]bool{}
|
|
|
|
|
for _, mapping := range ledger.LocalIDs {
|
|
|
|
|
if mapping.Table != table {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if mapping.Column != column {
|
|
|
|
|
t.Fatalf("%s mapping column = %s", table, mapping.Column)
|
|
|
|
|
}
|
|
|
|
|
source, _ := strconv.ParseInt(mapping.Source, 10, 64)
|
|
|
|
|
targetID, _ := strconv.ParseInt(mapping.Target, 10, 64)
|
|
|
|
|
if sources[source] || targets[targetID] {
|
|
|
|
|
t.Fatalf("%s mapping not bijective: source=%d target=%d", table, source, targetID)
|
|
|
|
|
}
|
|
|
|
|
sources[source] = true
|
|
|
|
|
targets[targetID] = true
|
|
|
|
|
}
|
|
|
|
|
if len(sources) != len(spec.rows[table]) {
|
|
|
|
|
t.Fatalf("%s mapping covers %d source rows, want %d", table, len(sources), len(spec.rows[table]))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
diffs := ds02Check(t, fx, spec, expected, paths, preserved, requests)
|
|
|
|
|
r10Save(t, "green-diffs", diffs)
|
|
|
|
|
if len(diffs) != 0 {
|
|
|
|
|
t.Fatalf("independent oracle disagrees on a correct publish:\n%s", strings.Join(diffs, "\n"))
|
|
|
|
|
}
|
|
|
|
|
r10Save(t, "green-requests", *requests)
|
|
|
|
|
|
|
|
|
|
// 变异自检:对真实目标改正文 / 向量 / 引用 / 日志映射,oracle 必须红;
|
|
|
|
|
// 改回后必须绿。这样每条断言都证明了自己能抓到它负责的东西。
|
|
|
|
|
content := fmt.Sprint(expected["chunks"][0]["content"])
|
|
|
|
|
mutations := []struct {
|
|
|
|
|
name string
|
|
|
|
|
marker string
|
|
|
|
|
mutate func()
|
|
|
|
|
restore func()
|
|
|
|
|
}{
|
|
|
|
|
{"content", "chunks:", func() {
|
|
|
|
|
r10Exec(t, fx, `UPDATE chunks SET content=content||'!MUT' WHERE id='chunk-src'`)
|
|
|
|
|
}, func() {
|
|
|
|
|
if _, err := fx.pool.Exec(fx.ctx, `UPDATE chunks SET content=$1 WHERE id='chunk-src'`, content); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}},
|
|
|
|
|
{"vector", "index doc chunk-src.embedding", func() {
|
|
|
|
|
body := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[9.5,9.5]}`
|
|
|
|
|
request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/chunk-src?refresh=true", strings.NewReader(body))
|
|
|
|
|
request.Header.Set("Content-Type", "application/json")
|
|
|
|
|
response, err := fx.client.Do(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response.Body.Close()
|
|
|
|
|
if response.StatusCode >= 300 {
|
|
|
|
|
t.Fatalf("mutate vector: %d", response.StatusCode)
|
|
|
|
|
}
|
|
|
|
|
}, func() {
|
|
|
|
|
body := `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-src","chunk_id":"chunk-src","source_id":"chunk-src","content":"来源正文一 resource://` + wsHandleSource + `","embedding":[0.5,1.5]}`
|
|
|
|
|
request, _ := http.NewRequestWithContext(fx.ctx, "PUT", fx.searchBase+"/"+fx.index+"/_doc/chunk-src?refresh=true", strings.NewReader(body))
|
|
|
|
|
request.Header.Set("Content-Type", "application/json")
|
|
|
|
|
response, err := fx.client.Do(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response.Body.Close()
|
|
|
|
|
if response.StatusCode >= 300 {
|
|
|
|
|
t.Fatalf("restore vector: %d", response.StatusCode)
|
|
|
|
|
}
|
|
|
|
|
}},
|
|
|
|
|
{"reference", "reference relation", func() {
|
|
|
|
|
r10Exec(t, fx, `UPDATE knowledge_tag_relations SET tag_id='missing-tag' WHERE knowledge_id='doc-src' AND tag_id='tag-src'`)
|
|
|
|
|
}, func() {
|
|
|
|
|
r10Exec(t, fx, `UPDATE knowledge_tag_relations SET tag_id='tag-src' WHERE knowledge_id='doc-src' AND tag_id='missing-tag'`)
|
|
|
|
|
}},
|
|
|
|
|
{"mapping", "wiki log", func() {
|
|
|
|
|
r10Exec(t, fx, `UPDATE wiki_log_entries SET action='tampered' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id='doc-src'`)
|
|
|
|
|
}, func() {
|
|
|
|
|
r10Exec(t, fx, `UPDATE wiki_log_entries SET action='ingest' WHERE tenant_id=10001 AND knowledge_base_id='kb-target' AND knowledge_id='doc-src'`)
|
|
|
|
|
}},
|
|
|
|
|
}
|
|
|
|
|
for _, mutation := range mutations {
|
|
|
|
|
mutation.mutate()
|
|
|
|
|
red := ds02Check(t, fx, spec, expected, paths, preserved, requests)
|
|
|
|
|
r10Save(t, "red-"+mutation.name, red)
|
|
|
|
|
if len(red) == 0 {
|
|
|
|
|
t.Fatalf("%s mutation: oracle stayed green", mutation.name)
|
|
|
|
|
}
|
|
|
|
|
joined := strings.Join(red, "\n")
|
|
|
|
|
if !strings.Contains(joined, mutation.marker) {
|
|
|
|
|
t.Fatalf("%s mutation: oracle failed for an unrelated reason:\n%s", mutation.name, joined)
|
|
|
|
|
}
|
|
|
|
|
mutation.restore()
|
|
|
|
|
green := ds02Check(t, fx, spec, expected, paths, preserved, requests)
|
|
|
|
|
if len(green) != 0 {
|
|
|
|
|
t.Fatalf("%s mutation not restored cleanly:\n%s", mutation.name, strings.Join(green, "\n"))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|