跨租户 bundle 的旧侧必须在 wiki 之前省略 resources/resource_bindings,
不能据此声称"同输入已收口"。新增同租户、不同知识库场景:旧 6fe98c9 的
resources 阶段只按 tenant_id 查资源行,同租户下不再提前失败,因此两侧
都能消费完整十表归档。
新增 ds02_identical_input_test.go:导出十表 bundle;两侧在执行前各自导出
逐表来源归档、实际目标种子、完整 DDL、三条序列、ES 文档与对象字节;
唯一 namespace 替换(桶名 %BUCKET%)按同一规范归一,物理身份单列。
This commit is contained in:
+8
-2
@@ -18,8 +18,14 @@
|
||||
- 新增冻结输入 bundle(十表来源行、目标种子、序列种子、ES 文档、对象字节、完整 DDL,
|
||||
时间固定、逐表 hash):旧源码 `6fe98c9` 与当前源码读取同一份文件、同一 sha256。旧侧真实
|
||||
复现 `wiki_log_entries_pkey / SQLSTATE 23505`,新侧同一输入发布成功并通过独立 oracle。
|
||||
差异单列:6fe98c9 在 wiki 之前无法跨租户搬运资源,旧侧归档省略 `resources` /
|
||||
`resource_bindings` 两表,其余八表与碰撞行逐字节相同;新侧按十表完整运行。
|
||||
该跨租户 bundle 是**受约束回归对照**:6fe98c9 在 wiki 之前无法跨租户搬运资源,旧侧归档
|
||||
只能省略 `resources` / `resource_bindings` 两表,因此不能据此声称"同输入已收口"。
|
||||
- 新增十表同输入对照(同租户、不同知识库):`ds02-identical-input-bundle.json` 两侧各自
|
||||
规范化并封存完整十表来源归档(含 `resources` / `resource_bindings`),旧侧 `6fe98c9` 用
|
||||
真实 PG/ES/MinIO 对象适配复现 `wiki_log_entries_pkey / SQLSTATE 23505`,新侧同输入
|
||||
`succeeded` 并通过映射/内容 oracle。两侧在执行前各自动导出逐表来源归档、实际目标种子、
|
||||
完整 DDL、三条序列、ES 文档与对象字节;唯一 namespace 替换(对象桶名 `%BUCKET%`)
|
||||
按同一规范归一后语义输入 hash 相同,物理桶名单独记录。
|
||||
- 新增测试进程身份断言:runner 注入 test-only nonce/commit/源码树摘要,测试进程自报
|
||||
`os.Executable` 的 sha256,与 runner 期望逐字符比较;构建与执行前后各算一次二进制哈希。
|
||||
- 新增隔离运行器 v2(`ds02_runner_v2.py`,v1 保留):超时/构建/准备异常都走 finally 清理,
|
||||
|
||||
@@ -0,0 +1,735 @@
|
||||
package v071
|
||||
|
||||
// DS-02 十表同输入对照(同租户、不同知识库)。
|
||||
//
|
||||
// 背景:跨租户冻结 bundle 的旧侧 6fe98c9 在 database 阶段不改写租户、resources 阶段
|
||||
// 却按目标租户查资源行,跨租户资源必然在 wiki 之前失败。旧侧归档因此省略了
|
||||
// resources / resource_bindings 两表 —— 那不是"同一输入",只能算受约束的回归对照。
|
||||
//
|
||||
// 本文件把场景改成"来源与目标同租户、不同知识库":
|
||||
// * 旧侧的资源查询只按 tenant_id,同租户下能在 resources 阶段找到刚插入的资源行,
|
||||
// 前置缺陷不再触发;
|
||||
// * wiki_log_entries 的数字 id 仍在同一租户的其它知识库里占用了 id=1,
|
||||
// 旧代码按归档原样插入时仍会撞 wiki_log_entries_pkey / 23505。
|
||||
//
|
||||
// 因此两侧都能消费"真正相同的十表输入":来源归档十表逐表内容、目标种子、完整 DDL、
|
||||
// 三条序列、ES 文档与对象字节。本文件负责生成新的 bundle 与执行前导出"实际消费输入"
|
||||
// 证据;旧侧探针(旧 SHA 工作树内未跟踪测试文件)导出同一形状的证据,再由证据根脚本
|
||||
// 逐项比较 —— 不是"读同一个文件再各自过滤"。
|
||||
//
|
||||
// 导出(不连接任何服务):
|
||||
// KG_DS02_IDENTICAL_BUNDLE_OUT=/path/dir go test -run TestDS02ExportIdenticalInputBundle
|
||||
// 新侧执行:
|
||||
// KG_DS02_BUNDLE=/path/dir/ds02-identical-input-bundle.json go test -run TestDS02IdenticalInputBundleIsolated
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// 时间固定:bundle 声明、归档 manifest、发布 request 都用这一个时刻。
|
||||
const ds02IdenticalTime = "2026-11-02T00:00:00Z"
|
||||
|
||||
func ds02IdenticalScope() map[string]any {
|
||||
return map[string]any{
|
||||
"source_tenant": uint64(wsTargetTenant),
|
||||
"source_kb": wsSourceKB,
|
||||
"target_tenant": uint64(wsTargetTenant),
|
||||
"target_kb": wsTargetKB,
|
||||
"other_kb": wsOtherKB,
|
||||
"foreign_tenant": uint64(wsForeignTnt),
|
||||
}
|
||||
}
|
||||
|
||||
// ds02IdenticalSourceRows 是十表来源行的纯声明:租户与目标相同,知识库不同。
|
||||
// 资源定位符里的租户段随租户一起改写(bucket 段保持固定的来源桶名,不引用两侧
|
||||
// 一次性桶,否则归档内容会随环境变化);location_hash 按真实定位符重算。
|
||||
func ds02IdenticalSourceRows() map[string][]map[string]any {
|
||||
rows := ds02SourceRows()
|
||||
for table, list := range rows {
|
||||
for _, row := range list {
|
||||
if _, ok := row["tenant_id"]; ok {
|
||||
row["tenant_id"] = uint64(wsTargetTenant)
|
||||
}
|
||||
if table != "resources" {
|
||||
continue
|
||||
}
|
||||
id := fmt.Sprint(row["id"])
|
||||
path := wsLocator("backend-source", "oss", "source-bucket", wsTargetTenant, id)
|
||||
row["physical_path"] = path
|
||||
sum := sha256.Sum256([]byte(path))
|
||||
row["location_hash"] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// ds02IdenticalSeedRows 给目标种子补上显式时间戳:种子行是"发布前目标已有数据",
|
||||
// 两侧都必须逐列相等;让 now()/CURRENT_TIMESTAMP 在两侧落地成不同值会让"实际目标种子
|
||||
// 相同"这条断言无法成立。DDL 与 catalog 本身不改。
|
||||
func ds02IdenticalSeedRows() []struct {
|
||||
table string
|
||||
row map[string]any
|
||||
} {
|
||||
timestamps := map[string][]string{
|
||||
"knowledges": {"created_at", "updated_at"},
|
||||
"chunks": {"created_at", "updated_at"},
|
||||
"knowledge_tags": {"created_at", "updated_at"},
|
||||
"knowledge_tag_relations": {"created_at"},
|
||||
"resources": {"created_at", "updated_at"},
|
||||
"resource_bindings": {"created_at"},
|
||||
"wiki_folders": {"created_at", "updated_at"},
|
||||
"wiki_pages": {"created_at", "updated_at"},
|
||||
"wiki_page_issues": {"created_at", "updated_at"},
|
||||
"wiki_log_entries": {"created_at"},
|
||||
}
|
||||
fixed := map[string]string{"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"}
|
||||
seeds := ds02SeedRows()
|
||||
out := make([]struct {
|
||||
table string
|
||||
row map[string]any
|
||||
}, 0, len(seeds))
|
||||
for _, seed := range seeds {
|
||||
row := map[string]any{}
|
||||
for key, value := range seed.row {
|
||||
row[key] = value
|
||||
}
|
||||
for _, column := range timestamps[seed.table] {
|
||||
if _, ok := row[column]; ok {
|
||||
continue
|
||||
}
|
||||
row[column] = fixed[column]
|
||||
}
|
||||
out = append(out, struct {
|
||||
table string
|
||||
row map[string]any
|
||||
}{table: seed.table, row: row})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ds02IdenticalDDL 用同一份真实 catalog 生成完整 DDL,只把来源知识库的租户改成与目标
|
||||
// 相同的值(bundle 的 scope 声明必须与 DDL 里的行一致)。其他语句逐字节相同。
|
||||
func ds02IdenticalDDL(t *testing.T) []string {
|
||||
t.Helper()
|
||||
ddl := ds02BundleDDL(t)
|
||||
old := "('" + wsSourceKB + "'," + strconv.Itoa(wsSourceTenant)
|
||||
replacement := "('" + wsSourceKB + "'," + strconv.Itoa(wsTargetTenant)
|
||||
replaced := 0
|
||||
out := make([]string, 0, len(ddl))
|
||||
for _, statement := range ddl {
|
||||
if strings.Contains(statement, old) {
|
||||
statement = strings.Replace(statement, old, replacement, 1)
|
||||
replaced++
|
||||
}
|
||||
out = append(out, statement)
|
||||
}
|
||||
if replaced != 1 {
|
||||
t.Fatalf("source knowledge base row in DDL: want exactly 1 statement, replaced %d", replaced)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDS02ExportIdenticalInputBundle(t *testing.T) {
|
||||
root := os.Getenv("KG_DS02_IDENTICAL_BUNDLE_OUT")
|
||||
if root == "" {
|
||||
t.Skip("set KG_DS02_IDENTICAL_BUNDLE_OUT to write the identical-input bundle")
|
||||
}
|
||||
rows := ds02IdenticalSourceRows()
|
||||
perTable := map[string]string{}
|
||||
for table, list := range rows {
|
||||
raw, err := json.Marshal(list)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
perTable[table] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
objects := map[string]string{}
|
||||
for id, body := range ds02Objects {
|
||||
objects[id] = body
|
||||
}
|
||||
for id, body := range ds02SeedObjects() {
|
||||
objects[id] = body
|
||||
}
|
||||
bundle := ds02FrozenBundle{
|
||||
Bundle: "ds02-identical-input",
|
||||
Version: 2,
|
||||
CreatedAt: ds02IdenticalTime,
|
||||
Scope: ds02IdenticalScope(),
|
||||
Sequences: ds02BundleSequences(),
|
||||
SourceRows: rows,
|
||||
SeedObjects: objects,
|
||||
Vectors: ds02Vectors,
|
||||
DDL: ds02IdenticalDDL(t),
|
||||
}
|
||||
for _, seed := range ds02IdenticalSeedRows() {
|
||||
bundle.TargetRows = append(bundle.TargetRows, ds02BundleSeedRow{Table: seed.table, Row: seed.row})
|
||||
}
|
||||
for _, doc := range ds02SeedDocs() {
|
||||
bundle.SeedDocs = append(bundle.SeedDocs, ds02BundleSeedDoc{ID: doc.id, Tenant: doc.tenant, KB: doc.kb, Knowledge: doc.knowledge, Content: doc.content})
|
||||
}
|
||||
raw, err := json.MarshalIndent(bundle, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "ds02-identical-input-bundle.json")
|
||||
if err := os.WriteFile(path, append(raw, '\n'), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(append(raw, '\n'))
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
if err := os.WriteFile(filepath.Join(root, "ds02-identical-input-bundle.sha256"), []byte(hash+" ds02-identical-input-bundle.json\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := map[string]any{"bundle": path, "sha256": hash, "table_hashes": perTable,
|
||||
"bytes": len(raw) + 1, "ddl_statements": len(bundle.DDL), "created_at": bundle.CreatedAt,
|
||||
"scope": bundle.Scope}
|
||||
if run := os.Getenv("KG_R10_RUN"); run != "" {
|
||||
r10Save(t, "identical-bundle", result)
|
||||
}
|
||||
encoded, _ := json.MarshalIndent(result, "", " ")
|
||||
t.Logf("%s", encoded)
|
||||
}
|
||||
|
||||
func ds02LoadIdenticalBundle(t *testing.T) (ds02FrozenBundle, string) {
|
||||
t.Helper()
|
||||
path := os.Getenv("KG_DS02_BUNDLE")
|
||||
if path == "" {
|
||||
t.Skip("set KG_DS02_BUNDLE to run the identical-input comparison")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read identical bundle: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
sidecar, err := os.ReadFile(strings.TrimSuffix(path, ".json") + ".sha256")
|
||||
if err != nil {
|
||||
t.Fatalf("read identical bundle sha256 sidecar: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(strings.TrimSpace(string(sidecar)), hash) {
|
||||
t.Fatalf("identical bundle sha256 %s does not match sidecar %q", hash, strings.TrimSpace(string(sidecar)))
|
||||
}
|
||||
var bundle ds02FrozenBundle
|
||||
if err := json.Unmarshal(raw, &bundle); err != nil {
|
||||
t.Fatalf("parse identical bundle: %v", err)
|
||||
}
|
||||
if bundle.Bundle != "ds02-identical-input" || bundle.Version != 2 || len(bundle.DDL) == 0 ||
|
||||
bundle.CreatedAt != ds02IdenticalTime {
|
||||
t.Fatalf("unexpected identical bundle header: %s v%d created=%s ddl=%d",
|
||||
bundle.Bundle, bundle.Version, bundle.CreatedAt, len(bundle.DDL))
|
||||
}
|
||||
return bundle, hash
|
||||
}
|
||||
|
||||
func ds02BundleUint(value any) uint64 {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return uint64(typed)
|
||||
case uint64:
|
||||
return typed
|
||||
case int:
|
||||
return uint64(typed)
|
||||
case json.Number:
|
||||
parsed, _ := strconv.ParseUint(string(typed), 10, 64)
|
||||
return parsed
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ds02InputTable 是一张表/一组对象的"规范化语义输入":逐行规范化 JSON + 计数 + 摘要。
|
||||
type ds02InputTable struct {
|
||||
Count int `json:"count"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Rows []string `json:"rows"`
|
||||
}
|
||||
|
||||
type ds02InputArchive struct {
|
||||
Manifest map[string]any `json:"manifest"`
|
||||
Tables map[string]ds02InputTable `json:"tables"`
|
||||
Resources map[string]ds02InputTable `json:"resources"`
|
||||
}
|
||||
|
||||
type ds02InputEvidence struct {
|
||||
Side string `json:"side"`
|
||||
BundleSHA string `json:"bundle_sha256"`
|
||||
Scope map[string]any `json:"scope"`
|
||||
FixedTime string `json:"fixed_time"`
|
||||
Physical map[string]string `json:"physical"`
|
||||
DDL ds02InputTable `json:"ddl"`
|
||||
Sequences map[string][]any `json:"sequences_before"`
|
||||
Archive ds02InputArchive `json:"archive"`
|
||||
PartPhys map[string]map[string]any `json:"archive_part_physical"`
|
||||
TargetSeed map[string]ds02InputTable `json:"target_seed"`
|
||||
Objects map[string]ds02InputTable `json:"objects"`
|
||||
ES map[string]ds02InputTable `json:"es"`
|
||||
Semantic string `json:"semantic_sha256"`
|
||||
}
|
||||
|
||||
// ds02DigestRows 把行集合规范化成稳定可比较的字符串:用 UseNumber 解码后用
|
||||
// encoding/json 重编码(键按字典序),数字原样保留。
|
||||
func ds02DigestRows(raw [][]byte) (ds02InputTable, error) {
|
||||
rows := make([]string, 0, len(raw))
|
||||
for _, item := range raw {
|
||||
dec := json.NewDecoder(bytes.NewReader(item))
|
||||
dec.UseNumber()
|
||||
var value map[string]any
|
||||
if err := dec.Decode(&value); err != nil {
|
||||
return ds02InputTable{}, err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return ds02InputTable{}, err
|
||||
}
|
||||
rows = append(rows, string(encoded))
|
||||
}
|
||||
sort.Strings(rows)
|
||||
sum := sha256.Sum256([]byte(strings.Join(rows, "\n")))
|
||||
return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows}, nil
|
||||
}
|
||||
|
||||
func ds02DigestQuery(t *testing.T, fx *writeSetFixture, query string, args ...any) ds02InputTable {
|
||||
t.Helper()
|
||||
items := ds02Read(t, fx, query, args...)
|
||||
raw := make([][]byte, 0, len(items))
|
||||
for _, item := range items {
|
||||
encoded, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw = append(raw, encoded)
|
||||
}
|
||||
table, err := ds02DigestRows(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
// ds02NormalizeBucket 把这一侧的一次性桶名换回 %BUCKET%:桶名是唯一不可避免的
|
||||
// namespace 替换,两侧必须用同一规范归一,才谈得上"语义输入相同"。物理桶名单独记。
|
||||
// resources.location_hash 是 sha256(physical_path) 的派生值:路径被归一后必须按同一
|
||||
// 规则重算(先验证它确实等于物理路径的摘要,避免把无关字段一起改写)。
|
||||
func ds02NormalizeBucket(table ds02InputTable, bucket string) ds02InputTable {
|
||||
if bucket == "" {
|
||||
return table
|
||||
}
|
||||
rows := make([]string, 0, len(table.Rows))
|
||||
for _, row := range table.Rows {
|
||||
normalized := strings.ReplaceAll(row, bucket, ds02BucketPlaceholder)
|
||||
var parsed map[string]any
|
||||
if json.Unmarshal([]byte(normalized), &parsed) == nil {
|
||||
path, hasPath := parsed["physical_path"].(string)
|
||||
hash, hasHash := parsed["location_hash"].(string)
|
||||
physicalPath := strings.ReplaceAll(path, ds02BucketPlaceholder, bucket)
|
||||
if hasPath && hasHash && hash == ds02SHA256Hex(physicalPath) {
|
||||
parsed["location_hash"] = ds02SHA256Hex(path)
|
||||
if encoded, err := json.Marshal(parsed); err == nil {
|
||||
normalized = string(encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, normalized)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(strings.Join(rows, "\n")))
|
||||
return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows}
|
||||
}
|
||||
|
||||
func ds02SHA256Hex(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ds02DatabaseName 只留库名:证据里不记连接串。
|
||||
func ds02DatabaseName(dsn string) string {
|
||||
name := dsn
|
||||
if index := strings.LastIndex(name, "/"); index >= 0 {
|
||||
name = name[index+1:]
|
||||
}
|
||||
if index := strings.Index(name, "?"); index >= 0 {
|
||||
name = name[:index]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// ds02ArchiveSnapshot 读回归档里"实际写下的"部件,逐表导出。
|
||||
func ds02ArchiveSnapshot(t *testing.T, root, snapshotID string) (ds02InputArchive, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
archive, err := librarysnapshot.Open(root, 128*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
manifest, err := archive.Load(snapshotID)
|
||||
if err != nil {
|
||||
t.Fatalf("load archive %s: %v", snapshotID, err)
|
||||
}
|
||||
out := ds02InputArchive{Tables: map[string]ds02InputTable{}, Resources: map[string]ds02InputTable{}}
|
||||
parts := make([]map[string]any, 0, len(manifest.Parts))
|
||||
physical := map[string]map[string]any{}
|
||||
for _, part := range manifest.Parts {
|
||||
// 只记语义字段:部件字节的 sha256 会因 JSON 键序(PG 列序 vs Go 字典序)不同,
|
||||
// 属于物理身份,单独记录,不混进"同一输入"的语义比较。
|
||||
parts = append(parts, map[string]any{"kind": part.Kind, "name": part.Name, "count": part.Count})
|
||||
physical[part.Kind+"/"+part.Name] = map[string]any{"bytes": part.Bytes, "sha256": part.SHA256}
|
||||
reader, err := archive.OpenPart(part)
|
||||
if err != nil {
|
||||
t.Fatalf("open archive part %s: %v", part.Name, err)
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch part.Kind {
|
||||
case "resource":
|
||||
sum := sha256.Sum256(body)
|
||||
out.Resources[part.Name] = ds02InputTable{Count: 1, SHA256: hex.EncodeToString(sum[:]),
|
||||
Rows: []string{string(body)}}
|
||||
case "database", "wiki", "index":
|
||||
var items []json.RawMessage
|
||||
if err := json.Unmarshal(body, &items); err != nil {
|
||||
t.Fatalf("archive part %s: %v", part.Name, err)
|
||||
}
|
||||
raw := make([][]byte, 0, len(items))
|
||||
for _, item := range items {
|
||||
raw = append(raw, []byte(item))
|
||||
}
|
||||
table, err := ds02DigestRows(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("archive part %s: %v", part.Name, err)
|
||||
}
|
||||
out.Tables[part.Kind+"/"+part.Name] = table
|
||||
}
|
||||
}
|
||||
out.Manifest = map[string]any{"tenant": manifest.TenantID, "knowledge_base": manifest.KnowledgeBaseID,
|
||||
"format": manifest.Format, "policy": manifest.ValidationPolicy, "parts": parts}
|
||||
sort.Slice(parts, func(i, j int) bool {
|
||||
left := fmt.Sprint(parts[i]["kind"]) + "/" + fmt.Sprint(parts[i]["name"])
|
||||
right := fmt.Sprint(parts[j]["kind"]) + "/" + fmt.Sprint(parts[j]["name"])
|
||||
return left < right
|
||||
})
|
||||
return out, physical
|
||||
}
|
||||
|
||||
// ds02IdenticalSeal 与 ds02SealScope 同形,但使用固定的创建时间与实际来源范围,
|
||||
// 并把 index 部件的范围写到声明的来源租户/知识库。
|
||||
func ds02IdenticalSeal(t *testing.T, fx *writeSetFixture, spec ds02Spec, sourceTenant uint64, sourceKB string) string {
|
||||
t.Helper()
|
||||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
encode := func(value any) []byte {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
||||
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
||||
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
||||
parts := []librarysnapshot.Part{}
|
||||
for _, table := range ds02TargetTables {
|
||||
part, err := archive.Put(bytes.NewReader(encode(spec.rows[table])))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Kind, part.Name, part.Count = kinds[table], table, int64(len(spec.rows[table]))
|
||||
parts = append(parts, part)
|
||||
}
|
||||
objects := make([]string, 0, len(spec.objects))
|
||||
for id := range spec.objects {
|
||||
objects = append(objects, id)
|
||||
}
|
||||
sort.Strings(objects)
|
||||
for _, id := range objects {
|
||||
part, err := archive.Put(bytes.NewReader([]byte(spec.objects[id])))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Kind, part.Name, part.Count = "resource", id, 1
|
||||
parts = append(parts, part)
|
||||
}
|
||||
indexRows := []map[string]any{}
|
||||
for _, chunk := range spec.rows["chunks"] {
|
||||
id := fmt.Sprint(chunk["id"])
|
||||
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
||||
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||||
"knowledge_id": chunk["knowledge_id"], "chunk_id": id, "source_id": id,
|
||||
"content": chunk["content"], "embedding": spec.vectors[id]}})
|
||||
}
|
||||
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
||||
parts = append(parts, indexPart)
|
||||
created, err := time.Parse(time.RFC3339, ds02IdenticalTime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||||
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
|
||||
CreatedAt: created, Parts: parts,
|
||||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||||
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
||||
id, err := archive.Seal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("seal identical snapshot: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// ds02ListObjects 读真实对象存储里"执行前实际存在"的对象:key + 字节摘要。
|
||||
func ds02ListObjects(t *testing.T, fx *writeSetFixture) map[string]ds02InputTable {
|
||||
t.Helper()
|
||||
out := map[string]ds02InputTable{}
|
||||
paginator := awss3.NewListObjectsV2Paginator(fx.objects.s3.client, &awss3.ListObjectsV2Input{
|
||||
Bucket: aws.String(fx.bucket)})
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(fx.ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list objects: %v", err)
|
||||
}
|
||||
for _, item := range page.Contents {
|
||||
key := aws.ToString(item.Key)
|
||||
object, err := fx.objects.s3.client.GetObject(fx.ctx, &awss3.GetObjectInput{
|
||||
Bucket: aws.String(fx.bucket), Key: aws.String(key)})
|
||||
if err != nil {
|
||||
t.Fatalf("read object %s: %v", key, err)
|
||||
}
|
||||
body, err := io.ReadAll(object.Body)
|
||||
object.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
out[key] = ds02InputTable{Count: 1, SHA256: hex.EncodeToString(sum[:]), Rows: []string{string(body)}}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ds02IdenticalDDLDigest(t *testing.T, ddl []string, bucket string) ds02InputTable {
|
||||
t.Helper()
|
||||
rows := make([]string, 0, len(ddl))
|
||||
for _, statement := range ddl {
|
||||
rows = append(rows, strings.ReplaceAll(statement, bucket, ds02BucketPlaceholder))
|
||||
}
|
||||
sum := sha256.Sum256([]byte(strings.Join(rows, "\n")))
|
||||
return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows}
|
||||
}
|
||||
|
||||
// ds02CaptureInputs 在执行前导出"这一侧真正消费的输入"。
|
||||
func ds02CaptureInputs(t *testing.T, fx *writeSetFixture, spec ds02Spec, bundle ds02FrozenBundle,
|
||||
hash, snapshotID string, sourceTenant uint64, sourceKB string, ddl []string) ds02InputEvidence {
|
||||
t.Helper()
|
||||
evidence := ds02InputEvidence{
|
||||
Side: "new",
|
||||
BundleSHA: hash,
|
||||
Scope: bundle.Scope,
|
||||
FixedTime: ds02IdenticalTime,
|
||||
Physical: map[string]string{"bucket": fx.bucket, "es_index": fx.index, "database": ds02DatabaseName(os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")), "archive_root": fx.archiveRoot, "snapshot_id": snapshotID, "object_store": "real MinIO via S3ObjectStore"},
|
||||
DDL: ds02IdenticalDDLDigest(t, ddl, fx.bucket),
|
||||
Sequences: map[string][]any{},
|
||||
TargetSeed: map[string]ds02InputTable{},
|
||||
Objects: ds02ListObjects(t, fx),
|
||||
ES: map[string]ds02InputTable{},
|
||||
}
|
||||
evidence.Archive, evidence.PartPhys = ds02ArchiveSnapshot(t, fx.archiveRoot, snapshotID)
|
||||
for name, table := range evidence.Archive.Tables {
|
||||
evidence.Archive.Tables[name] = ds02NormalizeBucket(table, fx.bucket)
|
||||
}
|
||||
for _, name := range []string{"chunks_seq_id_seq", "knowledge_tags_seq_id_seq", "wiki_log_entries_id_seq"} {
|
||||
var last int64
|
||||
var called bool
|
||||
if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+name).Scan(&last, &called); err != nil {
|
||||
t.Fatalf("sequence %s: %v", name, err)
|
||||
}
|
||||
evidence.Sequences[name] = []any{last, called}
|
||||
}
|
||||
for _, table := range ds02TargetTables {
|
||||
evidence.TargetSeed[table] = ds02NormalizeBucket(ds02DigestQuery(t, fx,
|
||||
"SELECT to_jsonb(x)::text FROM "+table+" AS x ORDER BY 1"), fx.bucket)
|
||||
}
|
||||
docs := ds02Docs(t, fx)
|
||||
ids := make([]string, 0, len(docs))
|
||||
for id := range docs {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
raw, err := json.Marshal(docs[id])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
table, err := ds02DigestRows([][]byte{raw})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence.ES[id] = table
|
||||
}
|
||||
semantic := map[string]any{"fixed_time": evidence.FixedTime, "scope": evidence.Scope,
|
||||
"ddl": evidence.DDL, "sequences_before": evidence.Sequences,
|
||||
"archive": evidence.Archive, "target_seed": evidence.TargetSeed,
|
||||
"objects_before": evidence.Objects, "es_before": evidence.ES}
|
||||
rawSemantic, err := json.Marshal(semantic)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(rawSemantic)
|
||||
evidence.Semantic = hex.EncodeToString(sum[:])
|
||||
_ = sourceTenant
|
||||
_ = sourceKB
|
||||
return evidence
|
||||
}
|
||||
|
||||
func TestDS02IdenticalInputBundleIsolated(t *testing.T) {
|
||||
bundle, hash := ds02LoadIdenticalBundle(t)
|
||||
sourceTenant := ds02BundleUint(bundle.Scope["source_tenant"])
|
||||
sourceKB := fmt.Sprint(bundle.Scope["source_kb"])
|
||||
if sourceTenant != wsTargetTenant || sourceKB == wsTargetKB {
|
||||
t.Fatalf("identical bundle must be same tenant, different knowledge base: source=%d/%s target=%d/%s",
|
||||
sourceTenant, sourceKB, uint64(wsTargetTenant), wsTargetKB)
|
||||
}
|
||||
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, ddl: bundle.DDL})
|
||||
ds02VerifyScopeDecl(t, fx)
|
||||
for name, value := range bundle.Sequences {
|
||||
last := int64(value[0].(float64))
|
||||
called := value[1].(bool)
|
||||
r10Exec(t, fx, fmt.Sprintf("SELECT setval('%s',%d,%t)", name, last, called))
|
||||
}
|
||||
for id, body := range bundle.SeedObjects {
|
||||
if id != "res-other" {
|
||||
continue
|
||||
}
|
||||
path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id)
|
||||
if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, row := range bundle.TargetRows {
|
||||
if _, ok := ds02ScopeDecl[row.Table]; !ok {
|
||||
t.Fatalf("bundle target row for undeclared table %s", row.Table)
|
||||
}
|
||||
r11Insert(t, fx, row.Table, ds02SeedRow(row.Row, fx.bucket))
|
||||
}
|
||||
for _, doc := range bundle.SeedDocs {
|
||||
fx.indexDoc(t, doc.ID, doc.Tenant, doc.KB, doc.Knowledge, doc.Content)
|
||||
}
|
||||
spec := ds02Spec{rows: map[string][]map[string]any{}, vectors: bundle.Vectors, objects: map[string]string{}}
|
||||
for table, rows := range bundle.SourceRows {
|
||||
for _, row := range rows {
|
||||
spec.rows[table] = append(spec.rows[table], ds02Canon(t, fx, table, row))
|
||||
}
|
||||
}
|
||||
for _, table := range ds02TargetTables {
|
||||
if len(spec.rows[table]) == 0 {
|
||||
t.Fatalf("bundle source archive has no rows for %s; ten-table input is required", table)
|
||||
}
|
||||
}
|
||||
for _, row := range bundle.SourceRows["resources"] {
|
||||
id := fmt.Sprint(row["id"])
|
||||
body, ok := bundle.SeedObjects[id]
|
||||
if !ok {
|
||||
t.Fatalf("bundle has no bytes for source resource %s", id)
|
||||
}
|
||||
spec.objects[id] = body
|
||||
}
|
||||
preserved := map[string][]map[string]any{}
|
||||
for _, table := range ds02TargetTables {
|
||||
preserved[table] = ds02OtherRows(t, fx, table)
|
||||
}
|
||||
expectation := ds02Expected(t, fx, spec)
|
||||
expected, paths := expectation.Rows, expectation.Paths
|
||||
// resources / resource_bindings 没有 knowledge_base_id:同租户其它知识库的行也在
|
||||
// 目标范围内,必须原样保留。
|
||||
for _, table := range []string{"resources", "resource_bindings"} {
|
||||
expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...)
|
||||
}
|
||||
expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"],
|
||||
ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x
|
||||
WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`,
|
||||
wsTargetTenant, wsTargetKB)...)
|
||||
snapshot := ds02IdenticalSeal(t, fx, spec, sourceTenant, sourceKB)
|
||||
evidence := ds02CaptureInputs(t, fx, spec, bundle, hash, snapshot, sourceTenant, sourceKB, bundle.DDL)
|
||||
r10Save(t, "actual-inputs", evidence)
|
||||
recorder, inventory := ds02AttachEgress(t, fx)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
recorder.setPhase("publish")
|
||||
out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil)
|
||||
recorder.setPhase("oracle-read")
|
||||
r10Save(t, "bundle-identity", map[string]any{"path": os.Getenv("KG_DS02_BUNDLE"), "sha256": hash,
|
||||
"source_tenant": sourceTenant, "source_kb": sourceKB, "target_tenant": wsTargetTenant,
|
||||
"target_kb": wsTargetKB, "created_at": bundle.CreatedAt})
|
||||
r10Save(t, "outcome", out)
|
||||
if out.State != "succeeded" {
|
||||
t.Fatalf("new source must publish the identical-input bundle: %+v", out)
|
||||
}
|
||||
if out.Backup.WriteSet == nil {
|
||||
t.Fatal("write set missing")
|
||||
}
|
||||
ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
||||
if err != nil {
|
||||
t.Fatalf("load write set: %v", err)
|
||||
}
|
||||
columns := ds02LocalColumns(t, fx, spec, expectation)
|
||||
mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs)
|
||||
r10Save(t, "mapping-diffs", mappingDiffs)
|
||||
if len(mappingDiffs) != 0 {
|
||||
t.Fatalf("identical-input publish fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n"))
|
||||
}
|
||||
diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory)
|
||||
r10Save(t, "green-diffs", diffs)
|
||||
if len(diffs) != 0 {
|
||||
t.Fatalf("identical-input independent oracle disagrees:\n%s", strings.Join(diffs, "\n"))
|
||||
}
|
||||
r10Save(t, "sequences-after", ds02SequenceSummary(t, fx))
|
||||
r10Save(t, "sentinels-after", map[string]any{
|
||||
"sibling_object": ds02CaptureObject(t, fx, wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-other")),
|
||||
"foreign_doc": ds02Docs(t, fx)["chunk-foreign"],
|
||||
"other_kb_log": ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM wiki_log_entries AS x WHERE tenant_id=$1 AND knowledge_base_id=$2", wsTargetTenant, wsOtherKB),
|
||||
})
|
||||
}
|
||||
|
||||
func ds02CaptureObject(t *testing.T, fx *writeSetFixture, path string) map[string]any {
|
||||
t.Helper()
|
||||
reader, err := fx.objects.s3.Open(fx.ctx, "minio", path)
|
||||
if err != nil {
|
||||
t.Fatalf("read object: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return map[string]any{"path": strings.ReplaceAll(path, fx.bucket, ds02BucketPlaceholder),
|
||||
"bytes": len(body), "sha256": hex.EncodeToString(sum[:])}
|
||||
}
|
||||
Reference in New Issue
Block a user