diff --git a/CHANGELOG.md b/CHANGELOG.md index 2213201..958d15f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,14 @@ - 新增冻结输入 bundle(十表来源行、目标种子、序列种子、ES 文档、对象字节、完整 DDL, 时间固定、逐表 hash):旧源码 `6fe98c9` 与当前源码读取同一份文件、同一 sha256。旧侧真实 复现 `wiki_log_entries_pkey / SQLSTATE 23505`,新侧同一输入发布成功并通过独立 oracle。 - 差异单列:6fe98c9 在 wiki 之前无法跨租户搬运资源,旧侧归档省略 `resources` / - `resource_bindings` 两表,其余八表与碰撞行逐字节相同;新侧按十表完整运行。 + 该跨租户 bundle 是**受约束回归对照**:6fe98c9 在 wiki 之前无法跨租户搬运资源,旧侧归档 + 只能省略 `resources` / `resource_bindings` 两表,因此不能据此声称"同输入已收口"。 +- 新增十表同输入对照(同租户、不同知识库):`ds02-identical-input-bundle.json` 两侧各自 + 规范化并封存完整十表来源归档(含 `resources` / `resource_bindings`),旧侧 `6fe98c9` 用 + 真实 PG/ES/MinIO 对象适配复现 `wiki_log_entries_pkey / SQLSTATE 23505`,新侧同输入 + `succeeded` 并通过映射/内容 oracle。两侧在执行前各自动导出逐表来源归档、实际目标种子、 + 完整 DDL、三条序列、ES 文档与对象字节;唯一 namespace 替换(对象桶名 `%BUCKET%`) + 按同一规范归一后语义输入 hash 相同,物理桶名单独记录。 - 新增测试进程身份断言:runner 注入 test-only nonce/commit/源码树摘要,测试进程自报 `os.Executable` 的 sha256,与 runner 期望逐字符比较;构建与执行前后各算一次二进制哈希。 - 新增隔离运行器 v2(`ds02_runner_v2.py`,v1 保留):超时/构建/准备异常都走 finally 清理, diff --git a/internal/releaseadapter/v071/ds02_identical_input_test.go b/internal/releaseadapter/v071/ds02_identical_input_test.go new file mode 100644 index 0000000..efa5c4d --- /dev/null +++ b/internal/releaseadapter/v071/ds02_identical_input_test.go @@ -0,0 +1,735 @@ +package v071 + +// DS-02 十表同输入对照(同租户、不同知识库)。 +// +// 背景:跨租户冻结 bundle 的旧侧 6fe98c9 在 database 阶段不改写租户、resources 阶段 +// 却按目标租户查资源行,跨租户资源必然在 wiki 之前失败。旧侧归档因此省略了 +// resources / resource_bindings 两表 —— 那不是"同一输入",只能算受约束的回归对照。 +// +// 本文件把场景改成"来源与目标同租户、不同知识库": +// * 旧侧的资源查询只按 tenant_id,同租户下能在 resources 阶段找到刚插入的资源行, +// 前置缺陷不再触发; +// * wiki_log_entries 的数字 id 仍在同一租户的其它知识库里占用了 id=1, +// 旧代码按归档原样插入时仍会撞 wiki_log_entries_pkey / 23505。 +// +// 因此两侧都能消费"真正相同的十表输入":来源归档十表逐表内容、目标种子、完整 DDL、 +// 三条序列、ES 文档与对象字节。本文件负责生成新的 bundle 与执行前导出"实际消费输入" +// 证据;旧侧探针(旧 SHA 工作树内未跟踪测试文件)导出同一形状的证据,再由证据根脚本 +// 逐项比较 —— 不是"读同一个文件再各自过滤"。 +// +// 导出(不连接任何服务): +// KG_DS02_IDENTICAL_BUNDLE_OUT=/path/dir go test -run TestDS02ExportIdenticalInputBundle +// 新侧执行: +// KG_DS02_BUNDLE=/path/dir/ds02-identical-input-bundle.json go test -run TestDS02IdenticalInputBundleIsolated + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// 时间固定:bundle 声明、归档 manifest、发布 request 都用这一个时刻。 +const ds02IdenticalTime = "2026-11-02T00:00:00Z" + +func ds02IdenticalScope() map[string]any { + return map[string]any{ + "source_tenant": uint64(wsTargetTenant), + "source_kb": wsSourceKB, + "target_tenant": uint64(wsTargetTenant), + "target_kb": wsTargetKB, + "other_kb": wsOtherKB, + "foreign_tenant": uint64(wsForeignTnt), + } +} + +// ds02IdenticalSourceRows 是十表来源行的纯声明:租户与目标相同,知识库不同。 +// 资源定位符里的租户段随租户一起改写(bucket 段保持固定的来源桶名,不引用两侧 +// 一次性桶,否则归档内容会随环境变化);location_hash 按真实定位符重算。 +func ds02IdenticalSourceRows() map[string][]map[string]any { + rows := ds02SourceRows() + for table, list := range rows { + for _, row := range list { + if _, ok := row["tenant_id"]; ok { + row["tenant_id"] = uint64(wsTargetTenant) + } + if table != "resources" { + continue + } + id := fmt.Sprint(row["id"]) + path := wsLocator("backend-source", "oss", "source-bucket", wsTargetTenant, id) + row["physical_path"] = path + sum := sha256.Sum256([]byte(path)) + row["location_hash"] = hex.EncodeToString(sum[:]) + } + } + return rows +} + +// ds02IdenticalSeedRows 给目标种子补上显式时间戳:种子行是"发布前目标已有数据", +// 两侧都必须逐列相等;让 now()/CURRENT_TIMESTAMP 在两侧落地成不同值会让"实际目标种子 +// 相同"这条断言无法成立。DDL 与 catalog 本身不改。 +func ds02IdenticalSeedRows() []struct { + table string + row map[string]any +} { + timestamps := map[string][]string{ + "knowledges": {"created_at", "updated_at"}, + "chunks": {"created_at", "updated_at"}, + "knowledge_tags": {"created_at", "updated_at"}, + "knowledge_tag_relations": {"created_at"}, + "resources": {"created_at", "updated_at"}, + "resource_bindings": {"created_at"}, + "wiki_folders": {"created_at", "updated_at"}, + "wiki_pages": {"created_at", "updated_at"}, + "wiki_page_issues": {"created_at", "updated_at"}, + "wiki_log_entries": {"created_at"}, + } + fixed := map[string]string{"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"} + seeds := ds02SeedRows() + out := make([]struct { + table string + row map[string]any + }, 0, len(seeds)) + for _, seed := range seeds { + row := map[string]any{} + for key, value := range seed.row { + row[key] = value + } + for _, column := range timestamps[seed.table] { + if _, ok := row[column]; ok { + continue + } + row[column] = fixed[column] + } + out = append(out, struct { + table string + row map[string]any + }{table: seed.table, row: row}) + } + return out +} + +// ds02IdenticalDDL 用同一份真实 catalog 生成完整 DDL,只把来源知识库的租户改成与目标 +// 相同的值(bundle 的 scope 声明必须与 DDL 里的行一致)。其他语句逐字节相同。 +func ds02IdenticalDDL(t *testing.T) []string { + t.Helper() + ddl := ds02BundleDDL(t) + old := "('" + wsSourceKB + "'," + strconv.Itoa(wsSourceTenant) + replacement := "('" + wsSourceKB + "'," + strconv.Itoa(wsTargetTenant) + replaced := 0 + out := make([]string, 0, len(ddl)) + for _, statement := range ddl { + if strings.Contains(statement, old) { + statement = strings.Replace(statement, old, replacement, 1) + replaced++ + } + out = append(out, statement) + } + if replaced != 1 { + t.Fatalf("source knowledge base row in DDL: want exactly 1 statement, replaced %d", replaced) + } + return out +} + +func TestDS02ExportIdenticalInputBundle(t *testing.T) { + root := os.Getenv("KG_DS02_IDENTICAL_BUNDLE_OUT") + if root == "" { + t.Skip("set KG_DS02_IDENTICAL_BUNDLE_OUT to write the identical-input bundle") + } + rows := ds02IdenticalSourceRows() + perTable := map[string]string{} + for table, list := range rows { + raw, err := json.Marshal(list) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + perTable[table] = hex.EncodeToString(sum[:]) + } + objects := map[string]string{} + for id, body := range ds02Objects { + objects[id] = body + } + for id, body := range ds02SeedObjects() { + objects[id] = body + } + bundle := ds02FrozenBundle{ + Bundle: "ds02-identical-input", + Version: 2, + CreatedAt: ds02IdenticalTime, + Scope: ds02IdenticalScope(), + Sequences: ds02BundleSequences(), + SourceRows: rows, + SeedObjects: objects, + Vectors: ds02Vectors, + DDL: ds02IdenticalDDL(t), + } + for _, seed := range ds02IdenticalSeedRows() { + bundle.TargetRows = append(bundle.TargetRows, ds02BundleSeedRow{Table: seed.table, Row: seed.row}) + } + for _, doc := range ds02SeedDocs() { + bundle.SeedDocs = append(bundle.SeedDocs, ds02BundleSeedDoc{ID: doc.id, Tenant: doc.tenant, KB: doc.kb, Knowledge: doc.knowledge, Content: doc.content}) + } + raw, err := json.MarshalIndent(bundle, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(root, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "ds02-identical-input-bundle.json") + if err := os.WriteFile(path, append(raw, '\n'), 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(append(raw, '\n')) + hash := hex.EncodeToString(sum[:]) + if err := os.WriteFile(filepath.Join(root, "ds02-identical-input-bundle.sha256"), []byte(hash+" ds02-identical-input-bundle.json\n"), 0600); err != nil { + t.Fatal(err) + } + result := map[string]any{"bundle": path, "sha256": hash, "table_hashes": perTable, + "bytes": len(raw) + 1, "ddl_statements": len(bundle.DDL), "created_at": bundle.CreatedAt, + "scope": bundle.Scope} + if run := os.Getenv("KG_R10_RUN"); run != "" { + r10Save(t, "identical-bundle", result) + } + encoded, _ := json.MarshalIndent(result, "", " ") + t.Logf("%s", encoded) +} + +func ds02LoadIdenticalBundle(t *testing.T) (ds02FrozenBundle, string) { + t.Helper() + path := os.Getenv("KG_DS02_BUNDLE") + if path == "" { + t.Skip("set KG_DS02_BUNDLE to run the identical-input comparison") + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read identical bundle: %v", err) + } + sum := sha256.Sum256(raw) + hash := hex.EncodeToString(sum[:]) + sidecar, err := os.ReadFile(strings.TrimSuffix(path, ".json") + ".sha256") + if err != nil { + t.Fatalf("read identical bundle sha256 sidecar: %v", err) + } + if !strings.HasPrefix(strings.TrimSpace(string(sidecar)), hash) { + t.Fatalf("identical bundle sha256 %s does not match sidecar %q", hash, strings.TrimSpace(string(sidecar))) + } + var bundle ds02FrozenBundle + if err := json.Unmarshal(raw, &bundle); err != nil { + t.Fatalf("parse identical bundle: %v", err) + } + if bundle.Bundle != "ds02-identical-input" || bundle.Version != 2 || len(bundle.DDL) == 0 || + bundle.CreatedAt != ds02IdenticalTime { + t.Fatalf("unexpected identical bundle header: %s v%d created=%s ddl=%d", + bundle.Bundle, bundle.Version, bundle.CreatedAt, len(bundle.DDL)) + } + return bundle, hash +} + +func ds02BundleUint(value any) uint64 { + switch typed := value.(type) { + case float64: + return uint64(typed) + case uint64: + return typed + case int: + return uint64(typed) + case json.Number: + parsed, _ := strconv.ParseUint(string(typed), 10, 64) + return parsed + } + return 0 +} + +// ds02InputTable 是一张表/一组对象的"规范化语义输入":逐行规范化 JSON + 计数 + 摘要。 +type ds02InputTable struct { + Count int `json:"count"` + SHA256 string `json:"sha256"` + Rows []string `json:"rows"` +} + +type ds02InputArchive struct { + Manifest map[string]any `json:"manifest"` + Tables map[string]ds02InputTable `json:"tables"` + Resources map[string]ds02InputTable `json:"resources"` +} + +type ds02InputEvidence struct { + Side string `json:"side"` + BundleSHA string `json:"bundle_sha256"` + Scope map[string]any `json:"scope"` + FixedTime string `json:"fixed_time"` + Physical map[string]string `json:"physical"` + DDL ds02InputTable `json:"ddl"` + Sequences map[string][]any `json:"sequences_before"` + Archive ds02InputArchive `json:"archive"` + PartPhys map[string]map[string]any `json:"archive_part_physical"` + TargetSeed map[string]ds02InputTable `json:"target_seed"` + Objects map[string]ds02InputTable `json:"objects"` + ES map[string]ds02InputTable `json:"es"` + Semantic string `json:"semantic_sha256"` +} + +// ds02DigestRows 把行集合规范化成稳定可比较的字符串:用 UseNumber 解码后用 +// encoding/json 重编码(键按字典序),数字原样保留。 +func ds02DigestRows(raw [][]byte) (ds02InputTable, error) { + rows := make([]string, 0, len(raw)) + for _, item := range raw { + dec := json.NewDecoder(bytes.NewReader(item)) + dec.UseNumber() + var value map[string]any + if err := dec.Decode(&value); err != nil { + return ds02InputTable{}, err + } + encoded, err := json.Marshal(value) + if err != nil { + return ds02InputTable{}, err + } + rows = append(rows, string(encoded)) + } + sort.Strings(rows) + sum := sha256.Sum256([]byte(strings.Join(rows, "\n"))) + return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows}, nil +} + +func ds02DigestQuery(t *testing.T, fx *writeSetFixture, query string, args ...any) ds02InputTable { + t.Helper() + items := ds02Read(t, fx, query, args...) + raw := make([][]byte, 0, len(items)) + for _, item := range items { + encoded, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + raw = append(raw, encoded) + } + table, err := ds02DigestRows(raw) + if err != nil { + t.Fatal(err) + } + return table +} + +// ds02NormalizeBucket 把这一侧的一次性桶名换回 %BUCKET%:桶名是唯一不可避免的 +// namespace 替换,两侧必须用同一规范归一,才谈得上"语义输入相同"。物理桶名单独记。 +// resources.location_hash 是 sha256(physical_path) 的派生值:路径被归一后必须按同一 +// 规则重算(先验证它确实等于物理路径的摘要,避免把无关字段一起改写)。 +func ds02NormalizeBucket(table ds02InputTable, bucket string) ds02InputTable { + if bucket == "" { + return table + } + rows := make([]string, 0, len(table.Rows)) + for _, row := range table.Rows { + normalized := strings.ReplaceAll(row, bucket, ds02BucketPlaceholder) + var parsed map[string]any + if json.Unmarshal([]byte(normalized), &parsed) == nil { + path, hasPath := parsed["physical_path"].(string) + hash, hasHash := parsed["location_hash"].(string) + physicalPath := strings.ReplaceAll(path, ds02BucketPlaceholder, bucket) + if hasPath && hasHash && hash == ds02SHA256Hex(physicalPath) { + parsed["location_hash"] = ds02SHA256Hex(path) + if encoded, err := json.Marshal(parsed); err == nil { + normalized = string(encoded) + } + } + } + rows = append(rows, normalized) + } + sum := sha256.Sum256([]byte(strings.Join(rows, "\n"))) + return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows} +} + +func ds02SHA256Hex(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +// ds02DatabaseName 只留库名:证据里不记连接串。 +func ds02DatabaseName(dsn string) string { + name := dsn + if index := strings.LastIndex(name, "/"); index >= 0 { + name = name[index+1:] + } + if index := strings.Index(name, "?"); index >= 0 { + name = name[:index] + } + return name +} + +// ds02ArchiveSnapshot 读回归档里"实际写下的"部件,逐表导出。 +func ds02ArchiveSnapshot(t *testing.T, root, snapshotID string) (ds02InputArchive, map[string]map[string]any) { + t.Helper() + archive, err := librarysnapshot.Open(root, 128*1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + manifest, err := archive.Load(snapshotID) + if err != nil { + t.Fatalf("load archive %s: %v", snapshotID, err) + } + out := ds02InputArchive{Tables: map[string]ds02InputTable{}, Resources: map[string]ds02InputTable{}} + parts := make([]map[string]any, 0, len(manifest.Parts)) + physical := map[string]map[string]any{} + for _, part := range manifest.Parts { + // 只记语义字段:部件字节的 sha256 会因 JSON 键序(PG 列序 vs Go 字典序)不同, + // 属于物理身份,单独记录,不混进"同一输入"的语义比较。 + parts = append(parts, map[string]any{"kind": part.Kind, "name": part.Name, "count": part.Count}) + physical[part.Kind+"/"+part.Name] = map[string]any{"bytes": part.Bytes, "sha256": part.SHA256} + reader, err := archive.OpenPart(part) + if err != nil { + t.Fatalf("open archive part %s: %v", part.Name, err) + } + body, err := io.ReadAll(reader) + reader.Close() + if err != nil { + t.Fatal(err) + } + switch part.Kind { + case "resource": + sum := sha256.Sum256(body) + out.Resources[part.Name] = ds02InputTable{Count: 1, SHA256: hex.EncodeToString(sum[:]), + Rows: []string{string(body)}} + case "database", "wiki", "index": + var items []json.RawMessage + if err := json.Unmarshal(body, &items); err != nil { + t.Fatalf("archive part %s: %v", part.Name, err) + } + raw := make([][]byte, 0, len(items)) + for _, item := range items { + raw = append(raw, []byte(item)) + } + table, err := ds02DigestRows(raw) + if err != nil { + t.Fatalf("archive part %s: %v", part.Name, err) + } + out.Tables[part.Kind+"/"+part.Name] = table + } + } + out.Manifest = map[string]any{"tenant": manifest.TenantID, "knowledge_base": manifest.KnowledgeBaseID, + "format": manifest.Format, "policy": manifest.ValidationPolicy, "parts": parts} + sort.Slice(parts, func(i, j int) bool { + left := fmt.Sprint(parts[i]["kind"]) + "/" + fmt.Sprint(parts[i]["name"]) + right := fmt.Sprint(parts[j]["kind"]) + "/" + fmt.Sprint(parts[j]["name"]) + return left < right + }) + return out, physical +} + +// ds02IdenticalSeal 与 ds02SealScope 同形,但使用固定的创建时间与实际来源范围, +// 并把 index 部件的范围写到声明的来源租户/知识库。 +func ds02IdenticalSeal(t *testing.T, fx *writeSetFixture, spec ds02Spec, sourceTenant uint64, sourceKB string) string { + t.Helper() + archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + encode := func(value any) []byte { + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return raw + } + kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database", + "knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database", + "wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"} + parts := []librarysnapshot.Part{} + for _, table := range ds02TargetTables { + part, err := archive.Put(bytes.NewReader(encode(spec.rows[table]))) + if err != nil { + t.Fatal(err) + } + part.Kind, part.Name, part.Count = kinds[table], table, int64(len(spec.rows[table])) + parts = append(parts, part) + } + objects := make([]string, 0, len(spec.objects)) + for id := range spec.objects { + objects = append(objects, id) + } + sort.Strings(objects) + for _, id := range objects { + part, err := archive.Put(bytes.NewReader([]byte(spec.objects[id]))) + if err != nil { + t.Fatal(err) + } + part.Kind, part.Name, part.Count = "resource", id, 1 + parts = append(parts, part) + } + indexRows := []map[string]any{} + for _, chunk := range spec.rows["chunks"] { + id := fmt.Sprint(chunk["id"]) + indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{ + "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, + "knowledge_id": chunk["knowledge_id"], "chunk_id": id, "source_id": id, + "content": chunk["content"], "embedding": spec.vectors[id]}}) + } + indexPart, err := archive.Put(bytes.NewReader(encode(indexRows))) + if err != nil { + t.Fatal(err) + } + indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows)) + parts = append(parts, indexPart) + created, err := time.Parse(time.RFC3339, ds02IdenticalTime) + if err != nil { + t.Fatal(err) + } + manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, + Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB, + CreatedAt: created, Parts: parts, + Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", + EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}} + id, err := archive.Seal(manifest) + if err != nil { + t.Fatalf("seal identical snapshot: %v", err) + } + return id +} + +// ds02ListObjects 读真实对象存储里"执行前实际存在"的对象:key + 字节摘要。 +func ds02ListObjects(t *testing.T, fx *writeSetFixture) map[string]ds02InputTable { + t.Helper() + out := map[string]ds02InputTable{} + paginator := awss3.NewListObjectsV2Paginator(fx.objects.s3.client, &awss3.ListObjectsV2Input{ + Bucket: aws.String(fx.bucket)}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(fx.ctx) + if err != nil { + t.Fatalf("list objects: %v", err) + } + for _, item := range page.Contents { + key := aws.ToString(item.Key) + object, err := fx.objects.s3.client.GetObject(fx.ctx, &awss3.GetObjectInput{ + Bucket: aws.String(fx.bucket), Key: aws.String(key)}) + if err != nil { + t.Fatalf("read object %s: %v", key, err) + } + body, err := io.ReadAll(object.Body) + object.Body.Close() + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + out[key] = ds02InputTable{Count: 1, SHA256: hex.EncodeToString(sum[:]), Rows: []string{string(body)}} + } + } + return out +} + +func ds02IdenticalDDLDigest(t *testing.T, ddl []string, bucket string) ds02InputTable { + t.Helper() + rows := make([]string, 0, len(ddl)) + for _, statement := range ddl { + rows = append(rows, strings.ReplaceAll(statement, bucket, ds02BucketPlaceholder)) + } + sum := sha256.Sum256([]byte(strings.Join(rows, "\n"))) + return ds02InputTable{Count: len(rows), SHA256: hex.EncodeToString(sum[:]), Rows: rows} +} + +// ds02CaptureInputs 在执行前导出"这一侧真正消费的输入"。 +func ds02CaptureInputs(t *testing.T, fx *writeSetFixture, spec ds02Spec, bundle ds02FrozenBundle, + hash, snapshotID string, sourceTenant uint64, sourceKB string, ddl []string) ds02InputEvidence { + t.Helper() + evidence := ds02InputEvidence{ + Side: "new", + BundleSHA: hash, + Scope: bundle.Scope, + FixedTime: ds02IdenticalTime, + Physical: map[string]string{"bucket": fx.bucket, "es_index": fx.index, "database": ds02DatabaseName(os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")), "archive_root": fx.archiveRoot, "snapshot_id": snapshotID, "object_store": "real MinIO via S3ObjectStore"}, + DDL: ds02IdenticalDDLDigest(t, ddl, fx.bucket), + Sequences: map[string][]any{}, + TargetSeed: map[string]ds02InputTable{}, + Objects: ds02ListObjects(t, fx), + ES: map[string]ds02InputTable{}, + } + evidence.Archive, evidence.PartPhys = ds02ArchiveSnapshot(t, fx.archiveRoot, snapshotID) + for name, table := range evidence.Archive.Tables { + evidence.Archive.Tables[name] = ds02NormalizeBucket(table, fx.bucket) + } + for _, name := range []string{"chunks_seq_id_seq", "knowledge_tags_seq_id_seq", "wiki_log_entries_id_seq"} { + var last int64 + var called bool + if err := fx.pool.QueryRow(fx.ctx, "SELECT last_value,is_called FROM "+name).Scan(&last, &called); err != nil { + t.Fatalf("sequence %s: %v", name, err) + } + evidence.Sequences[name] = []any{last, called} + } + for _, table := range ds02TargetTables { + evidence.TargetSeed[table] = ds02NormalizeBucket(ds02DigestQuery(t, fx, + "SELECT to_jsonb(x)::text FROM "+table+" AS x ORDER BY 1"), fx.bucket) + } + docs := ds02Docs(t, fx) + ids := make([]string, 0, len(docs)) + for id := range docs { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + raw, err := json.Marshal(docs[id]) + if err != nil { + t.Fatal(err) + } + table, err := ds02DigestRows([][]byte{raw}) + if err != nil { + t.Fatal(err) + } + evidence.ES[id] = table + } + semantic := map[string]any{"fixed_time": evidence.FixedTime, "scope": evidence.Scope, + "ddl": evidence.DDL, "sequences_before": evidence.Sequences, + "archive": evidence.Archive, "target_seed": evidence.TargetSeed, + "objects_before": evidence.Objects, "es_before": evidence.ES} + rawSemantic, err := json.Marshal(semantic) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(rawSemantic) + evidence.Semantic = hex.EncodeToString(sum[:]) + _ = sourceTenant + _ = sourceKB + return evidence +} + +func TestDS02IdenticalInputBundleIsolated(t *testing.T) { + bundle, hash := ds02LoadIdenticalBundle(t) + sourceTenant := ds02BundleUint(bundle.Scope["source_tenant"]) + sourceKB := fmt.Sprint(bundle.Scope["source_kb"]) + if sourceTenant != wsTargetTenant || sourceKB == wsTargetKB { + t.Fatalf("identical bundle must be same tenant, different knowledge base: source=%d/%s target=%d/%s", + sourceTenant, sourceKB, uint64(wsTargetTenant), wsTargetKB) + } + fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, ddl: bundle.DDL}) + ds02VerifyScopeDecl(t, fx) + for name, value := range bundle.Sequences { + last := int64(value[0].(float64)) + called := value[1].(bool) + r10Exec(t, fx, fmt.Sprintf("SELECT setval('%s',%d,%t)", name, last, called)) + } + for id, body := range bundle.SeedObjects { + if id != "res-other" { + continue + } + path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id) + if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil { + t.Fatal(err) + } + } + for _, row := range bundle.TargetRows { + if _, ok := ds02ScopeDecl[row.Table]; !ok { + t.Fatalf("bundle target row for undeclared table %s", row.Table) + } + r11Insert(t, fx, row.Table, ds02SeedRow(row.Row, fx.bucket)) + } + for _, doc := range bundle.SeedDocs { + fx.indexDoc(t, doc.ID, doc.Tenant, doc.KB, doc.Knowledge, doc.Content) + } + spec := ds02Spec{rows: map[string][]map[string]any{}, vectors: bundle.Vectors, objects: map[string]string{}} + for table, rows := range bundle.SourceRows { + for _, row := range rows { + spec.rows[table] = append(spec.rows[table], ds02Canon(t, fx, table, row)) + } + } + for _, table := range ds02TargetTables { + if len(spec.rows[table]) == 0 { + t.Fatalf("bundle source archive has no rows for %s; ten-table input is required", table) + } + } + for _, row := range bundle.SourceRows["resources"] { + id := fmt.Sprint(row["id"]) + body, ok := bundle.SeedObjects[id] + if !ok { + t.Fatalf("bundle has no bytes for source resource %s", id) + } + spec.objects[id] = body + } + preserved := map[string][]map[string]any{} + for _, table := range ds02TargetTables { + preserved[table] = ds02OtherRows(t, fx, table) + } + expectation := ds02Expected(t, fx, spec) + expected, paths := expectation.Rows, expectation.Paths + // resources / resource_bindings 没有 knowledge_base_id:同租户其它知识库的行也在 + // 目标范围内,必须原样保留。 + for _, table := range []string{"resources", "resource_bindings"} { + expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...) + } + expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"], + ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x + WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`, + wsTargetTenant, wsTargetKB)...) + snapshot := ds02IdenticalSeal(t, fx, spec, sourceTenant, sourceKB) + evidence := ds02CaptureInputs(t, fx, spec, bundle, hash, snapshot, sourceTenant, sourceKB, bundle.DDL) + r10Save(t, "actual-inputs", evidence) + recorder, inventory := ds02AttachEgress(t, fx) + target := fx.target(t) + backups := fx.backups(t, target) + recorder.setPhase("publish") + out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil) + recorder.setPhase("oracle-read") + r10Save(t, "bundle-identity", map[string]any{"path": os.Getenv("KG_DS02_BUNDLE"), "sha256": hash, + "source_tenant": sourceTenant, "source_kb": sourceKB, "target_tenant": wsTargetTenant, + "target_kb": wsTargetKB, "created_at": bundle.CreatedAt}) + r10Save(t, "outcome", out) + if out.State != "succeeded" { + t.Fatalf("new source must publish the identical-input bundle: %+v", out) + } + if out.Backup.WriteSet == nil { + t.Fatal("write set missing") + } + ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest) + if err != nil { + t.Fatalf("load write set: %v", err) + } + columns := ds02LocalColumns(t, fx, spec, expectation) + mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs) + r10Save(t, "mapping-diffs", mappingDiffs) + if len(mappingDiffs) != 0 { + t.Fatalf("identical-input publish fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n")) + } + diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory) + r10Save(t, "green-diffs", diffs) + if len(diffs) != 0 { + t.Fatalf("identical-input independent oracle disagrees:\n%s", strings.Join(diffs, "\n")) + } + r10Save(t, "sequences-after", ds02SequenceSummary(t, fx)) + r10Save(t, "sentinels-after", map[string]any{ + "sibling_object": ds02CaptureObject(t, fx, wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-other")), + "foreign_doc": ds02Docs(t, fx)["chunk-foreign"], + "other_kb_log": ds02Read(t, fx, "SELECT to_jsonb(x)::text FROM wiki_log_entries AS x WHERE tenant_id=$1 AND knowledge_base_id=$2", wsTargetTenant, wsOtherKB), + }) +} + +func ds02CaptureObject(t *testing.T, fx *writeSetFixture, path string) map[string]any { + t.Helper() + reader, err := fx.objects.s3.Open(fx.ctx, "minio", path) + if err != nil { + t.Fatalf("read object: %v", err) + } + body, err := io.ReadAll(reader) + reader.Close() + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + return map[string]any{"path": strings.ReplaceAll(path, fx.bucket, ds02BucketPlaceholder), + "bytes": len(body), "sha256": hex.EncodeToString(sum[:])} +}