07a9caf22b
build / build (push) Failing after 19s
为什么:Astra 复验已独立复现旧 LocalIDs 循环的三种盲区——目标互换、错误来源集合、
不存在的目标都能绿;引用把 pre/next/parent 混在一个 OR 里,parent 分支没有覆盖;
成功路径没有序列非空比对;零 Embedding 只测了一个 client;旧侧对照用 rev-list 的
258 冒充 0.1.258(该工作树 identity 实际是 0.1.148);runner 在异常/超时路径不保证
清理,读回把任何非零退出都当"不存在"。
改了什么:
- 抽出独立语义 mapping checker:三表 (table,column,source,target) 精确集合 + 真实行
对应;持久账本正例与语义副本变异走同一 checker,digest gate 与语义 gate 分开。
- 测试自带十表 scope 声明并与 schema 列集合对拍;逐种引用非空命中,软导航字段显式
列为范围外;外围 ES 完整 _source、res-other 字节各补负控制。
- 零 Embedding 改为发布段独立记账:ES + 对象存储两个可达 client,自定义 /v1/encode
负控制;范围只声明本测试可达 client,正式全链留给 DS03。
- 冻结输入 bundle(十表行/目标种子/序列/ES/对象/DDL,逐表 hash):旧 6fe98c9 与新源码
同一 bundle 文件;旧侧真实 23505,新侧发布成功 + 独立 oracle。
- 测试进程身份:runner 注入 test-only nonce/commit/树摘要,进程自报 executable hash。
- 历史兼容矩阵改同 catalog 初始化,三条序列显式存在并逐例 before/after。
- runner v2(证据根,不在本仓):失败关闭、finally 清理、所有权重核、keep-failure。
验证:make verify 16 项全绿;ds02_runner_v2 三次隔离运行(含负控制与 keep-failure)
raw 见 handoffs/develop-release-contract/ds02-v2-*;旧侧探针 ds02-bundle-old2。
320 lines
13 KiB
Go
320 lines
13 KiB
Go
package v071
|
|
|
|
// DS-02 冻结输入 bundle:同一份文件同时给旧源码探针(6fe98c9 工作树)与当前源码
|
|
// 使用。bundle 只含业务输入(十表来源行、目标种子行、序列种子、ES 文档、对象字节、
|
|
// 固定时间与完整 DDL);两个隔离目标各自执行同一份 DDL、插入同一批行。
|
|
//
|
|
// 唯一的环境替换是对象桶名:bundle 里写 %BUCKET%,两侧各自换成自己的桶。
|
|
// 这不是输入差异,必须在报告里显式列出。
|
|
//
|
|
// 导出(不连接任何服务):
|
|
// KG_DS02_BUNDLE_OUT=/path/dir go test -run TestDS02ExportFrozenBundle
|
|
// 新侧对照:
|
|
// KG_DS02_BUNDLE=/path/dir/ds02-frozen-bundle.json go test -run TestDS02FrozenBundleIsolated
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"knowledge-governance/internal/libraryrelease"
|
|
)
|
|
|
|
type ds02BundleSeedRow struct {
|
|
Table string `json:"table"`
|
|
Row map[string]any `json:"row"`
|
|
}
|
|
|
|
type ds02BundleSeedDoc struct {
|
|
ID string `json:"id"`
|
|
Tenant uint64 `json:"tenant"`
|
|
KB string `json:"kb"`
|
|
Knowledge string `json:"knowledge"`
|
|
Content string `json:"content"`
|
|
}
|
|
|
|
type ds02FrozenBundle struct {
|
|
Bundle string `json:"bundle"`
|
|
Version int `json:"version"`
|
|
CreatedAt string `json:"created_at"`
|
|
Scope map[string]any `json:"scope"`
|
|
Sequences map[string][]any `json:"sequences"`
|
|
SourceRows map[string][]map[string]any `json:"source_rows"`
|
|
TargetRows []ds02BundleSeedRow `json:"target_rows"`
|
|
// OldAdapterRows 只给旧源码探针用:6fe98c9 在 resources 阶段按目标范围查资源行
|
|
// (它在 database 阶段不重写租户),所以目标必须先有这两行才能走到 wiki。
|
|
// 这是旧入口适配差异,不是业务输入;新侧不读这个字段。
|
|
OldAdapterRows []ds02BundleSeedRow `json:"old_adapter_target_rows"`
|
|
SeedDocs []ds02BundleSeedDoc `json:"seed_docs"`
|
|
SeedObjects map[string]string `json:"seed_objects"`
|
|
Vectors map[string][]float64 `json:"vectors"`
|
|
DDL []string `json:"ddl"`
|
|
}
|
|
|
|
// ds02BundleDDL 从保留的真实 v071 catalog 生成完整 DDL;旧/新隔离目标执行同一份。
|
|
func ds02BundleDDL(t *testing.T) []string {
|
|
t.Helper()
|
|
catalog := r11CatalogData(t)
|
|
ddl := []string{
|
|
`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`,
|
|
`CREATE EXTENSION IF NOT EXISTS pg_trgm`,
|
|
// 预备份的 schema 契约从这张台账读版本;bundle 必须自带,不能靠夹具残留。
|
|
`CREATE TABLE schema_migrations(version int,dirty bool)`,
|
|
`INSERT INTO schema_migrations VALUES(74,false)`,
|
|
// 旧源码的"库已结清"查询要读这两张任务表;同一份 DDL 两侧都建。
|
|
`CREATE TABLE task_pending_ops(tenant_id bigint,task_type text,scope text,scope_id text)`,
|
|
`CREATE TABLE task_dead_letters(tenant_id bigint,task_type text,scope text,scope_id text)`,
|
|
`CREATE TABLE knowledge_bases(id text,tenant_id bigint,indexing_strategy jsonb,deleted_at timestamptz)`,
|
|
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
|
|
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
|
|
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + ds02BucketPlaceholder + `"}',now())`,
|
|
}
|
|
for _, sequence := range catalog.Sequences {
|
|
ddl = append(ddl, fmt.Sprintf("CREATE SEQUENCE %s START %d", sequence.Name, sequence.Start))
|
|
}
|
|
for _, table := range productionLibraryTables {
|
|
defs := []string{}
|
|
for _, column := range catalog.Columns {
|
|
if column.Table != table {
|
|
continue
|
|
}
|
|
definition := pgxIdentifier(column.Name) + " " + column.Type
|
|
if column.Required {
|
|
definition += " NOT NULL"
|
|
}
|
|
if column.Default != "" {
|
|
definition += " DEFAULT " + column.Default
|
|
}
|
|
defs = append(defs, definition)
|
|
}
|
|
ddl = append(ddl, "CREATE TABLE "+table+"("+strings.Join(defs, ",")+")")
|
|
}
|
|
ddl = append(ddl, `ALTER SEQUENCE wiki_log_entries_id_seq OWNED BY wiki_log_entries.id`,
|
|
`CREATE TABLE resource_access_grants(id text,resource_id varchar(36))`)
|
|
constraints := append([]struct {
|
|
Table string `json:"table_name"`
|
|
Name string `json:"name"`
|
|
Definition string `json:"definition"`
|
|
}{}, catalog.Constraints...)
|
|
sort.SliceStable(constraints, func(i, j int) bool {
|
|
return !strings.HasPrefix(constraints[i].Definition, "FOREIGN KEY") && strings.HasPrefix(constraints[j].Definition, "FOREIGN KEY")
|
|
})
|
|
names := map[string]bool{}
|
|
for _, constraint := range constraints {
|
|
ddl = append(ddl, "ALTER TABLE "+constraint.Table+" ADD CONSTRAINT "+constraint.Name+" "+constraint.Definition)
|
|
names[constraint.Name] = true
|
|
}
|
|
for _, index := range catalog.Indexes {
|
|
if names[index.Name] {
|
|
continue
|
|
}
|
|
ddl = append(ddl, index.Definition)
|
|
}
|
|
return ddl
|
|
}
|
|
|
|
// pgxIdentifier 只处理本 catalog 里的普通小写列名。
|
|
func pgxIdentifier(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` }
|
|
|
|
func ds02BundleSequences() map[string][]any {
|
|
return map[string][]any{
|
|
"chunks_seq_id_seq": {int64(100000005), false},
|
|
"knowledge_tags_seq_id_seq": {int64(50), false},
|
|
"wiki_log_entries_id_seq": {int64(10), false},
|
|
}
|
|
}
|
|
|
|
func TestDS02ExportFrozenBundle(t *testing.T) {
|
|
root := os.Getenv("KG_DS02_BUNDLE_OUT")
|
|
if root == "" {
|
|
t.Skip("set KG_DS02_BUNDLE_OUT to write the frozen input bundle")
|
|
}
|
|
rows := ds02SourceRows()
|
|
// 逐表 hash,便于两侧各自核对同一份输入。
|
|
perTable := map[string]string{}
|
|
for table, list := range rows {
|
|
raw, err := json.Marshal(list)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(raw)
|
|
perTable[table] = hex.EncodeToString(sum[:])
|
|
}
|
|
bundle := ds02FrozenBundle{
|
|
Bundle: "ds02-frozen-input",
|
|
Version: 1,
|
|
CreatedAt: "2026-11-02T00:00:00Z",
|
|
Scope: map[string]any{"source_tenant": wsSourceTenant, "source_kb": wsSourceKB, "target_tenant": wsTargetTenant, "target_kb": wsTargetKB, "other_kb": wsOtherKB, "foreign_tenant": 20000},
|
|
Sequences: ds02BundleSequences(),
|
|
SourceRows: rows,
|
|
SeedObjects: func() map[string]string {
|
|
out := map[string]string{}
|
|
for id, body := range ds02Objects {
|
|
out[id] = body
|
|
}
|
|
for id, body := range ds02SeedObjects() {
|
|
out[id] = body
|
|
}
|
|
return out
|
|
}(),
|
|
Vectors: ds02Vectors,
|
|
DDL: ds02BundleDDL(t),
|
|
}
|
|
for _, row := range ds02SeedRows() {
|
|
bundle.TargetRows = append(bundle.TargetRows, ds02BundleSeedRow{Table: row.table, Row: row.row})
|
|
}
|
|
for _, doc := range ds02SeedDocs() {
|
|
bundle.SeedDocs = append(bundle.SeedDocs, ds02BundleSeedDoc{ID: doc.id, Tenant: doc.tenant, KB: doc.kb, Knowledge: doc.knowledge, Content: doc.content})
|
|
}
|
|
raw, err := json.MarshalIndent(bundle, "", " ")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(root, "ds02-frozen-bundle.json")
|
|
if err := os.WriteFile(path, append(raw, '\n'), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(append(raw, '\n'))
|
|
hash := hex.EncodeToString(sum[:])
|
|
if err := os.WriteFile(filepath.Join(root, "ds02-frozen-bundle.sha256"), []byte(hash+" ds02-frozen-bundle.json\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result := map[string]any{"bundle": path, "sha256": hash, "table_hashes": perTable,
|
|
"bytes": len(raw) + 1, "ddl_statements": len(bundle.DDL), "created_at": bundle.CreatedAt}
|
|
if run := os.Getenv("KG_R10_RUN"); run != "" {
|
|
r10Save(t, "bundle", result)
|
|
}
|
|
encoded, _ := json.MarshalIndent(result, "", " ")
|
|
t.Logf("%s", encoded)
|
|
}
|
|
|
|
func ds02LoadBundle(t *testing.T) (ds02FrozenBundle, string) {
|
|
t.Helper()
|
|
path := os.Getenv("KG_DS02_BUNDLE")
|
|
if path == "" {
|
|
t.Skip("set KG_DS02_BUNDLE to run the frozen-bundle comparison")
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read frozen bundle: %v", err)
|
|
}
|
|
sum := sha256.Sum256(raw)
|
|
hash := hex.EncodeToString(sum[:])
|
|
sidecar, err := os.ReadFile(strings.TrimSuffix(path, ".json") + ".sha256")
|
|
if err != nil {
|
|
t.Fatalf("read bundle sha256 sidecar: %v", err)
|
|
}
|
|
if !strings.HasPrefix(strings.TrimSpace(string(sidecar)), hash) {
|
|
t.Fatalf("bundle sha256 %s does not match sidecar %q", hash, strings.TrimSpace(string(sidecar)))
|
|
}
|
|
var bundle ds02FrozenBundle
|
|
if err := json.Unmarshal(raw, &bundle); err != nil {
|
|
t.Fatalf("parse frozen bundle: %v", err)
|
|
}
|
|
if bundle.Bundle != "ds02-frozen-input" || bundle.Version != 1 || len(bundle.DDL) == 0 {
|
|
t.Fatalf("unexpected frozen bundle header: %+v", bundle.Bundle)
|
|
}
|
|
return bundle, hash
|
|
}
|
|
|
|
// TestDS02FrozenBundleIsolated 用与旧源码探针同一份 bundle 初始化新目标并发布,
|
|
// 再用独立 oracle 检查;bundle hash 写进 raw,供与旧侧结果对齐。
|
|
func TestDS02FrozenBundleIsolated(t *testing.T) {
|
|
bundle, hash := ds02LoadBundle(t)
|
|
// 桶名在两个隔离目标里不同:fixture 在真正建 schema 时把 %BUCKET% 换成自己的桶。
|
|
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, ddl: bundle.DDL})
|
|
for name, value := range bundle.Sequences {
|
|
last := int64(value[0].(float64))
|
|
called := value[1].(bool)
|
|
r10Exec(t, fx, fmt.Sprintf("SELECT setval('%s',%d,%t)", name, last, called))
|
|
}
|
|
for id, body := range bundle.SeedObjects {
|
|
if id != "res-other" {
|
|
continue
|
|
}
|
|
path := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, id)
|
|
if err := fx.objects.s3.Put(fx.ctx, "minio", path, strings.NewReader(body), int64(len(body))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, row := range bundle.TargetRows {
|
|
if _, ok := ds02ScopeDecl[row.Table]; !ok {
|
|
t.Fatalf("bundle target row for undeclared table %s", row.Table)
|
|
}
|
|
r11Insert(t, fx, row.Table, ds02SeedRow(row.Row, fx.bucket))
|
|
}
|
|
for _, doc := range bundle.SeedDocs {
|
|
fx.indexDoc(t, doc.ID, doc.Tenant, doc.KB, doc.Knowledge, doc.Content)
|
|
}
|
|
spec := ds02Spec{rows: map[string][]map[string]any{}, vectors: bundle.Vectors, objects: map[string]string{}}
|
|
for table, rows := range bundle.SourceRows {
|
|
for _, row := range rows {
|
|
spec.rows[table] = append(spec.rows[table], ds02Canon(t, fx, table, row))
|
|
}
|
|
}
|
|
// 来源对象只取"来源资源行声明的那几个";res-other 是外围哨兵,不参与来源对象比对。
|
|
for _, row := range bundle.SourceRows["resources"] {
|
|
id := fmt.Sprint(row["id"])
|
|
body, ok := bundle.SeedObjects[id]
|
|
if !ok {
|
|
t.Fatalf("bundle has no bytes for source resource %s", id)
|
|
}
|
|
spec.objects[id] = body
|
|
}
|
|
preserved := map[string][]map[string]any{}
|
|
for _, table := range ds02TargetTables {
|
|
preserved[table] = ds02OtherRows(t, fx, table)
|
|
}
|
|
expectation := ds02Expected(t, fx, spec)
|
|
expected, paths := expectation.Rows, expectation.Paths
|
|
for _, table := range []string{"resources", "resource_bindings"} {
|
|
expected[table] = append(expected[table], ds02SameTenantRows(t, fx, table)...)
|
|
}
|
|
expected["knowledge_tag_relations"] = append(expected["knowledge_tag_relations"],
|
|
ds02Read(t, fx, `SELECT to_jsonb(x)::text FROM knowledge_tag_relations AS x
|
|
WHERE knowledge_id NOT IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`,
|
|
wsTargetTenant, wsTargetKB)...)
|
|
snapshot := ds02Seal(t, fx, spec)
|
|
recorder, inventory := ds02AttachEgress(t, fx)
|
|
target := fx.target(t)
|
|
backups := fx.backups(t, target)
|
|
recorder.setPhase("publish")
|
|
out := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, backups, nil)
|
|
recorder.setPhase("oracle-read")
|
|
r10Save(t, "bundle-identity", map[string]any{"path": os.Getenv("KG_DS02_BUNDLE"), "sha256": hash,
|
|
"source_tenant": bundle.Scope["source_tenant"], "target_tenant": bundle.Scope["target_tenant"],
|
|
"created_at": bundle.CreatedAt, "adapter_note": "DDL 与行内容相同;对象桶名以 %BUCKET% 占位,两侧各自替换。"})
|
|
r10Save(t, "outcome", out)
|
|
if out.State != "succeeded" {
|
|
t.Fatalf("new source must publish the frozen bundle: %+v", out)
|
|
}
|
|
if out.Backup.WriteSet == nil {
|
|
t.Fatal("write set missing")
|
|
}
|
|
ledger, err := LoadWriteSet(fx.archiveRoot, out.Backup.WriteSet.Digest)
|
|
if err != nil {
|
|
t.Fatalf("load write set: %v", err)
|
|
}
|
|
columns := ds02LocalColumns(t, fx, spec, expectation)
|
|
mappingDiffs := ds02CheckLocalIDMapping(columns, ledger.LocalIDs)
|
|
r10Save(t, "mapping-diffs", mappingDiffs)
|
|
if len(mappingDiffs) != 0 {
|
|
t.Fatalf("frozen-bundle publish fails the independent mapping oracle:\n%s", strings.Join(mappingDiffs, "\n"))
|
|
}
|
|
diffs := ds02Check(t, fx, spec, expected, paths, preserved, recorder, inventory)
|
|
r10Save(t, "green-diffs", diffs)
|
|
if len(diffs) != 0 {
|
|
t.Fatalf("frozen-bundle independent oracle disagrees:\n%s", strings.Join(diffs, "\n"))
|
|
}
|
|
r10Save(t, "sequences-after", ds02SequenceSummary(t, fx))
|
|
}
|