整库发布写入路径原先按 0.7.1 的固定表集合工作:0.9.0 没有 wiki_log_entries, Current/清理/编目/整实例指纹都会在不存在表上失败,跨版本闸门也因快照丢掉 SchemaVersion 而失效(未知版本被当作 74)。 改为在装配时解析目标 profile,表集合与清理顺序按解析结果;快照复制与范围改写 带上 SchemaVersion/Profile;跨版本发布显式拒绝并同时点名两个版本。单文档发布包 仍只支持 0.7.1,但改为明确拒绝并说明原因。 本机隔离验证:真实 0.9.0 schema 源库→导出→封存→写入干净 134 目标→逐表核对→ 写前镜像回滚;74→134 拒绝与 134 缺表不报错两个负例通过。make verify 17 项全绿。
This commit is contained in:
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照 0.1.378(发布写入支持 WeKnora 0.9.0)
|
||||
|
||||
### Added
|
||||
|
||||
- 整库发布写入路径支持 WeKnora 0.9.0(schema 134):生产整库目标在装配时解析目标
|
||||
profile,表集合、清理顺序、编目与整实例指纹都按解析结果取数——0.9.0 没有
|
||||
`wiki_log_entries`,不再对目标查询或删除该表;0.7.1 少一张必需表仍然失败,不靠
|
||||
「表不存在就跳过」放行。
|
||||
- `chunks.seq_id` 语义保持:发布时由目标序列分配,回滚时按写前镜像原值重放;
|
||||
0.9.0 新增的 NOT NULL 列走目标默认值,不用来源值硬填。
|
||||
- 跨版本发布(74 快照 → 134 目标,或反之)明确拒绝,错误同时点名两个版本号。
|
||||
|
||||
### Fixed
|
||||
|
||||
- 范围改写重建快照时会丢掉 `SchemaVersion`/`Profile`(归档重建、快照克隆两处):
|
||||
丢掉后下游按「未知版本 = 0.7.1」处理,0.9.0 的归档会被要求提供它根本没有的
|
||||
`wiki_log_entries`,跨版本闸门也形同不存在。现在快照复制一律带上 schema 契约。
|
||||
- 单文档发布包(`releasebundle`)仍只支持 0.7.1,但改为**明确拒绝并说明原因**:
|
||||
报错点名读到的版本与「0.9.0 应走整库发布路径」,不再与字段写错共用一句泛化错误。
|
||||
|
||||
### Tests
|
||||
|
||||
- 真实 0.9.0 schema + 索引 + 对象存储上的发布路径:导出 → 封存 → 真实
|
||||
`ProductionLibraryTarget` 写入干净 134 目标 → 逐表内容核对(Wiki 非空、资源指向
|
||||
目标桶/后端、`seq_id` 由目标序列分配)→ 用写前镜像恢复回滚并核对回到写前状态;
|
||||
另加 74→134 拒绝(含两个版本号)与「134 目标无 `wiki_log_entries` 不报错」的负例。
|
||||
|
||||
## 开发快照 0.1.376(WeKnora 0.9.0 整库发布 profile)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -3,6 +3,7 @@ package v071
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -63,8 +64,14 @@ func (exporter *DocumentExporter) Export(ctx context.Context, item releaseworkfl
|
||||
return nil, errors.New("invalid document release plan")
|
||||
}
|
||||
sourceContract, err := exporter.config.Contract.Contract(ctx, item)
|
||||
if err != nil || sourceContract.WeKnoraVersion != releasebundle.WeKnoraVersionV071 ||
|
||||
sourceContract.MigrationVersion != releasebundle.WeKnoraMigrationV071 || sourceContract.TenantID != exporter.config.TenantID ||
|
||||
if err == nil && (sourceContract.WeKnoraVersion != releasebundle.WeKnoraVersionV071 ||
|
||||
sourceContract.MigrationVersion != releasebundle.WeKnoraMigrationV071) {
|
||||
// 单文档发布包是 0.7.1 契约:读到的库是别的版本时必须说清楚原因,
|
||||
// 而不是和"租户/模型不对"共用一句泛化错误。
|
||||
return nil, fmt.Errorf("single-document release supports only v0.7.1 (74): source library declares %s (%d), so its rows and index documents must go through the library-level publish path",
|
||||
sourceContract.WeKnoraVersion, sourceContract.MigrationVersion)
|
||||
}
|
||||
if err != nil || sourceContract.TenantID != exporter.config.TenantID ||
|
||||
sourceContract.KnowledgeBaseID != item.TestKnowledgeBaseID || sourceContract.RetrieverEngine != "elasticsearch_v8" ||
|
||||
sourceContract.EmbeddingModel != exporter.config.EmbeddingModel || sourceContract.EmbeddingDimension != releasebundle.EmbeddingDimensionV071 {
|
||||
return nil, errors.New("source WeKnora release contract mismatch")
|
||||
|
||||
@@ -30,6 +30,8 @@ type ProductionBackupStore struct {
|
||||
maxDocuments int
|
||||
maxObjectByte int64
|
||||
now func() time.Time
|
||||
// tables 是目标库解析出的知识域表集合(0.9.0 没有 wiki_log_entries)。
|
||||
tables []string
|
||||
}
|
||||
|
||||
type ProductionBackupConfig struct {
|
||||
@@ -58,10 +60,16 @@ func NewProductionBackupStore(config ProductionBackupConfig) (*ProductionBackupS
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
// 预备份的表集合必须与写入/校验同源:按 0.7.1 固定清单会在 0.9.0 目标上
|
||||
// 查询不存在的 wiki_log_entries。
|
||||
schema, err := ResolveSchemaProfile(context.Background(), config.Pool, "prebackup target")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ProductionBackupStore{pool: config.Pool, search: config.Search, objects: config.Objects,
|
||||
archiveRoot: config.ArchiveRoot, scope: config.Scope, identity: config.Identity,
|
||||
compatibility: config.Compatibility, maxDocuments: config.MaxDocuments,
|
||||
maxObjectByte: config.MaxObjectByte, now: now}, nil
|
||||
maxObjectByte: config.MaxObjectByte, now: now, tables: schema.Tables}, nil
|
||||
}
|
||||
|
||||
// tableFingerprint 是一张表的"数量 + 内容摘要"指纹:同数量改写也会被识别。
|
||||
@@ -75,7 +83,7 @@ func (s *ProductionBackupStore) fingerprints(ctx context.Context) (map[string]ta
|
||||
result := map[string]tableFingerprint{}
|
||||
exporter := PostgresExporter{db: s.pool}
|
||||
var bodies, bindings []json.RawMessage
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range s.tables {
|
||||
if table == "resources" {
|
||||
continue
|
||||
}
|
||||
@@ -267,7 +275,7 @@ func (s *ProductionBackupStore) objectVersions(ctx context.Context, pg PostgresS
|
||||
|
||||
func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) {
|
||||
counts := map[string]int64{}
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range s.tables {
|
||||
var count int64
|
||||
if err := s.pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`,
|
||||
table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID).Scan(&count); err != nil {
|
||||
|
||||
@@ -30,7 +30,7 @@ func exportLibraryBeforeImage(ctx context.Context, db librarySnapshotDB, identit
|
||||
if table == "resources" {
|
||||
continue
|
||||
}
|
||||
rows, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s ORDER BY (%s)::text`, table, table, scopeWhere(table), identityExpr(table)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
rows, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s ORDER BY (%s)::text`, table, table, scopeWhereFor(profile.Tables, table), identityExpr(table)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return PostgresSnapshot{}, err
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func (t *ProductionLibraryTarget) preflightNaturalKeys(ctx context.Context, arch
|
||||
targetFilter := predicate
|
||||
args := []any{raw}
|
||||
if part.Name != "resources" {
|
||||
scope := strings.ReplaceAll(strings.ReplaceAll(scopeWhere(part.Name), "$2", "$3"), "$1", "$2")
|
||||
scope := strings.ReplaceAll(strings.ReplaceAll(t.scopeWhere(part.Name), "$2", "$3"), "$1", "$2")
|
||||
targetFilter += " AND NOT (" + scope + ")"
|
||||
args = append(args, t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/libraryprofile"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
@@ -85,13 +86,22 @@ func (config ProductionLibraryConfig) validate(scope ProductionScope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 依赖顺序:插入按正序,删除按倒序。
|
||||
// 真实 Schema 有外键(resource_bindings.resource_id → resources.id),
|
||||
// 所以父表 resources 必须排在子表 resource_bindings 之前。
|
||||
var productionLibraryTables = []string{
|
||||
"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations",
|
||||
"resources", "resource_bindings", "wiki_folders", "wiki_pages",
|
||||
"wiki_page_issues", "wiki_log_entries",
|
||||
// productionLibraryTables 是 0.7.1 的知识域表集合(依赖顺序:插入按正序,删除按
|
||||
// 倒序;真实 Schema 有外键 resource_bindings.resource_id → resources.id,父表
|
||||
// resources 必须排在子表 resource_bindings 之前)。
|
||||
//
|
||||
// 它**不是**本包整库读写的表清单来源:真实路径一律用 ResolveSchemaProfile 解析出的
|
||||
// Tables——0.9.0 没有 wiki_log_entries,任何一条固定清单的查询都会在真库上失败,而
|
||||
// "表不存在就跳过"又会让 0.7.1 少一张必需表也悄悄通过。这里只留给明确 74-only 的
|
||||
// 入口(编目、残留复核、0.7.1 夹具),定义直接取自 profile 声明,避免两份清单走偏。
|
||||
var productionLibraryTables = legacyKnowledgeTables()
|
||||
|
||||
func legacyKnowledgeTables() []string {
|
||||
profile, err := libraryprofile.For(libraryprofile.LegacyVersion)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return append([]string(nil), profile.RequiredTables...)
|
||||
}
|
||||
|
||||
type ProductionLibraryTarget struct {
|
||||
@@ -100,6 +110,11 @@ type ProductionLibraryTarget struct {
|
||||
scope ProductionScope
|
||||
search *SearchExporter
|
||||
now func() time.Time
|
||||
// schema / tables 是目标库解析出的版本声明与"实际存在的知识域表"。写入、校验、
|
||||
// 清理、编目全部按它取数:0.9.0 缺 wiki_log_entries 时不得查询/删除该表,
|
||||
// 而 0.7.1 少一张必需表必须失败,不能靠"表不存在就跳过"。
|
||||
schema SchemaProfile
|
||||
tables []string
|
||||
// storageBackend / storageProvider 是本次写入使用的目标存储后端与提供方
|
||||
// (写入与校验共用,缺一不可:只换桶不换提供方,目标取文件仍会走来源实例的路)。
|
||||
storageBackend string
|
||||
@@ -215,7 +230,15 @@ func NewProductionLibraryTarget(ctx context.Context, config ProductionLibraryCon
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &ProductionLibraryTarget{pool: pool, config: config, scope: scope, search: search, now: now}, nil
|
||||
// 目标版本在装配时定型:未知版本/dirty/缺表在这里就失败,不能等到第一条写入
|
||||
// 才发现"目标其实没有这张表"。
|
||||
schema, err := ResolveSchemaProfile(ctx, pool, "publish target")
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &ProductionLibraryTarget{pool: pool, config: config, scope: scope, search: search, now: now,
|
||||
schema: schema, tables: schema.Tables}, nil
|
||||
}
|
||||
|
||||
func (t *ProductionLibraryTarget) Close() { t.pool.Close() }
|
||||
@@ -235,7 +258,7 @@ func (t *ProductionLibraryTarget) Identity() PostgresIdentity {
|
||||
func (t *ProductionLibraryTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
|
||||
state := libraryrelease.CurrentState{Counts: map[string]int64{}, Digests: map[string]string{},
|
||||
CapturedAt: t.now().UTC()}
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range t.tables {
|
||||
count, digest, err := t.tableState(ctx, table)
|
||||
if err != nil {
|
||||
return libraryrelease.CurrentState{}, err
|
||||
@@ -255,10 +278,19 @@ func (t *ProductionLibraryTarget) Current(ctx context.Context) (libraryrelease.C
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// scopeWhere 返回"这张表里属于本知识库的行"的过滤条件($1=tenant,$2=kb)。
|
||||
// scopeWhere 是 74-only 入口(残留复核、0.7.1 夹具)使用的固定表集合版本。
|
||||
func scopeWhere(table string) string { return scopeWhereFor(productionLibraryTables, table) }
|
||||
|
||||
// scopeWhere 用本次目标解析出的表集合。
|
||||
func (t *ProductionLibraryTarget) scopeWhere(table string) string {
|
||||
return scopeWhereFor(t.tables, table)
|
||||
}
|
||||
|
||||
// scopeWhereFor 返回"这张表里属于本知识库的行"的过滤条件($1=tenant,$2=kb)。
|
||||
// 不同表的归属列不同:关系表与绑定表没有 knowledge_base_id,必须走父表判定,
|
||||
// 否则要么删错、要么把别的知识库数据算进来。
|
||||
func scopeWhere(table string) string {
|
||||
// 否则要么删错、要么把别的知识库数据算进来。表集合来自解析出的 profile:
|
||||
// 0.9.0 的资源正文引用子查询不能提到不存在的 wiki_log_entries。
|
||||
func scopeWhereFor(tables []string, table string) string {
|
||||
switch table {
|
||||
case "knowledge_tag_relations":
|
||||
// 真实 WeKnora 的这张表只有 knowledge_id/tag_id/created_at,
|
||||
@@ -283,7 +315,7 @@ func scopeWhere(table string) string {
|
||||
SELECT 1 FROM resource_bindings b
|
||||
JOIN chunks c ON c.id=b.owner_id AND c.tenant_id=b.tenant_id
|
||||
WHERE b.resource_id=resources.id AND b.tenant_id=$1 AND b.owner_type='chunk' AND c.knowledge_base_id=$2)
|
||||
OR EXISTS (` + libraryResourceBodyReferenceSQL("c.tenant_id=$1 AND c.knowledge_base_id=$2") + `))`
|
||||
OR EXISTS (` + libraryResourceBodyReferenceSQL(tables, "c.tenant_id=$1 AND c.knowledge_base_id=$2") + `))`
|
||||
default:
|
||||
return "tenant_id=$1 AND knowledge_base_id=$2"
|
||||
}
|
||||
@@ -308,12 +340,12 @@ func (t *ProductionLibraryTarget) tableState(ctx context.Context, table string)
|
||||
// scan used by the legacy scope predicate.
|
||||
exporter := PostgresExporter{db: t.pool}
|
||||
var bodies, bindings []json.RawMessage
|
||||
for _, name := range productionLibraryTables {
|
||||
for _, name := range t.tables {
|
||||
if name == "resources" {
|
||||
continue
|
||||
}
|
||||
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
|
||||
name, name, scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
name, name, t.scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("读取生产 %s 失败", table)
|
||||
}
|
||||
@@ -335,7 +367,7 @@ func (t *ProductionLibraryTarget) tableState(ctx context.Context, table string)
|
||||
// canonicalJSON on text and fail on every non-empty table.
|
||||
if table != "resources" {
|
||||
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT %s FROM %s WHERE %s ORDER BY 1`,
|
||||
"("+identityExpr(table)+")::text", table, scopeWhere(table)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
"("+identityExpr(table)+")::text", table, t.scopeWhere(table)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("读取生产 %s 失败", table)
|
||||
}
|
||||
@@ -404,6 +436,9 @@ func (t *ProductionLibraryTarget) Apply(ctx context.Context, stage libraryreleas
|
||||
if err != nil {
|
||||
return errors.New("预发布快照不可读")
|
||||
}
|
||||
if err := requireSameProfile(manifest.Compatibility.SchemaVersion, t.schema, "publish"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -444,8 +479,8 @@ func (t *ProductionLibraryTarget) importRows(ctx context.Context, archive *libra
|
||||
return errors.New("开启生产事务失败")
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
for index := len(productionLibraryTables) - 1; index >= 0; index-- {
|
||||
table := productionLibraryTables[index]
|
||||
for index := len(t.tables) - 1; index >= 0; index-- {
|
||||
table := t.tables[index]
|
||||
if _, ok := parts[table]; !ok {
|
||||
continue
|
||||
}
|
||||
@@ -453,7 +488,7 @@ func (t *ProductionLibraryTarget) importRows(ctx context.Context, archive *libra
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range t.tables {
|
||||
part, ok := parts[table]
|
||||
if !ok {
|
||||
continue
|
||||
@@ -473,7 +508,7 @@ func (t *ProductionLibraryTarget) deleteScope(ctx context.Context, tx pgx.Tx, ta
|
||||
// 资源可能被多个知识库共享:这里不删,只做插入/更新。
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf(`DELETE FROM %s WHERE %s`, table, scopeWhere(table))
|
||||
statement := fmt.Sprintf(`DELETE FROM %s WHERE %s`, table, t.scopeWhere(table))
|
||||
if _, err := tx.Exec(ctx, statement, t.scope.TenantID, t.scope.KnowledgeBaseID); err != nil {
|
||||
return fmt.Errorf("清理生产 %s 失败", table)
|
||||
}
|
||||
@@ -810,6 +845,9 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele
|
||||
if err != nil {
|
||||
return errors.New("预发布快照不可读")
|
||||
}
|
||||
if err := requireSameProfile(manifest.Compatibility.SchemaVersion, t.schema, "publish verify"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1292,7 +1330,7 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
|
||||
if err := t.AdoptWriteSet(*backup.WriteSet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureRestorableResourceReferences(ctx, t.pool, *backup.WriteSet); err != nil {
|
||||
if err := ensureRestorableResourceReferences(ctx, t.pool, t.tables, *backup.WriteSet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureNoResourceGrants(ctx, t.pool, backup.WriteSet.CreatedIDs("resources")); err != nil {
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// 0.9.0 整库发布目标的隔离夹具身份。来源=0.9.0 测试实例,目标=另一租户/知识库,
|
||||
// 这样"资源定位符换桶/后端/租户段"与"行改写目标范围"都被真实覆盖。
|
||||
const (
|
||||
v090TargetTenant = 10001
|
||||
v090TargetKB = "kb-v090-prod"
|
||||
v090TargetBackend = "backend-v090-target"
|
||||
v090TargetBucket = "kg-v090-target-bucket"
|
||||
v090SourceBackend = "backend-v090-source"
|
||||
// 来源定位符是目标应用真实使用的形状:storage://<backend>/<scheme>://<bucket>/<key>。
|
||||
v090SourceLocator = "storage://backend-v090-source/oss://kg-v090-source-bucket/weknora/10000/rs-v090-1/kn.bin"
|
||||
)
|
||||
|
||||
// v090ObjectBytes 按"提供方|路径"供字节,既能给封存读来源对象,也能给目标写回。
|
||||
type v090ObjectBytes map[string][]byte
|
||||
|
||||
func (o v090ObjectBytes) Open(_ context.Context, provider, physicalPath string) (io.ReadCloser, error) {
|
||||
content, ok := o[provider+"|"+physicalPath]
|
||||
if !ok {
|
||||
return nil, errors.New("fixture object missing: " + provider + "|" + physicalPath)
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(content)), nil
|
||||
}
|
||||
|
||||
// TestV090ProductionTargetPublishIsolated 是 0.9.0(schema 134)**发布写入路径**的
|
||||
// 端到端证据:真实导出 → 真实封存 → 真实 ProductionLibraryTarget 写进干净 134 目标
|
||||
// → 逐表核对 → 用写前镜像恢复回滚 → 再核对回到写前状态。
|
||||
//
|
||||
// 这里不允许打桩 PG/索引/对象:134 与 74 的差别(没有 wiki_log_entries、chunks 多
|
||||
// NOT NULL 列、seq_id 由目标序列分配)只有在真 schema 上才会暴露。因此默认 CI 跳过,
|
||||
// 在 scripts/test-scope.json 里声明为可选未运行,并写明理由。
|
||||
func TestV090ProductionTargetPublishIsolated(t *testing.T) {
|
||||
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
|
||||
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" || base == "" {
|
||||
t.Skip("requires isolated PG plus ES opt-in (KG_LIBRARY_TEST_ISOLATED=1, KG_LIBRARY_TEST_ELASTIC_URL)")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
src := v090Pool(t, "KG_V090_SOURCE_DSN")
|
||||
dst := v090Pool(t, "KG_V090_TARGET_DSN")
|
||||
identity := seedV090(t, ctx, src)
|
||||
// 资源行必须带目标实例真实可解析的定位符,否则"换桶/换后端"这条改写没有被覆盖。
|
||||
if _, err := src.Exec(ctx, `UPDATE resources SET provider='oss', storage_backend_id=$2,
|
||||
physical_path=$3, location_hash=$4 WHERE id=$1`,
|
||||
v090Resource, v090SourceBackend, v090SourceLocator, resourceLocationHash(v090SourceLocator)); err != nil {
|
||||
t.Fatalf("retarget source resource locator: %v", err)
|
||||
}
|
||||
|
||||
source, err := ExportLibrary(ctx, src, identity)
|
||||
if err != nil {
|
||||
t.Fatalf("export 0.9.0 library: %v", err)
|
||||
}
|
||||
if source.SchemaVersion != 134 || source.Profile != "v090" {
|
||||
t.Fatalf("snapshot must record the 0.9.0 profile, got %d/%q", source.SchemaVersion, source.Profile)
|
||||
}
|
||||
if tablePresent(source, "wiki_log_entries") {
|
||||
t.Fatal("0.9.0 export must not produce a wiki_log_entries table")
|
||||
}
|
||||
|
||||
token := make([]byte, 8)
|
||||
if _, err := rand.Read(token); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := "kg_v090_publish_" + hex.EncodeToString(token)
|
||||
client := &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}}
|
||||
call := func(method, path, body string, want int) {
|
||||
t.Helper()
|
||||
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("fixture HTTP: %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != want {
|
||||
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
|
||||
}
|
||||
}
|
||||
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200)
|
||||
defer func() {
|
||||
call("DELETE", "/"+index, "", 200)
|
||||
call("HEAD", "/"+index, "", 404)
|
||||
}()
|
||||
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: base, Index: index, PageSize: 100, HTTPClient: client})
|
||||
if err != nil {
|
||||
t.Fatalf("search exporter: %v", err)
|
||||
}
|
||||
mappingSHA, err := exporter.MappingSHA256(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("index mapping digest: %v", err)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
if err := os.Chmod(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, err := librarysnapshot.Open(root, 64<<20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
objectBytes := []byte("fixture-object-bytes")
|
||||
objects := v090ObjectBytes{"oss|" + v090SourceLocator: objectBytes}
|
||||
contract := librarysnapshot.Compatibility{
|
||||
WeKnoraVersion: "v0.9.0", SchemaVersion: 134,
|
||||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3",
|
||||
EmbeddingDimension: 1024, IndexMappingSHA256: mappingSHA,
|
||||
}
|
||||
sealed, err := ArchiveLibraryCandidate(ctx, archive, "internal", identity, contract, source,
|
||||
v090PublishIndexSnapshot(t, source, mappingSHA), objects, time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("seal 0.9.0 archive: %v", err)
|
||||
}
|
||||
manifest, err := archive.Load(sealed)
|
||||
if err != nil {
|
||||
t.Fatalf("load sealed manifest: %v", err)
|
||||
}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Name == "wiki_log_entries" {
|
||||
t.Fatal("0.9.0 archive must not contain a wiki_log_entries part")
|
||||
}
|
||||
}
|
||||
|
||||
// 目标库:干净 134 schema + KB/模型绑定 + 目标存储后端(桶名用于解析定位符)。
|
||||
if err := seedV090Target(t, ctx, dst); err != nil {
|
||||
t.Fatalf("seed target binding: %v", err)
|
||||
}
|
||||
store := newMemoryObjectStore(nil)
|
||||
target, err := NewProductionLibraryTarget(ctx, ProductionLibraryConfig{
|
||||
Postgres: os.Getenv("KG_V090_TARGET_DSN"), SearchURL: base, SearchIndex: index,
|
||||
Objects: store, ObjectsReader: store, ArchiveRoot: root,
|
||||
ObjectsBucket: v090TargetBucket, MaxDocuments: 100, HTTPClient: client,
|
||||
}, ProductionScope{TenantID: v090TargetTenant, KnowledgeBaseID: v090TargetKB})
|
||||
if err != nil {
|
||||
t.Fatalf("target: %v", err)
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
backups, err := NewProductionBackupStore(ProductionBackupConfig{
|
||||
Pool: target.Pool(), Search: target.Search(), Objects: store, ArchiveRoot: root,
|
||||
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100, Compatibility: contract,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("backup store: %v", err)
|
||||
}
|
||||
|
||||
// 现状:真实 Current() 必须在 134 上可用,且不得去查 0.9.0 里不存在的 wiki_log_entries。
|
||||
before, err := target.Current(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("current target state on 0.9.0: %v", err)
|
||||
}
|
||||
if _, present := before.Counts["wiki_log_entries"]; present {
|
||||
t.Fatalf("0.9.0 target state must not include wiki_log_entries: %+v", before.Counts)
|
||||
}
|
||||
backup, err := backups.Prepare(ctx, sealed, before)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare before-image: %v", err)
|
||||
}
|
||||
backup.WriteSet = planLedgerWithUniverse(t, target, sealed, backup.SnapshotID, backup.Universe)
|
||||
|
||||
for _, stage := range libraryrelease.Stages() {
|
||||
if err := target.Apply(ctx, stage, sealed); err != nil {
|
||||
t.Fatalf("apply %s on 0.9.0 target: %v", stage, err)
|
||||
}
|
||||
}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: sealed}); err != nil {
|
||||
t.Fatalf("verify after 0.9.0 publish: %v", err)
|
||||
}
|
||||
|
||||
assertV090Published(t, ctx, dst, target, source)
|
||||
|
||||
// 跨版本必须拒绝,并且错误同时点名两个版本号。用另一个没有冻结计划的目标实例:
|
||||
// 已接管写集合的执行目标会先被"快照与计划不符"挡住,那不是版本闸门。
|
||||
legacy := sealTargetFixture(t, archive, "internal", []string{"doc-safe"}, []string{"chunk-safe"}, "wiki-safe")
|
||||
legacyTarget, err := NewProductionLibraryTarget(ctx, ProductionLibraryConfig{
|
||||
Postgres: os.Getenv("KG_V090_TARGET_DSN"), SearchURL: base, SearchIndex: index,
|
||||
Objects: store, ObjectsReader: store, ArchiveRoot: root,
|
||||
ObjectsBucket: v090TargetBucket, MaxDocuments: 100, HTTPClient: client,
|
||||
}, ProductionScope{TenantID: v090TargetTenant, KnowledgeBaseID: v090TargetKB})
|
||||
if err != nil {
|
||||
t.Fatalf("legacy target: %v", err)
|
||||
}
|
||||
defer legacyTarget.Close()
|
||||
err = legacyTarget.Apply(ctx, libraryrelease.StageDatabase, legacy)
|
||||
if err == nil || !strings.Contains(err.Error(), "74") || !strings.Contains(err.Error(), "134") {
|
||||
t.Fatalf("cross-version publish must be refused naming both versions, got %v", err)
|
||||
}
|
||||
if err := legacyTarget.Verify(ctx, libraryrelease.Expectation{SnapshotID: legacy}); err == nil ||
|
||||
!strings.Contains(err.Error(), "74") || !strings.Contains(err.Error(), "134") {
|
||||
t.Fatalf("cross-version verify must be refused naming both versions, got %v", err)
|
||||
}
|
||||
|
||||
// 写前镜像恢复:目标必须回到写之前的样子(0.9.0 表集合)。
|
||||
if err := target.Restore(ctx, backup); err != nil {
|
||||
t.Fatalf("restore from before-image: %v", err)
|
||||
}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backup.SnapshotID}); err != nil {
|
||||
t.Fatalf("verify after restore: %v", err)
|
||||
}
|
||||
for _, table := range []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations",
|
||||
"wiki_pages", "wiki_folders", "wiki_page_issues"} {
|
||||
statement := `SELECT count(*) FROM ` + table + ` WHERE tenant_id=$1 AND knowledge_base_id=$2`
|
||||
args := []any{v090TargetTenant, v090TargetKB}
|
||||
if table == "knowledge_tag_relations" {
|
||||
// 关系表没有租户/知识库列:归属只能走父表。
|
||||
statement = `SELECT count(*) FROM knowledge_tag_relations r WHERE r.knowledge_id IN
|
||||
(SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)`
|
||||
}
|
||||
var count int64
|
||||
if err := dst.QueryRow(ctx, statement, args...).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("rollback left %d rows in %s", count, table)
|
||||
}
|
||||
}
|
||||
var restoredResource int64
|
||||
if err := dst.QueryRow(ctx, `SELECT count(*) FROM resources WHERE tenant_id=$1`, v090TargetTenant).Scan(&restoredResource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restoredResource != 0 {
|
||||
t.Fatalf("rollback left %d published resources in the target tenant", restoredResource)
|
||||
}
|
||||
t.Logf("v0.9.0 publish verified: snapshot=%s backup=%s bytes=%d", sealed, backup.SnapshotID, backup.Bytes)
|
||||
}
|
||||
|
||||
// assertV090Published 逐表核对目标与来源内容一致(身份列走目标隔离映射,不作为内容比对)。
|
||||
func assertV090Published(t *testing.T, ctx context.Context, dst *pgxpool.Pool, target *ProductionLibraryTarget, source PostgresSnapshot) {
|
||||
t.Helper()
|
||||
wantChunks := int64(source.Table("chunks").RowCount)
|
||||
var chunks int64
|
||||
if err := dst.QueryRow(ctx, `SELECT count(*) FROM chunks WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&chunks); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chunks != wantChunks || chunks < 2 {
|
||||
t.Fatalf("chunks in target = %d, want %d", chunks, wantChunks)
|
||||
}
|
||||
// 目标隔离映射:写入的是目标身份,不是来源身份。
|
||||
for _, chunkID := range []string{v090ChunkOne, v090ChunkTwo} {
|
||||
var content string
|
||||
if err := dst.QueryRow(ctx, `SELECT content FROM chunks WHERE id=$1`,
|
||||
expectedTargetID(v090TargetTenant, v090TargetKB, "chunks", chunkID)).Scan(&content); err != nil {
|
||||
t.Fatalf("mapped chunk %s missing: %v", chunkID, err)
|
||||
}
|
||||
if content == "" {
|
||||
t.Fatalf("mapped chunk %s has empty content", chunkID)
|
||||
}
|
||||
}
|
||||
// 0.9.0 新增 NOT NULL 列走目标默认值;seq_id 由目标序列分配(来源值不得搬过来)。
|
||||
var defaults, assigned int64
|
||||
if err := dst.QueryRow(ctx, `SELECT count(*) FROM chunks WHERE tenant_id=$1 AND knowledge_base_id=$2
|
||||
AND source_content IS NOT NULL AND content_revision IS NOT NULL AND index_status IS NOT NULL
|
||||
AND context_header IS NOT NULL AND last_editor_id IS NOT NULL AND seq_id IS NOT NULL`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&defaults); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if defaults != wantChunks {
|
||||
t.Fatalf("0.9.0 NOT NULL columns unsatisfied on %d/%d chunks", defaults, wantChunks)
|
||||
}
|
||||
if err := dst.QueryRow(ctx, `SELECT count(DISTINCT seq_id) FROM chunks WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&assigned); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assigned != wantChunks {
|
||||
t.Fatalf("target seq_id must be assigned per chunk, got %d distinct for %d chunks", assigned, wantChunks)
|
||||
}
|
||||
|
||||
// Wiki 必须非空:空表通过等于没验证 wiki 部件。
|
||||
var pages int64
|
||||
var content, slug string
|
||||
if err := dst.QueryRow(ctx, `SELECT count(*) FROM wiki_pages WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&pages); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pages != 1 {
|
||||
t.Fatalf("wiki_pages in target = %d, want 1", pages)
|
||||
}
|
||||
if err := dst.QueryRow(ctx, `SELECT content, slug FROM wiki_pages WHERE id=$1`,
|
||||
expectedTargetID(v090TargetTenant, v090TargetKB, "wiki_pages", v090Page)).Scan(&content, &slug); err != nil {
|
||||
t.Fatalf("published wiki page missing: %v", err)
|
||||
}
|
||||
if content != "wiki 正文" || slug != "fixture-page" {
|
||||
t.Fatalf("wiki page content changed: content=%q slug=%q", content, slug)
|
||||
}
|
||||
|
||||
var tags int64
|
||||
if err := dst.QueryRow(ctx, `SELECT count(*) FROM knowledge_tags WHERE tenant_id=$1 AND knowledge_base_id=$2 AND name='夹具标签'`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&tags); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags != 1 {
|
||||
t.Fatalf("knowledge_tags in target = %d, want 1", tags)
|
||||
}
|
||||
|
||||
var title, model string
|
||||
if err := dst.QueryRow(ctx, `SELECT title, embedding_model_id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||||
v090TargetTenant, v090TargetKB).Scan(&title, &model); err != nil {
|
||||
t.Fatalf("published knowledge missing: %v", err)
|
||||
}
|
||||
if title != "0.9.0 夹具" || model != "bge-m3-v090" {
|
||||
t.Fatalf("knowledge content changed: title=%q model=%q", title, model)
|
||||
}
|
||||
|
||||
// 资源必须指向目标桶/后端,且定位符哈希按最终路径重算。
|
||||
wantLocator := "storage://" + v090TargetBackend + "/minio://" + v090TargetBucket + "/" +
|
||||
strconv.FormatUint(v090TargetTenant, 10) + "/" + v090Resource + "/kn.bin"
|
||||
var provider, backend, locator, locationHash string
|
||||
var size int64
|
||||
if err := dst.QueryRow(ctx, `SELECT provider, coalesce(storage_backend_id,''), physical_path, location_hash, size
|
||||
FROM resources WHERE tenant_id=$1`, v090TargetTenant).
|
||||
Scan(&provider, &backend, &locator, &locationHash, &size); err != nil {
|
||||
t.Fatalf("published resource missing: %v", err)
|
||||
}
|
||||
if provider != "minio" || backend != v090TargetBackend {
|
||||
t.Fatalf("resource must point at the target backend: provider=%q backend=%q", provider, backend)
|
||||
}
|
||||
if locator != wantLocator {
|
||||
t.Fatalf("resource locator not retargeted:\n got %s\nwant %s", locator, wantLocator)
|
||||
}
|
||||
if locationHash != resourceLocationHash(wantLocator) {
|
||||
t.Fatalf("resource location_hash must hash the target locator")
|
||||
}
|
||||
if size != int64(len("fixture-object-bytes")) {
|
||||
t.Fatalf("resource size changed: %d", size)
|
||||
}
|
||||
// 目标对象存储里必须真的有这份字节(写入阶段按目标定位符上传)。
|
||||
if _, err := target.Objects().Open(ctx, provider, locator); err != nil {
|
||||
t.Fatalf("published object unreadable in target store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// v090PublishIndexSnapshot 按导出分块合成索引快照:这批文档走的是与线上同一套
|
||||
// 映射/摘要校验,不能凭空绕过。
|
||||
func v090PublishIndexSnapshot(t *testing.T, pg PostgresSnapshot, mappingSHA string) SearchSnapshot {
|
||||
t.Helper()
|
||||
var documents []SearchDocument
|
||||
for _, raw := range pg.Table("chunks").Rows {
|
||||
var chunk struct {
|
||||
ID string `json:"id"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &chunk); err != nil {
|
||||
t.Fatalf("decode chunk: %v", err)
|
||||
}
|
||||
source, err := json.Marshal(map[string]any{
|
||||
"chunk_id": chunk.ID, "tenant_id": chunk.TenantID,
|
||||
"knowledge_base_id": chunk.KnowledgeBaseID, "knowledge_id": chunk.KnowledgeID,
|
||||
"content": chunk.Content, "source_id": chunk.ID, "source_type": 0,
|
||||
"embedding": []float64{0.1, 0.2},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("encode index document: %v", err)
|
||||
}
|
||||
documents = append(documents, SearchDocument{ID: "doc-" + chunk.ID, Source: source})
|
||||
}
|
||||
digest, err := searchDocumentsDigest(documents)
|
||||
if err != nil {
|
||||
t.Fatalf("index digest: %v", err)
|
||||
}
|
||||
return SearchSnapshot{Documents: documents, MappingSHA256: mappingSHA, DocumentsSHA256: digest}
|
||||
}
|
||||
|
||||
// seedV090Target 在目标库造 KB/模型绑定与目标存储后端:发布写的是内容行,不是 KB 定义。
|
||||
func seedV090Target(t *testing.T, ctx context.Context, db *pgxpool.Pool) error {
|
||||
t.Helper()
|
||||
for _, statement := range []string{
|
||||
`INSERT INTO models(id,tenant_id,name,type,source,parameters)
|
||||
VALUES ('bge-m3-v090',$1,'bge-m3','embedding','local','{"embedding_parameters":{"dimension":1024}}'::jsonb)`,
|
||||
`INSERT INTO knowledge_bases(id,name,tenant_id,embedding_model_id,summary_model_id,indexing_strategy)
|
||||
VALUES ('` + v090TargetKB + `','v090 publish target',$1,'bge-m3-v090','bge-m3-v090','{"graph_enabled":false}'::jsonb)`,
|
||||
`INSERT INTO storage_backends(id,tenant_id,name,provider,config)
|
||||
VALUES ('` + v090TargetBackend + `',$1,'v090 target','minio','{"bucket_name":"` + v090TargetBucket + `"}'::jsonb)`,
|
||||
} {
|
||||
if _, err := db.Exec(ctx, strings.ReplaceAll(statement, "$1", strconv.Itoa(v090TargetTenant))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"knowledge-governance/internal/libraryprofile"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
@@ -118,7 +119,12 @@ func libraryResourceHandles(rows []json.RawMessage) ([]string, error) {
|
||||
// Revalidate frozen source content at the actual planning boundary. A sealed
|
||||
// manifest proves bytes, not that every row has a parent or a supported owner.
|
||||
func validateArchivedLibraryClosure(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
|
||||
var pg PostgresSnapshot
|
||||
// 快照的 schema 契约来自 manifest:不带上版本号,重建出来的快照就是"版本未知",
|
||||
// 而未知版本按 0.7.1 校验——0.9.0 的归档会被要求提供它根本没有的 wiki_log_entries。
|
||||
pg := PostgresSnapshot{SchemaVersion: manifest.Compatibility.SchemaVersion}
|
||||
if profile, err := libraryprofile.For(pg.SchemaVersion); err == nil {
|
||||
pg.Profile = profile.Name
|
||||
}
|
||||
es := SearchSnapshot{MappingSHA256: manifest.Compatibility.IndexMappingSHA256}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" {
|
||||
|
||||
@@ -152,8 +152,8 @@ func (t *ProductionLibraryTarget) ClearForeignScope(ctx context.Context, source
|
||||
return report, nil
|
||||
}
|
||||
// 删除顺序:子表先、父表后(resource_bindings → resources,chunks → knowledges)。
|
||||
for index := len(productionLibraryTables) - 1; index >= 0; index-- {
|
||||
table := productionLibraryTables[index]
|
||||
for index := len(t.tables) - 1; index >= 0; index-- {
|
||||
table := t.tables[index]
|
||||
ids, ok := identities[table]
|
||||
if !ok || len(ids) == 0 {
|
||||
continue
|
||||
@@ -216,7 +216,7 @@ func (t *ProductionLibraryTarget) legacyResourceRow(row map[string]json.RawMessa
|
||||
// VerifyResidueCleared 复核清理结果:来源身份不再存在、其它租户的解释得清、
|
||||
// 且(可选)目标内容与预备份完全一致。
|
||||
func (t *ProductionLibraryTarget) VerifyResidueCleared(ctx context.Context, source ProductionScope) error {
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range t.tables {
|
||||
if !tableHasTenant(table) {
|
||||
continue
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func (t *ProductionLibraryTarget) VerifyResidueCleared(ctx context.Context, sour
|
||||
// ForeignScopes 列出目标里所有"不是目标租户"的租户,用于人工确认没有别的泄漏。
|
||||
func (t *ProductionLibraryTarget) ForeignScopes(ctx context.Context) (map[string]string, error) {
|
||||
foreign := map[string]string{}
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range t.tables {
|
||||
if !tableHasTenant(table) {
|
||||
continue
|
||||
}
|
||||
@@ -272,7 +272,7 @@ func (t *ProductionLibraryTarget) foreignScopeRows(ctx context.Context, table st
|
||||
if !tableHasTenant(table) {
|
||||
return nil, nil
|
||||
}
|
||||
statement := fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table))
|
||||
statement := fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, t.scopeWhere(table))
|
||||
arguments := []any{source.TenantID, source.KnowledgeBaseID}
|
||||
if table == "resources" || table == "resource_bindings" {
|
||||
statement = fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE tenant_id=$1`, table, table)
|
||||
|
||||
@@ -10,10 +10,11 @@ import (
|
||||
)
|
||||
|
||||
// Keep capture/verification and deletion protection on the same set of bodies.
|
||||
// predicate is an internal SQL fragment, never caller-supplied content.
|
||||
func libraryResourceBodyReferenceSQL(predicate string) string {
|
||||
// predicate is an internal SQL fragment, never caller-supplied content; tables comes
|
||||
// from the resolved schema profile (0.9.0 has no wiki_log_entries to scan).
|
||||
func libraryResourceBodyReferenceSQL(tables []string, predicate string) string {
|
||||
queries := []string{}
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range tables {
|
||||
if table == "resources" || table == "resource_bindings" || table == "knowledge_tag_relations" {
|
||||
continue
|
||||
}
|
||||
@@ -24,7 +25,7 @@ func libraryResourceBodyReferenceSQL(predicate string) string {
|
||||
|
||||
// This must precede *all* restore mutations, including ES and object deletes.
|
||||
// A created row is not permission to invalidate a later out-of-plan reader.
|
||||
func ensureRestorableResourceReferences(ctx context.Context, q libraryClosureReader, set libraryrelease.WriteSet) error {
|
||||
func ensureRestorableResourceReferences(ctx context.Context, q libraryClosureReader, tables []string, set libraryrelease.WriteSet) error {
|
||||
ids := set.CreatedIDs("resources")
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
@@ -49,7 +50,7 @@ func ensureRestorableResourceReferences(ctx context.Context, q libraryClosureRea
|
||||
if referenced {
|
||||
return errors.New("created resource has out-of-plan bindings; restore held before writes")
|
||||
}
|
||||
bodySQL := libraryResourceBodyReferenceSQL("c.tenant_id IS DISTINCT FROM $2 OR c.knowledge_base_id IS DISTINCT FROM $3")
|
||||
bodySQL := libraryResourceBodyReferenceSQL(tables, "c.tenant_id IS DISTINCT FROM $2 OR c.knowledge_base_id IS DISTINCT FROM $3")
|
||||
err = q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM resources WHERE id=ANY($1::text[]) AND EXISTS (`+bodySQL+`))`, ids, set.TenantID, set.KnowledgeBaseID).Scan(&referenced)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -62,10 +63,10 @@ func ensureRestorableResourceReferences(ctx context.Context, q libraryClosureRea
|
||||
|
||||
// Recheck under the already-held resource parent locks. This does not claim
|
||||
// cross-store isolation from uncontrolled writers; deployed exclusivity is separate.
|
||||
func ensureCreatedResourcesUnreferenced(ctx context.Context, q libraryClosureReader, ids []string) error {
|
||||
func ensureCreatedResourcesUnreferenced(ctx context.Context, q libraryClosureReader, tables, ids []string) error {
|
||||
var referenced bool
|
||||
err := q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM resources WHERE id=ANY($1::text[]) AND
|
||||
(EXISTS(SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id) OR EXISTS (`+libraryResourceBodyReferenceSQL("TRUE")+`)))`, ids).Scan(&referenced)
|
||||
(EXISTS(SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id) OR EXISTS (`+libraryResourceBodyReferenceSQL(tables, "TRUE")+`)))`, ids).Scan(&referenced)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ func (t *ProductionLibraryTarget) migrationCatalog(ctx context.Context) (migrati
|
||||
return nil, "", err
|
||||
}
|
||||
defer tx.Rollback(context.WithoutCancel(ctx))
|
||||
catalog, digest, err := captureMigrationCatalog(ctx, tx)
|
||||
catalog, digest, err := captureMigrationCatalogFor(ctx, tx, t.tables)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@@ -150,7 +150,7 @@ func (t *ProductionLibraryTarget) preflightMigrationCatalog(ctx context.Context,
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateMigrationCatalog(catalog); err != nil {
|
||||
if err := validateMigrationCatalogFor(catalog, t.tables, t.schema.Profile); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var columns []struct {
|
||||
|
||||
@@ -17,8 +17,12 @@ func ScopeContentCounts(ctx context.Context, pool *pgxpool.Pool, identity Produc
|
||||
if pool == nil || identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
|
||||
return nil, errors.New("invalid scope identity")
|
||||
}
|
||||
counts := make(map[string]int64, len(productionLibraryTables))
|
||||
for _, table := range productionLibraryTables {
|
||||
profile, err := ResolveSchemaProfile(ctx, pool, "scope content probe")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
counts := make(map[string]int64, len(profile.Tables))
|
||||
for _, table := range profile.Tables {
|
||||
var count int64
|
||||
if err := pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`, table, scopeWhere(table)),
|
||||
identity.TenantID, identity.KnowledgeBaseID).Scan(&count); err != nil {
|
||||
|
||||
@@ -24,8 +24,14 @@ import (
|
||||
// 写坏"(资源表的 ON CONFLICT 就会这样)在指纹上完全看不出来——数量、身份都一样。
|
||||
// 因此摘要取自每行 to_jsonb 的 md5,再按身份排序聚合。
|
||||
func universeFingerprint(ctx context.Context, pool *pgxpool.Pool) (map[string]libraryrelease.TableFingerprint, error) {
|
||||
result := make(map[string]libraryrelease.TableFingerprint, len(productionLibraryTables))
|
||||
for _, table := range productionLibraryTables {
|
||||
// 表集合来自目标库自己的 profile:0.9.0 没有 wiki_log_entries,按 0.7.1 的固定
|
||||
// 清单取指纹会在第一张不存在的表上失败,而"跳过缺表"又会让 0.7.1 少表蒙混过去。
|
||||
schema, err := ResolveSchemaProfile(ctx, pool, "whole-instance fingerprint")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make(map[string]libraryrelease.TableFingerprint, len(schema.Tables))
|
||||
for _, table := range schema.Tables {
|
||||
identity := identityExpr(table)
|
||||
statement := fmt.Sprintf(
|
||||
`SELECT count(*), coalesce(md5(string_agg(md5(to_jsonb(x)::text), E'\n' ORDER BY (%s)::text)), '') FROM %s AS x`,
|
||||
|
||||
@@ -76,12 +76,12 @@ func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, ta
|
||||
// the legacy per-resource body scan made post-sync verification minutes
|
||||
// long on a small target inside a large tenant.
|
||||
var bodies, bindings []json.RawMessage
|
||||
for _, name := range productionLibraryTables {
|
||||
for _, name := range t.tables {
|
||||
if name == "resources" {
|
||||
continue
|
||||
}
|
||||
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
|
||||
name, name, scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
name, name, t.scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, ta
|
||||
}
|
||||
rows = items
|
||||
} else {
|
||||
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
|
||||
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, t.scopeWhere(table)),
|
||||
identity.TenantID, identity.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
|
||||
|
||||
@@ -401,7 +401,7 @@ func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set l
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ensureCreatedResourcesUnreferenced(ctx, tx, ids); err != nil {
|
||||
if err := ensureCreatedResourcesUnreferenced(ctx, tx, t.tables, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -412,7 +412,7 @@ func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set l
|
||||
// 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到
|
||||
// 预备份状态,必须报出来让维护保持,而不是静默放过。
|
||||
leftovers := make([]string, 0, 4)
|
||||
for _, table := range productionLibraryTables {
|
||||
for _, table := range t.tables {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, id := range set.CreatedIDs(table) {
|
||||
if restored[table][id] {
|
||||
|
||||
@@ -525,7 +525,9 @@ func validObjectLocator(value string) bool {
|
||||
}
|
||||
|
||||
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
|
||||
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))}
|
||||
// 复制快照不得改契约:SchemaVersion/Profile 丢掉会让下游按未知版本(=0.7.1)处理。
|
||||
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables)),
|
||||
SchemaVersion: snapshot.SchemaVersion, Profile: snapshot.Profile}
|
||||
for index, table := range snapshot.Tables {
|
||||
result.Tables[index] = table
|
||||
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
||||
|
||||
@@ -135,5 +135,8 @@ func replaceTable(snapshot PostgresSnapshot, replacement TableSnapshot) Postgres
|
||||
if !replaced {
|
||||
tables = append(tables, replacement)
|
||||
}
|
||||
return PostgresSnapshot{Tables: tables}
|
||||
// SchemaVersion/Profile 描述的是这份快照的 schema 契约,不是表内容:重建快照时
|
||||
// 丢掉它们,下游读到的就是"版本未知",而未知版本按 0.7.1 处理——134 的归档
|
||||
// 于是被当成 74 搬运,跨版本闸门形同不存在。
|
||||
return PostgresSnapshot{Tables: tables, SchemaVersion: snapshot.SchemaVersion, Profile: snapshot.Profile}
|
||||
}
|
||||
|
||||
@@ -173,3 +173,26 @@ func TestPlanItemTargetBaseFallsBackToTestedBase(t *testing.T) {
|
||||
t.Fatalf("unbound TargetBase = %q, want the tested base", unbound.TargetBase())
|
||||
}
|
||||
}
|
||||
|
||||
// 范围改写只换身份,不改 schema 契约:改写后的快照若丢掉 SchemaVersion/Profile,
|
||||
// 下游读到的就是"版本未知",而未知版本会被当作 0.7.1 处理——134 的归档于是被当成
|
||||
// 74 搬,跨版本闸门形同不存在。
|
||||
func TestRemapKnowledgeBasePreservesSchemaProfile(t *testing.T) {
|
||||
bundle := crossBaseBundle(t)
|
||||
bundle.Postgres.SchemaVersion = 134
|
||||
bundle.Postgres.Profile = "v090"
|
||||
if err := bundle.RemapKnowledgeBase("prod-kb"); err != nil {
|
||||
t.Fatalf("remap: %v", err)
|
||||
}
|
||||
if bundle.Postgres.SchemaVersion != 134 || bundle.Postgres.Profile != "v090" {
|
||||
t.Fatalf("remapped snapshot lost its schema profile: %d/%q",
|
||||
bundle.Postgres.SchemaVersion, bundle.Postgres.Profile)
|
||||
}
|
||||
tables := map[string]bool{}
|
||||
for _, table := range bundle.Postgres.Tables {
|
||||
tables[table.Name] = true
|
||||
}
|
||||
if len(tables) != len(bundle.Postgres.Tables) {
|
||||
t.Fatal("remap duplicated a table")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -152,8 +153,16 @@ func Validate(manifest Manifest, target TargetContract, mapping KnowledgeBaseMap
|
||||
}
|
||||
|
||||
func validateManifestFields(manifest Manifest) error {
|
||||
if manifest.FormatVersion != FormatVersionV1 || manifest.Contract.WeKnoraVersion != WeKnoraVersionV071 ||
|
||||
manifest.Contract.MigrationVersion != WeKnoraMigrationV071 || manifest.Contract.EmbeddingDimension != EmbeddingDimensionV071 ||
|
||||
// 单文档发布包只认 0.7.1 的表集合与契约。别的版本必须**明确拒绝并说明原因**,
|
||||
// 不能落进下面那句泛化的 "unsupported release contract"——那让人分不清是版本
|
||||
// 不支持还是字段写错了。0.9.0 的整库内容走整库发布路径(libraryprofile v090)。
|
||||
if manifest.Contract.MigrationVersion != WeKnoraMigrationV071 ||
|
||||
manifest.Contract.WeKnoraVersion != WeKnoraVersionV071 {
|
||||
return fmt.Errorf("release bundle schema contract %s/%d is not supported: single-document release handles only v0.7.1 (74); WeKnora 0.9.0 (134) requires the library-level publish path",
|
||||
manifest.Contract.WeKnoraVersion, manifest.Contract.MigrationVersion)
|
||||
}
|
||||
if manifest.FormatVersion != FormatVersionV1 ||
|
||||
manifest.Contract.EmbeddingDimension != EmbeddingDimensionV071 ||
|
||||
manifest.Contract.RetrieverEngine != "elasticsearch_v8" || manifest.Contract.TenantID == 0 || manifest.CreatedAt.IsZero() {
|
||||
return errors.New("unsupported release contract")
|
||||
}
|
||||
|
||||
@@ -165,3 +165,27 @@ func TestValidateRejectsUndeclaredOrDriftedMapping(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 单文档发布包只支持 0.7.1 契约。0.9.0(134)的包必须被**明确拒绝并说明原因**:
|
||||
// 静默按 74 处理会让 0.9.0 的表集合/索引文档走错写入路径,而泛化错误又让人分不清
|
||||
// "版本不支持"和"字段写错"。
|
||||
func TestSealRejectsNonV071SchemaContractExplicitly(t *testing.T) {
|
||||
manifest := validManifest()
|
||||
manifest.Contract.WeKnoraVersion = "v0.9.0"
|
||||
manifest.Contract.MigrationVersion = 134
|
||||
_, err := Seal(manifest)
|
||||
if err == nil {
|
||||
t.Fatal("0.9.0 release bundle must be refused")
|
||||
}
|
||||
for _, want := range []string{"v0.9.0", "134", "74", "library-level publish"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("rejection message must name %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
// 只改版本号、不改 WeKnora 版本同样必须拒绝:两者必须成对出现。
|
||||
manifest = validManifest()
|
||||
manifest.Contract.MigrationVersion = 134
|
||||
if _, err := Seal(manifest); err == nil {
|
||||
t.Fatal("schema version 134 with a v0.7.1 label must be refused")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -652,6 +652,11 @@
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestV090LibraryRoundTripIsolated",
|
||||
"reason": "需 KG_V090_SCHEMA_SQL 指向真实 WeKnora 0.9.0 schema dump、KG_V090_MAPPING_JSON 指向真实索引映射,以及显式隔离的 kg_library_test_* PG;普通 CI 没有这些输入,跳过不是通过"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestV090ProductionTargetPublishIsolated",
|
||||
"reason": "需显式一次性本机PG(两份 0.9.0 真 schema 库)/ES/对象存储:真实导出→封存→ProductionLibraryTarget 写入 134 目标→逐表核对→写前镜像回滚,验证 0.9.0 无 wiki_log_entries、chunks 新增 NOT NULL 列走默认值、seq_id 由目标序列分配;普通CI没有这些服务,跳过不是通过"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user