Files
knowledge-governance/internal/releaseadapter/v071/postgres_import.go
T
姚章浩 fba2713b4e
build / build (push) Successful in 1m54s
feat(library): 发布写入支持 WeKnora 0.9.0 schema 134
整库发布写入路径原先按 0.7.1 的固定表集合工作:0.9.0 没有 wiki_log_entries,
Current/清理/编目/整实例指纹都会在不存在表上失败,跨版本闸门也因快照丢掉
SchemaVersion 而失效(未知版本被当作 74)。

改为在装配时解析目标 profile,表集合与清理顺序按解析结果;快照复制与范围改写
带上 SchemaVersion/Profile;跨版本发布显式拒绝并同时点名两个版本。单文档发布包
仍只支持 0.7.1,但改为明确拒绝并说明原因。

本机隔离验证:真实 0.9.0 schema 源库→导出→封存→写入干净 134 目标→逐表核对→
写前镜像回滚;74→134 拒绝与 134 缺表不报错两个负例通过。make verify 17 项全绿。
2026-11-16 20:42:23 +08:00

537 lines
22 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"github.com/jackc/pgx/v5"
)
type postgresTransactionalDB interface {
Begin(ctx context.Context) (pgx.Tx, error)
}
type PostgresImporterConfig struct {
TargetStorageBackendID string
TargetProvider string
// TargetBucket rewrites the bucket embedded in nested provider locators
// (oss://<bucket>/<key>). WeKnora resolves GetFile from the path's inner
// bucket, so a source-env bucket leaking into the target row makes every
// preview/download read the wrong bucket with target credentials.
TargetBucket string
}
type PostgresImporter struct {
db postgresTransactionalDB
config PostgresImporterConfig
}
// clearStatement 是导入前置清理语句(文档级或库级 wiki)。
type clearStatement struct {
sql string
args []any
}
func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) {
if db == nil || len(options) > 1 {
return nil, errors.New("invalid PostgreSQL importer configuration")
}
var config PostgresImporterConfig
if len(options) == 1 {
config = options[0]
}
config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID)
config.TargetProvider = strings.TrimSpace(config.TargetProvider)
config.TargetBucket = strings.TrimSpace(config.TargetBucket)
if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") ||
(config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) {
return nil, errors.New("invalid target storage backend configuration")
}
if config.TargetBucket != "" && !validSearchIdentifier(config.TargetBucket) {
return nil, errors.New("invalid target storage bucket configuration")
}
return &PostgresImporter{db: db, config: config}, nil
}
func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot)
if err != nil {
return err
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL document import")
}
defer tx.Rollback(ctx)
profile, err := ResolveSchemaProfile(ctx, tx, "target WeKnora")
if err != nil {
return err
}
// 目标版本必须与归档同 profile:0.7.1 的归档不能写进 0.9.0,反之亦然。
if err := requireSameProfile(snapshot.SchemaVersion, profile, "import"); err != nil {
return err
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
return errors.New("lock target knowledge")
}
_, hasWiki := wikiSnapshotRows(prepared)
if hasWiki {
// 库级 wiki 替换加知识库维度的锁(命名空间 1),与文档维度的锁区分开。
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
return errors.New("lock target wiki knowledge base")
}
}
var crossScope int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::INTEGER
FROM knowledges
WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2)
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target knowledge ID conflicts with another scope")
}
for _, pair := range resourcePairs {
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::INTEGER
FROM resources
WHERE (id = $1 OR handle = $2)
AND NOT (id = $1 AND handle = $2 AND tenant_id = $3)
`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target resource identity conflict")
}
}
for _, raw := range prepared.Table("knowledge_tags").Rows {
var tag struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &tag) != nil {
return errors.New("invalid target knowledge tag")
}
var crossScope int
if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target knowledge tag identity conflict")
}
}
clearStatements := []clearStatement{
{`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}},
{`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}},
{`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}},
}
for _, statement := range clearStatements {
if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil {
return errors.New("clear prior target document rows")
}
}
insertOrder := []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"}
for _, tableName := range insertOrder {
table := prepared.Table(tableName)
if err := insertSnapshotRows(ctx, tx, table); err != nil {
return err
}
}
// wiki 是库级快照,但不能盲删整库:先按 source_refs 判定归属,只替换本包
// 拥有的页面,其余(其他文档、agent 手写页)一行不动。
if hasWiki {
if _, err := importer.replaceWiki(ctx, tx, identity, prepared); err != nil {
return err
}
}
if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil {
return errors.New("advance target chunk sequence")
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL document import")
}
return nil
}
// ReplaceWiki 只应用库级 wiki 快照(撤权 / 回滚到旧版时使用)。
// 撤权路径上文档行只做 disable(不能把文档行重新灌回去),而 wiki 必须整体
// 替换成撤权后的快照,否则生产 wiki 会一直留着已撤权文档的页面与引用。
func (importer *PostgresImporter) ReplaceWiki(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return errors.New("invalid PostgreSQL wiki import identity")
}
tables, hasWiki := wikiSnapshotRows(snapshot)
if !hasWiki {
// 老包 / 未启用 wiki 的库:不动生产 wiki。
return nil
}
for _, table := range tables {
if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
return errors.New("PostgreSQL wiki import snapshot is invalid")
}
}
for _, row := range snapshot.Table("wiki_pages").Rows {
if !matchesWikiPageRow(row, identity) {
return errors.New("PostgreSQL wiki page scope mismatch")
}
}
for _, row := range snapshot.Table("wiki_folders").Rows {
if !matchesWikiFolderRow(row, identity) {
return errors.New("PostgreSQL wiki folder scope mismatch")
}
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL wiki import")
}
defer tx.Rollback(ctx)
profile, err := ResolveSchemaProfile(ctx, tx, "target WeKnora")
if err != nil {
return err
}
if err := requireSameProfile(snapshot.SchemaVersion, profile, "wiki import"); err != nil {
return err
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
return errors.New("lock target wiki knowledge base")
}
// 撤权 / 回滚也走同一套归属判定:只撤本包知识拥有的页面与引用,
// 目标库里其他文档和 agent 手写的内容一行不动。
if _, err := importer.replaceWiki(ctx, tx, identity, snapshot); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL wiki import")
}
return nil
}
func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return errors.New("invalid PostgreSQL disable identity")
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL disable")
}
defer tx.Rollback(ctx)
if _, err := ResolveSchemaProfile(ctx, tx, "target WeKnora"); err != nil {
return err
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
return errors.New("lock target knowledge")
}
command, err := tx.Exec(ctx, `
UPDATE knowledges
SET enable_status = 'disabled', updated_at = NOW()
WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID)
if err != nil || command.RowsAffected() != 1 {
return errors.New("disable target knowledge")
}
if _, err := tx.Exec(ctx, `
UPDATE chunks
SET is_enabled = FALSE, updated_at = NOW()
WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil {
return errors.New("disable target chunks")
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL disable")
}
return nil
}
type resourceIdentityPair struct{ id, handle string }
func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity")
}
if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete")
}
prepared := cloneSnapshot(snapshot)
// 表集合按快照所属 profile 判定:0.9.0 的归档里有 wiki_page_issues,
// 面向单文档发布的白名单没有它,用白名单会把合法归档判成非法。
profile, err := snapshot.schemaProfile()
if err != nil {
return PostgresSnapshot{}, nil, nil, err
}
known := map[string]bool{}
for index, table := range prepared.Tables {
if !profile.HasTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid")
}
known[table.Name] = true
prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
}
if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch")
}
chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount)
chunkRows := prepared.Table("chunks").Rows
for index, row := range chunkRows {
id, ok := matchesChunkRow(row, identity)
if !ok {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
}
chunkIDs = append(chunkIDs, id)
// chunks.seq_id is a per-database global sequence value, not document
// content (WeKnora retrieval and Elasticsearch never reference it).
// Carrying a source value across environments collides with rows the
// target has already written, so let the target sequence assign it and
// advance the sequence after the transaction below.
var fields map[string]any
if err := json.Unmarshal(row, &fields); err != nil {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
}
if _, present := fields["seq_id"]; present {
delete(fields, "seq_id")
rewritten, err := json.Marshal(fields)
if err != nil {
return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row")
}
chunkRows[index] = rewritten
}
}
chunks := newTableSnapshot("chunks", chunkRows)
for index := range prepared.Tables {
if prepared.Tables[index].Name == "chunks" {
prepared.Tables[index] = chunks
}
}
tagIDs := map[string]bool{}
for _, row := range prepared.Table("knowledge_tags").Rows {
if !matchesKnowledgeTagRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
}
var tag struct {
ID string `json:"id"`
}
if json.Unmarshal(row, &tag) != nil {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
}
tagIDs[tag.ID] = true
}
for _, row := range prepared.Table("knowledge_tag_relations").Rows {
var relation struct {
TagID string `json:"tag_id"`
}
if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch")
}
}
resources := prepared.Table("resources")
resourceIDs := make([]string, 0, len(resources.Rows))
resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows))
for rowIndex, row := range resources.Rows {
rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID)
if err != nil {
return PostgresSnapshot{}, nil, nil, err
}
resources.Rows[rowIndex] = rewritten
resourceIDs = append(resourceIDs, pair.id)
resourcePairs = append(resourcePairs, pair)
}
resources = newTableSnapshot("resources", resources.Rows)
for index := range prepared.Tables {
if prepared.Tables[index].Name == "resources" {
prepared.Tables[index] = resources
}
}
for _, row := range prepared.Table("resource_bindings").Rows {
if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch")
}
}
// wiki 是库级快照:行必须落在同一个 tenant + knowledge base 里。
for _, row := range prepared.Table("wiki_pages").Rows {
if !matchesWikiPageRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki page scope mismatch")
}
}
for _, row := range prepared.Table("wiki_folders").Rows {
if !matchesWikiFolderRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki folder scope mismatch")
}
}
return prepared, chunkIDs, resourcePairs, nil
}
func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) {
var row map[string]any
if json.Unmarshal(raw, &row) != nil {
return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource")
}
id, _ := row["id"].(string)
handle, _ := row["handle"].(string)
if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id {
return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch")
}
sourceBackendID, _ := row["storage_backend_id"].(string)
physicalPath, _ := row["physical_path"].(string)
if importer.config.TargetStorageBackendID == "" {
if sourceBackendID != "" {
return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required")
}
return raw, resourceIdentityPair{id: id, handle: handle}, nil
}
providerPath := physicalPath
if strings.HasPrefix(physicalPath, "storage://") {
rest := strings.TrimPrefix(physicalPath, "storage://")
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
}
providerPath = parts[1]
} else if !validObjectLocator(physicalPath) {
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
}
// WeKnora's official file service parses the bucket out of the nested
// provider locator (oss://<bucket>/<key>) and uses it directly for
// GetObject. Carrying the source bucket across environments makes every
// target-side preview read the wrong bucket, so rewrite it to the target
// bucket whenever the locator embeds one and a target bucket is configured.
if inner := strings.Index(providerPath, "://"); inner >= 0 && importer.config.TargetBucket != "" {
rest := providerPath[inner+3:]
if boundary := strings.Index(rest, "/"); boundary > 0 && rest[:boundary] != importer.config.TargetBucket {
providerPath = providerPath[:inner+3] + importer.config.TargetBucket + rest[boundary:]
}
}
targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath
row["storage_backend_id"] = importer.config.TargetStorageBackendID
row["provider"] = importer.config.TargetProvider
row["physical_path"] = targetPath
// 与全库搬运共用同一个哈希实现(F08):契约是 sha256(最终 physical_path)。
row["location_hash"] = resourceLocationHash(targetPath)
encoded, err := json.Marshal(row)
if err != nil {
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
}
canonical, err := canonicalJSON(encoded)
if err != nil {
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
}
return canonical, resourceIdentityPair{id: id, handle: handle}, nil
}
func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error {
if len(table.Rows) == 0 {
return nil
}
columns, err := tableColumns(ctx, tx, table.Name)
if err != nil {
return err
}
if table.Name == "chunks" {
// seq_id is assigned by the target sequence; it must not appear in the
// explicit column list (an explicit NULL would violate NOT NULL) and it
// is removed from snapshot rows during prepare.
filtered := columns[:0]
for _, column := range columns {
if column != "seq_id" {
filtered = append(filtered, column)
}
}
columns = filtered
}
quotedTable := pgx.Identifier{table.Name}.Sanitize()
quotedColumns := make([]string, len(columns))
for index, column := range columns {
quotedColumns[index] = pgx.Identifier{column}.Sanitize()
}
columnList := strings.Join(quotedColumns, ", ")
selectList := make([]string, len(quotedColumns))
for index, column := range quotedColumns {
selectList[index] = "record_value." + column
}
query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable)
if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" {
updates := make([]string, 0, len(quotedColumns)-1)
for index, column := range columns {
if column == "id" {
continue
}
updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index])
}
query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ")
}
for _, row := range table.Rows {
if _, err := tx.Exec(ctx, query, string(row)); err != nil {
return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err)
}
}
return nil
}
func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) {
if !validReleaseTable(tableName) {
return nil, errors.New("invalid PostgreSQL import table")
}
rows, err := tx.Query(ctx, `
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER'
ORDER BY ordinal_position
`, tableName)
if err != nil {
return nil, errors.New("inspect target PostgreSQL table")
}
defer rows.Close()
columns := make([]string, 0)
for rows.Next() {
var column string
if rows.Scan(&column) != nil || !validSQLIdentifier(column) {
return nil, errors.New("invalid target PostgreSQL column")
}
columns = append(columns, column)
}
if rows.Err() != nil || len(columns) == 0 {
return nil, errors.New("inspect target PostgreSQL table")
}
return columns, nil
}
func validSQLIdentifier(value string) bool {
if value == "" || len(value) > 63 {
return false
}
for index, char := range value {
if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') {
return false
}
}
return true
}
// validObjectLocator 接受两种真实形态:
// - 无 scheme 的对象键:"weknora/10000/<doc>/<file>.docx"
// - 带 scheme 的定位符:"oss://<bucket>/<key>"(WeKnora 的资源路径就是这个形状)
//
// 只校验结构,不改值;真正的 bucket/key 拆分由 providerPathFromLocator 做。
func validObjectLocator(value string) bool {
if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") {
return false
}
if separator := strings.Index(value, "://"); separator >= 0 {
if !validSearchIdentifier(value[:separator]) {
return false
}
remainder := value[separator+3:]
boundary := strings.Index(remainder, "/")
if boundary <= 0 || !validSearchIdentifier(remainder[:boundary]) {
return false
}
value = remainder[boundary+1:]
}
cleaned := path.Clean(value)
return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../")
}
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
// 复制快照不得改契约:SchemaVersion/Profile 丢掉会让下游按未知版本(=0.7.1)处理。
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables)),
SchemaVersion: snapshot.SchemaVersion, Profile: snapshot.Profile}
for index, table := range snapshot.Tables {
result.Tables[index] = table
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
}
return result
}