Files
knowledge-governance/internal/releaseadapter/v071/postgres_import.go
T
姚章浩 ac3b395102
build / build (push) Successful in 1m21s
fix(library): 对齐真实 WeKnora Schema + 入口级真实服务验收
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设,
以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。

- knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。
- 写入顺序按真实外键;无代理键表用内容摘要核对。
- 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。
- 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。
- 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。
- finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。
- 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
2026-10-28 17:53:43 +08:00

528 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"github.com/jackc/pgx/v5"
)
type postgresTransactionalDB interface {
Begin(ctx context.Context) (pgx.Tx, error)
}
type PostgresImporterConfig struct {
TargetStorageBackendID string
TargetProvider string
// TargetBucket rewrites the bucket embedded in nested provider locators
// (oss://<bucket>/<key>). WeKnora resolves GetFile from the path's inner
// bucket, so a source-env bucket leaking into the target row makes every
// preview/download read the wrong bucket with target credentials.
TargetBucket string
}
type PostgresImporter struct {
db postgresTransactionalDB
config PostgresImporterConfig
}
// clearStatement 是导入前置清理语句(文档级或库级 wiki)。
type clearStatement struct {
sql string
args []any
}
func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) {
if db == nil || len(options) > 1 {
return nil, errors.New("invalid PostgreSQL importer configuration")
}
var config PostgresImporterConfig
if len(options) == 1 {
config = options[0]
}
config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID)
config.TargetProvider = strings.TrimSpace(config.TargetProvider)
config.TargetBucket = strings.TrimSpace(config.TargetBucket)
if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") ||
(config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) {
return nil, errors.New("invalid target storage backend configuration")
}
if config.TargetBucket != "" && !validSearchIdentifier(config.TargetBucket) {
return nil, errors.New("invalid target storage bucket configuration")
}
return &PostgresImporter{db: db, config: config}, nil
}
func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot)
if err != nil {
return err
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL document import")
}
defer tx.Rollback(ctx)
var version int
var dirty bool
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
return errors.New("target WeKnora schema contract mismatch")
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
return errors.New("lock target knowledge")
}
_, hasWiki := wikiSnapshotRows(prepared)
if hasWiki {
// 库级 wiki 替换加知识库维度的锁(命名空间 1),与文档维度的锁区分开。
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
return errors.New("lock target wiki knowledge base")
}
}
var crossScope int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::INTEGER
FROM knowledges
WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2)
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target knowledge ID conflicts with another scope")
}
for _, pair := range resourcePairs {
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::INTEGER
FROM resources
WHERE (id = $1 OR handle = $2)
AND NOT (id = $1 AND handle = $2 AND tenant_id = $3)
`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target resource identity conflict")
}
}
for _, raw := range prepared.Table("knowledge_tags").Rows {
var tag struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &tag) != nil {
return errors.New("invalid target knowledge tag")
}
var crossScope int
if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 {
return errors.New("target knowledge tag identity conflict")
}
}
clearStatements := []clearStatement{
{`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}},
{`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}},
{`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}},
}
for _, statement := range clearStatements {
if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil {
return errors.New("clear prior target document rows")
}
}
insertOrder := []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"}
for _, tableName := range insertOrder {
table := prepared.Table(tableName)
if err := insertSnapshotRows(ctx, tx, table); err != nil {
return err
}
}
// wiki 是库级快照,但不能盲删整库:先按 source_refs 判定归属,只替换本包
// 拥有的页面,其余(其他文档、agent 手写页)一行不动。
if hasWiki {
if _, err := importer.replaceWiki(ctx, tx, identity, prepared); err != nil {
return err
}
}
if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil {
return errors.New("advance target chunk sequence")
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL document import")
}
return nil
}
// ReplaceWiki 只应用库级 wiki 快照(撤权 / 回滚到旧版时使用)。
// 撤权路径上文档行只做 disable(不能把文档行重新灌回去),而 wiki 必须整体
// 替换成撤权后的快照,否则生产 wiki 会一直留着已撤权文档的页面与引用。
func (importer *PostgresImporter) ReplaceWiki(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return errors.New("invalid PostgreSQL wiki import identity")
}
tables, hasWiki := wikiSnapshotRows(snapshot)
if !hasWiki {
// 老包 / 未启用 wiki 的库:不动生产 wiki。
return nil
}
for _, table := range tables {
if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
return errors.New("PostgreSQL wiki import snapshot is invalid")
}
}
for _, row := range snapshot.Table("wiki_pages").Rows {
if !matchesWikiPageRow(row, identity) {
return errors.New("PostgreSQL wiki page scope mismatch")
}
}
for _, row := range snapshot.Table("wiki_folders").Rows {
if !matchesWikiFolderRow(row, identity) {
return errors.New("PostgreSQL wiki folder scope mismatch")
}
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL wiki import")
}
defer tx.Rollback(ctx)
var version int
var dirty bool
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
return errors.New("target WeKnora schema contract mismatch")
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
return errors.New("lock target wiki knowledge base")
}
// 撤权 / 回滚也走同一套归属判定:只撤本包知识拥有的页面与引用,
// 目标库里其他文档和 agent 手写的内容一行不动。
if _, err := importer.replaceWiki(ctx, tx, identity, snapshot); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL wiki import")
}
return nil
}
func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return errors.New("invalid PostgreSQL disable identity")
}
tx, err := importer.db.Begin(ctx)
if err != nil {
return errors.New("begin PostgreSQL disable")
}
defer tx.Rollback(ctx)
var version int
var dirty bool
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
return errors.New("target WeKnora schema contract mismatch")
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
return errors.New("lock target knowledge")
}
command, err := tx.Exec(ctx, `
UPDATE knowledges
SET enable_status = 'disabled', updated_at = NOW()
WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID)
if err != nil || command.RowsAffected() != 1 {
return errors.New("disable target knowledge")
}
if _, err := tx.Exec(ctx, `
UPDATE chunks
SET is_enabled = FALSE, updated_at = NOW()
WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil {
return errors.New("disable target chunks")
}
if err := tx.Commit(ctx); err != nil {
return errors.New("commit PostgreSQL disable")
}
return nil
}
type resourceIdentityPair struct{ id, handle string }
func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity")
}
if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete")
}
prepared := cloneSnapshot(snapshot)
known := map[string]bool{}
for index, table := range prepared.Tables {
if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid")
}
known[table.Name] = true
prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
}
if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch")
}
chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount)
chunkRows := prepared.Table("chunks").Rows
for index, row := range chunkRows {
id, ok := matchesChunkRow(row, identity)
if !ok {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
}
chunkIDs = append(chunkIDs, id)
// chunks.seq_id is a per-database global sequence value, not document
// content (WeKnora retrieval and Elasticsearch never reference it).
// Carrying a source value across environments collides with rows the
// target has already written, so let the target sequence assign it and
// advance the sequence after the transaction below.
var fields map[string]any
if err := json.Unmarshal(row, &fields); err != nil {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
}
if _, present := fields["seq_id"]; present {
delete(fields, "seq_id")
rewritten, err := json.Marshal(fields)
if err != nil {
return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row")
}
chunkRows[index] = rewritten
}
}
chunks := newTableSnapshot("chunks", chunkRows)
for index := range prepared.Tables {
if prepared.Tables[index].Name == "chunks" {
prepared.Tables[index] = chunks
}
}
tagIDs := map[string]bool{}
for _, row := range prepared.Table("knowledge_tags").Rows {
if !matchesKnowledgeTagRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
}
var tag struct {
ID string `json:"id"`
}
if json.Unmarshal(row, &tag) != nil {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
}
tagIDs[tag.ID] = true
}
for _, row := range prepared.Table("knowledge_tag_relations").Rows {
var relation struct {
TagID string `json:"tag_id"`
}
if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch")
}
}
resources := prepared.Table("resources")
resourceIDs := make([]string, 0, len(resources.Rows))
resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows))
for rowIndex, row := range resources.Rows {
rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID)
if err != nil {
return PostgresSnapshot{}, nil, nil, err
}
resources.Rows[rowIndex] = rewritten
resourceIDs = append(resourceIDs, pair.id)
resourcePairs = append(resourcePairs, pair)
}
resources = newTableSnapshot("resources", resources.Rows)
for index := range prepared.Tables {
if prepared.Tables[index].Name == "resources" {
prepared.Tables[index] = resources
}
}
for _, row := range prepared.Table("resource_bindings").Rows {
if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch")
}
}
// wiki 是库级快照:行必须落在同一个 tenant + knowledge base 里。
for _, row := range prepared.Table("wiki_pages").Rows {
if !matchesWikiPageRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki page scope mismatch")
}
}
for _, row := range prepared.Table("wiki_folders").Rows {
if !matchesWikiFolderRow(row, identity) {
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki folder scope mismatch")
}
}
return prepared, chunkIDs, resourcePairs, nil
}
func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) {
var row map[string]any
if json.Unmarshal(raw, &row) != nil {
return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource")
}
id, _ := row["id"].(string)
handle, _ := row["handle"].(string)
if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id {
return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch")
}
sourceBackendID, _ := row["storage_backend_id"].(string)
physicalPath, _ := row["physical_path"].(string)
if importer.config.TargetStorageBackendID == "" {
if sourceBackendID != "" {
return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required")
}
return raw, resourceIdentityPair{id: id, handle: handle}, nil
}
providerPath := physicalPath
if strings.HasPrefix(physicalPath, "storage://") {
rest := strings.TrimPrefix(physicalPath, "storage://")
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
}
providerPath = parts[1]
} else if !validObjectLocator(physicalPath) {
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
}
// WeKnora's official file service parses the bucket out of the nested
// provider locator (oss://<bucket>/<key>) and uses it directly for
// GetObject. Carrying the source bucket across environments makes every
// target-side preview read the wrong bucket, so rewrite it to the target
// bucket whenever the locator embeds one and a target bucket is configured.
if inner := strings.Index(providerPath, "://"); inner >= 0 && importer.config.TargetBucket != "" {
rest := providerPath[inner+3:]
if boundary := strings.Index(rest, "/"); boundary > 0 && rest[:boundary] != importer.config.TargetBucket {
providerPath = providerPath[:inner+3] + importer.config.TargetBucket + rest[boundary:]
}
}
targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath
digest := sha256.Sum256([]byte(targetPath))
row["storage_backend_id"] = importer.config.TargetStorageBackendID
row["provider"] = importer.config.TargetProvider
row["physical_path"] = targetPath
row["location_hash"] = hex.EncodeToString(digest[:])
encoded, err := json.Marshal(row)
if err != nil {
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
}
canonical, err := canonicalJSON(encoded)
if err != nil {
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
}
return canonical, resourceIdentityPair{id: id, handle: handle}, nil
}
func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error {
if len(table.Rows) == 0 {
return nil
}
columns, err := tableColumns(ctx, tx, table.Name)
if err != nil {
return err
}
if table.Name == "chunks" {
// seq_id is assigned by the target sequence; it must not appear in the
// explicit column list (an explicit NULL would violate NOT NULL) and it
// is removed from snapshot rows during prepare.
filtered := columns[:0]
for _, column := range columns {
if column != "seq_id" {
filtered = append(filtered, column)
}
}
columns = filtered
}
quotedTable := pgx.Identifier{table.Name}.Sanitize()
quotedColumns := make([]string, len(columns))
for index, column := range columns {
quotedColumns[index] = pgx.Identifier{column}.Sanitize()
}
columnList := strings.Join(quotedColumns, ", ")
selectList := make([]string, len(quotedColumns))
for index, column := range quotedColumns {
selectList[index] = "record_value." + column
}
query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable)
if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" {
updates := make([]string, 0, len(quotedColumns)-1)
for index, column := range columns {
if column == "id" {
continue
}
updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index])
}
query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ")
}
for _, row := range table.Rows {
if _, err := tx.Exec(ctx, query, string(row)); err != nil {
return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err)
}
}
return nil
}
func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) {
if !validReleaseTable(tableName) {
return nil, errors.New("invalid PostgreSQL import table")
}
rows, err := tx.Query(ctx, `
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER'
ORDER BY ordinal_position
`, tableName)
if err != nil {
return nil, errors.New("inspect target PostgreSQL table")
}
defer rows.Close()
columns := make([]string, 0)
for rows.Next() {
var column string
if rows.Scan(&column) != nil || !validSQLIdentifier(column) {
return nil, errors.New("invalid target PostgreSQL column")
}
columns = append(columns, column)
}
if rows.Err() != nil || len(columns) == 0 {
return nil, errors.New("inspect target PostgreSQL table")
}
return columns, nil
}
func validSQLIdentifier(value string) bool {
if value == "" || len(value) > 63 {
return false
}
for index, char := range value {
if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') {
return false
}
}
return true
}
// validObjectLocator 接受两种真实形态:
// - 无 scheme 的对象键:"weknora/10000/<doc>/<file>.docx"
// - 带 scheme 的定位符:"oss://<bucket>/<key>"(WeKnora 的资源路径就是这个形状)
//
// 只校验结构,不改值;真正的 bucket/key 拆分由 providerPathFromLocator 做。
func validObjectLocator(value string) bool {
if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") {
return false
}
if separator := strings.Index(value, "://"); separator >= 0 {
if !validSearchIdentifier(value[:separator]) {
return false
}
remainder := value[separator+3:]
boundary := strings.Index(remainder, "/")
if boundary <= 0 || !validSearchIdentifier(remainder[:boundary]) {
return false
}
value = remainder[boundary+1:]
}
cleaned := path.Clean(value)
return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../")
}
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))}
for index, table := range snapshot.Tables {
result.Tables[index] = table
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
}
return result
}