ac3b395102
build / build (push) Successful in 1m21s
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设, 以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。 - knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。 - 写入顺序按真实外键;无代理键表用内容摘要核对。 - 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。 - 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。 - 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。 - finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。 - 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
528 lines
21 KiB
Go
528 lines
21 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"path"
|
||
"strings"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
)
|
||
|
||
type postgresTransactionalDB interface {
|
||
Begin(ctx context.Context) (pgx.Tx, error)
|
||
}
|
||
|
||
type PostgresImporterConfig struct {
|
||
TargetStorageBackendID string
|
||
TargetProvider string
|
||
// TargetBucket rewrites the bucket embedded in nested provider locators
|
||
// (oss://<bucket>/<key>). WeKnora resolves GetFile from the path's inner
|
||
// bucket, so a source-env bucket leaking into the target row makes every
|
||
// preview/download read the wrong bucket with target credentials.
|
||
TargetBucket string
|
||
}
|
||
|
||
type PostgresImporter struct {
|
||
db postgresTransactionalDB
|
||
config PostgresImporterConfig
|
||
}
|
||
|
||
// clearStatement 是导入前置清理语句(文档级或库级 wiki)。
|
||
type clearStatement struct {
|
||
sql string
|
||
args []any
|
||
}
|
||
|
||
func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) {
|
||
if db == nil || len(options) > 1 {
|
||
return nil, errors.New("invalid PostgreSQL importer configuration")
|
||
}
|
||
var config PostgresImporterConfig
|
||
if len(options) == 1 {
|
||
config = options[0]
|
||
}
|
||
config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID)
|
||
config.TargetProvider = strings.TrimSpace(config.TargetProvider)
|
||
config.TargetBucket = strings.TrimSpace(config.TargetBucket)
|
||
if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") ||
|
||
(config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) {
|
||
return nil, errors.New("invalid target storage backend configuration")
|
||
}
|
||
if config.TargetBucket != "" && !validSearchIdentifier(config.TargetBucket) {
|
||
return nil, errors.New("invalid target storage bucket configuration")
|
||
}
|
||
return &PostgresImporter{db: db, config: config}, nil
|
||
}
|
||
|
||
func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
|
||
prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
tx, err := importer.db.Begin(ctx)
|
||
if err != nil {
|
||
return errors.New("begin PostgreSQL document import")
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
var version int
|
||
var dirty bool
|
||
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
||
return errors.New("target WeKnora schema contract mismatch")
|
||
}
|
||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
||
return errors.New("lock target knowledge")
|
||
}
|
||
_, hasWiki := wikiSnapshotRows(prepared)
|
||
if hasWiki {
|
||
// 库级 wiki 替换加知识库维度的锁(命名空间 1),与文档维度的锁区分开。
|
||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
|
||
return errors.New("lock target wiki knowledge base")
|
||
}
|
||
}
|
||
var crossScope int
|
||
if err := tx.QueryRow(ctx, `
|
||
SELECT COUNT(*)::INTEGER
|
||
FROM knowledges
|
||
WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2)
|
||
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 {
|
||
return errors.New("target knowledge ID conflicts with another scope")
|
||
}
|
||
for _, pair := range resourcePairs {
|
||
if err := tx.QueryRow(ctx, `
|
||
SELECT COUNT(*)::INTEGER
|
||
FROM resources
|
||
WHERE (id = $1 OR handle = $2)
|
||
AND NOT (id = $1 AND handle = $2 AND tenant_id = $3)
|
||
`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 {
|
||
return errors.New("target resource identity conflict")
|
||
}
|
||
}
|
||
for _, raw := range prepared.Table("knowledge_tags").Rows {
|
||
var tag struct {
|
||
ID string `json:"id"`
|
||
}
|
||
if json.Unmarshal(raw, &tag) != nil {
|
||
return errors.New("invalid target knowledge tag")
|
||
}
|
||
var crossScope int
|
||
if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 {
|
||
return errors.New("target knowledge tag identity conflict")
|
||
}
|
||
}
|
||
clearStatements := []clearStatement{
|
||
{`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}},
|
||
{`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}},
|
||
{`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}},
|
||
}
|
||
for _, statement := range clearStatements {
|
||
if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil {
|
||
return errors.New("clear prior target document rows")
|
||
}
|
||
}
|
||
insertOrder := []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"}
|
||
for _, tableName := range insertOrder {
|
||
table := prepared.Table(tableName)
|
||
if err := insertSnapshotRows(ctx, tx, table); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
// wiki 是库级快照,但不能盲删整库:先按 source_refs 判定归属,只替换本包
|
||
// 拥有的页面,其余(其他文档、agent 手写页)一行不动。
|
||
if hasWiki {
|
||
if _, err := importer.replaceWiki(ctx, tx, identity, prepared); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil {
|
||
return errors.New("advance target chunk sequence")
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return errors.New("commit PostgreSQL document import")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ReplaceWiki 只应用库级 wiki 快照(撤权 / 回滚到旧版时使用)。
|
||
// 撤权路径上文档行只做 disable(不能把文档行重新灌回去),而 wiki 必须整体
|
||
// 替换成撤权后的快照,否则生产 wiki 会一直留着已撤权文档的页面与引用。
|
||
func (importer *PostgresImporter) ReplaceWiki(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
|
||
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
|
||
return errors.New("invalid PostgreSQL wiki import identity")
|
||
}
|
||
tables, hasWiki := wikiSnapshotRows(snapshot)
|
||
if !hasWiki {
|
||
// 老包 / 未启用 wiki 的库:不动生产 wiki。
|
||
return nil
|
||
}
|
||
for _, table := range tables {
|
||
if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
|
||
return errors.New("PostgreSQL wiki import snapshot is invalid")
|
||
}
|
||
}
|
||
for _, row := range snapshot.Table("wiki_pages").Rows {
|
||
if !matchesWikiPageRow(row, identity) {
|
||
return errors.New("PostgreSQL wiki page scope mismatch")
|
||
}
|
||
}
|
||
for _, row := range snapshot.Table("wiki_folders").Rows {
|
||
if !matchesWikiFolderRow(row, identity) {
|
||
return errors.New("PostgreSQL wiki folder scope mismatch")
|
||
}
|
||
}
|
||
tx, err := importer.db.Begin(ctx)
|
||
if err != nil {
|
||
return errors.New("begin PostgreSQL wiki import")
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
var version int
|
||
var dirty bool
|
||
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
||
return errors.New("target WeKnora schema contract mismatch")
|
||
}
|
||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
|
||
return errors.New("lock target wiki knowledge base")
|
||
}
|
||
// 撤权 / 回滚也走同一套归属判定:只撤本包知识拥有的页面与引用,
|
||
// 目标库里其他文档和 agent 手写的内容一行不动。
|
||
if _, err := importer.replaceWiki(ctx, tx, identity, snapshot); err != nil {
|
||
return err
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return errors.New("commit PostgreSQL wiki import")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error {
|
||
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
||
return errors.New("invalid PostgreSQL disable identity")
|
||
}
|
||
tx, err := importer.db.Begin(ctx)
|
||
if err != nil {
|
||
return errors.New("begin PostgreSQL disable")
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
var version int
|
||
var dirty bool
|
||
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
||
return errors.New("target WeKnora schema contract mismatch")
|
||
}
|
||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
||
return errors.New("lock target knowledge")
|
||
}
|
||
command, err := tx.Exec(ctx, `
|
||
UPDATE knowledges
|
||
SET enable_status = 'disabled', updated_at = NOW()
|
||
WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL
|
||
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID)
|
||
if err != nil || command.RowsAffected() != 1 {
|
||
return errors.New("disable target knowledge")
|
||
}
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE chunks
|
||
SET is_enabled = FALSE, updated_at = NOW()
|
||
WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL
|
||
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil {
|
||
return errors.New("disable target chunks")
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return errors.New("commit PostgreSQL disable")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type resourceIdentityPair struct{ id, handle string }
|
||
|
||
func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) {
|
||
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity")
|
||
}
|
||
if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete")
|
||
}
|
||
prepared := cloneSnapshot(snapshot)
|
||
known := map[string]bool{}
|
||
for index, table := range prepared.Tables {
|
||
if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid")
|
||
}
|
||
known[table.Name] = true
|
||
prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
||
}
|
||
if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch")
|
||
}
|
||
chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount)
|
||
chunkRows := prepared.Table("chunks").Rows
|
||
for index, row := range chunkRows {
|
||
id, ok := matchesChunkRow(row, identity)
|
||
if !ok {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
||
}
|
||
chunkIDs = append(chunkIDs, id)
|
||
// chunks.seq_id is a per-database global sequence value, not document
|
||
// content (WeKnora retrieval and Elasticsearch never reference it).
|
||
// Carrying a source value across environments collides with rows the
|
||
// target has already written, so let the target sequence assign it and
|
||
// advance the sequence after the transaction below.
|
||
var fields map[string]any
|
||
if err := json.Unmarshal(row, &fields); err != nil {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
||
}
|
||
if _, present := fields["seq_id"]; present {
|
||
delete(fields, "seq_id")
|
||
rewritten, err := json.Marshal(fields)
|
||
if err != nil {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row")
|
||
}
|
||
chunkRows[index] = rewritten
|
||
}
|
||
}
|
||
chunks := newTableSnapshot("chunks", chunkRows)
|
||
for index := range prepared.Tables {
|
||
if prepared.Tables[index].Name == "chunks" {
|
||
prepared.Tables[index] = chunks
|
||
}
|
||
}
|
||
tagIDs := map[string]bool{}
|
||
for _, row := range prepared.Table("knowledge_tags").Rows {
|
||
if !matchesKnowledgeTagRow(row, identity) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
||
}
|
||
var tag struct {
|
||
ID string `json:"id"`
|
||
}
|
||
if json.Unmarshal(row, &tag) != nil {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
||
}
|
||
tagIDs[tag.ID] = true
|
||
}
|
||
for _, row := range prepared.Table("knowledge_tag_relations").Rows {
|
||
var relation struct {
|
||
TagID string `json:"tag_id"`
|
||
}
|
||
if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch")
|
||
}
|
||
}
|
||
resources := prepared.Table("resources")
|
||
resourceIDs := make([]string, 0, len(resources.Rows))
|
||
resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows))
|
||
for rowIndex, row := range resources.Rows {
|
||
rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID)
|
||
if err != nil {
|
||
return PostgresSnapshot{}, nil, nil, err
|
||
}
|
||
resources.Rows[rowIndex] = rewritten
|
||
resourceIDs = append(resourceIDs, pair.id)
|
||
resourcePairs = append(resourcePairs, pair)
|
||
}
|
||
resources = newTableSnapshot("resources", resources.Rows)
|
||
for index := range prepared.Tables {
|
||
if prepared.Tables[index].Name == "resources" {
|
||
prepared.Tables[index] = resources
|
||
}
|
||
}
|
||
for _, row := range prepared.Table("resource_bindings").Rows {
|
||
if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch")
|
||
}
|
||
}
|
||
// wiki 是库级快照:行必须落在同一个 tenant + knowledge base 里。
|
||
for _, row := range prepared.Table("wiki_pages").Rows {
|
||
if !matchesWikiPageRow(row, identity) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki page scope mismatch")
|
||
}
|
||
}
|
||
for _, row := range prepared.Table("wiki_folders").Rows {
|
||
if !matchesWikiFolderRow(row, identity) {
|
||
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki folder scope mismatch")
|
||
}
|
||
}
|
||
return prepared, chunkIDs, resourcePairs, nil
|
||
}
|
||
|
||
func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) {
|
||
var row map[string]any
|
||
if json.Unmarshal(raw, &row) != nil {
|
||
return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource")
|
||
}
|
||
id, _ := row["id"].(string)
|
||
handle, _ := row["handle"].(string)
|
||
if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id {
|
||
return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch")
|
||
}
|
||
sourceBackendID, _ := row["storage_backend_id"].(string)
|
||
physicalPath, _ := row["physical_path"].(string)
|
||
if importer.config.TargetStorageBackendID == "" {
|
||
if sourceBackendID != "" {
|
||
return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required")
|
||
}
|
||
return raw, resourceIdentityPair{id: id, handle: handle}, nil
|
||
}
|
||
providerPath := physicalPath
|
||
if strings.HasPrefix(physicalPath, "storage://") {
|
||
rest := strings.TrimPrefix(physicalPath, "storage://")
|
||
parts := strings.SplitN(rest, "/", 2)
|
||
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
|
||
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
||
}
|
||
providerPath = parts[1]
|
||
} else if !validObjectLocator(physicalPath) {
|
||
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
||
}
|
||
// WeKnora's official file service parses the bucket out of the nested
|
||
// provider locator (oss://<bucket>/<key>) and uses it directly for
|
||
// GetObject. Carrying the source bucket across environments makes every
|
||
// target-side preview read the wrong bucket, so rewrite it to the target
|
||
// bucket whenever the locator embeds one and a target bucket is configured.
|
||
if inner := strings.Index(providerPath, "://"); inner >= 0 && importer.config.TargetBucket != "" {
|
||
rest := providerPath[inner+3:]
|
||
if boundary := strings.Index(rest, "/"); boundary > 0 && rest[:boundary] != importer.config.TargetBucket {
|
||
providerPath = providerPath[:inner+3] + importer.config.TargetBucket + rest[boundary:]
|
||
}
|
||
}
|
||
targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath
|
||
digest := sha256.Sum256([]byte(targetPath))
|
||
row["storage_backend_id"] = importer.config.TargetStorageBackendID
|
||
row["provider"] = importer.config.TargetProvider
|
||
row["physical_path"] = targetPath
|
||
row["location_hash"] = hex.EncodeToString(digest[:])
|
||
encoded, err := json.Marshal(row)
|
||
if err != nil {
|
||
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
||
}
|
||
canonical, err := canonicalJSON(encoded)
|
||
if err != nil {
|
||
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
||
}
|
||
return canonical, resourceIdentityPair{id: id, handle: handle}, nil
|
||
}
|
||
|
||
func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error {
|
||
if len(table.Rows) == 0 {
|
||
return nil
|
||
}
|
||
columns, err := tableColumns(ctx, tx, table.Name)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if table.Name == "chunks" {
|
||
// seq_id is assigned by the target sequence; it must not appear in the
|
||
// explicit column list (an explicit NULL would violate NOT NULL) and it
|
||
// is removed from snapshot rows during prepare.
|
||
filtered := columns[:0]
|
||
for _, column := range columns {
|
||
if column != "seq_id" {
|
||
filtered = append(filtered, column)
|
||
}
|
||
}
|
||
columns = filtered
|
||
}
|
||
quotedTable := pgx.Identifier{table.Name}.Sanitize()
|
||
quotedColumns := make([]string, len(columns))
|
||
for index, column := range columns {
|
||
quotedColumns[index] = pgx.Identifier{column}.Sanitize()
|
||
}
|
||
columnList := strings.Join(quotedColumns, ", ")
|
||
selectList := make([]string, len(quotedColumns))
|
||
for index, column := range quotedColumns {
|
||
selectList[index] = "record_value." + column
|
||
}
|
||
query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable)
|
||
if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" {
|
||
updates := make([]string, 0, len(quotedColumns)-1)
|
||
for index, column := range columns {
|
||
if column == "id" {
|
||
continue
|
||
}
|
||
updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index])
|
||
}
|
||
query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ")
|
||
}
|
||
for _, row := range table.Rows {
|
||
if _, err := tx.Exec(ctx, query, string(row)); err != nil {
|
||
return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) {
|
||
if !validReleaseTable(tableName) {
|
||
return nil, errors.New("invalid PostgreSQL import table")
|
||
}
|
||
rows, err := tx.Query(ctx, `
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER'
|
||
ORDER BY ordinal_position
|
||
`, tableName)
|
||
if err != nil {
|
||
return nil, errors.New("inspect target PostgreSQL table")
|
||
}
|
||
defer rows.Close()
|
||
columns := make([]string, 0)
|
||
for rows.Next() {
|
||
var column string
|
||
if rows.Scan(&column) != nil || !validSQLIdentifier(column) {
|
||
return nil, errors.New("invalid target PostgreSQL column")
|
||
}
|
||
columns = append(columns, column)
|
||
}
|
||
if rows.Err() != nil || len(columns) == 0 {
|
||
return nil, errors.New("inspect target PostgreSQL table")
|
||
}
|
||
return columns, nil
|
||
}
|
||
|
||
func validSQLIdentifier(value string) bool {
|
||
if value == "" || len(value) > 63 {
|
||
return false
|
||
}
|
||
for index, char := range value {
|
||
if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// validObjectLocator 接受两种真实形态:
|
||
// - 无 scheme 的对象键:"weknora/10000/<doc>/<file>.docx"
|
||
// - 带 scheme 的定位符:"oss://<bucket>/<key>"(WeKnora 的资源路径就是这个形状)
|
||
//
|
||
// 只校验结构,不改值;真正的 bucket/key 拆分由 providerPathFromLocator 做。
|
||
func validObjectLocator(value string) bool {
|
||
if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") {
|
||
return false
|
||
}
|
||
if separator := strings.Index(value, "://"); separator >= 0 {
|
||
if !validSearchIdentifier(value[:separator]) {
|
||
return false
|
||
}
|
||
remainder := value[separator+3:]
|
||
boundary := strings.Index(remainder, "/")
|
||
if boundary <= 0 || !validSearchIdentifier(remainder[:boundary]) {
|
||
return false
|
||
}
|
||
value = remainder[boundary+1:]
|
||
}
|
||
cleaned := path.Clean(value)
|
||
return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../")
|
||
}
|
||
|
||
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
|
||
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))}
|
||
for index, table := range snapshot.Tables {
|
||
result.Tables[index] = table
|
||
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
||
}
|
||
return result
|
||
}
|