姚章浩
e5fa063d7d
WeKnora 的 wiki 图谱与页面同表（wiki_pages），旧的库级 DELETE + 灌包在共享真实 知识库上会把其他文档的页面与 agent 手写页一起抹掉（2026-10-08 生产演练现场）。 - 新增 internal/releaseadapter/v071/wiki_scope.go：按 source_refs 判定归属 （全部引用属于本包 = InScope；引用包外知识 = 外来页；无引用 = index/log/ agent 页永不触碰），目标库存在外来页时降级为只替换包内页集，外来页上仅摘掉 本包知识引用，包内库级页不写；包内会写入的页与外来页同 slug 时 fail-closed 拒绝发布且不落任何改动。 - Replace 与 ReplaceWiki（撤权/回滚）共用同一套判定，不再整库清空。 - 作用域以一行日志落盘：release wiki replace scope in_scope=… degraded=… - 测试：单测 7 条（纯函数判定）+ 真库集成 3 条（共享库保护 / slug 冲突拒绝 / 镜像库整库替换不变），集成用例只用带随机后缀的临时知识库并自清理。 - 文档：P6 新增「wiki 通道的归属门禁」，P8 补验收点与镜像前提。
518 lines
21 KiB
Go
518 lines
21 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
type postgresTransactionalDB interface {
|
|
Begin(ctx context.Context) (pgx.Tx, error)
|
|
}
|
|
|
|
type PostgresImporterConfig struct {
|
|
TargetStorageBackendID string
|
|
TargetProvider string
|
|
// TargetBucket rewrites the bucket embedded in nested provider locators
|
|
// (oss://<bucket>/<key>). WeKnora resolves GetFile from the path's inner
|
|
// bucket, so a source-env bucket leaking into the target row makes every
|
|
// preview/download read the wrong bucket with target credentials.
|
|
TargetBucket string
|
|
}
|
|
|
|
type PostgresImporter struct {
|
|
db postgresTransactionalDB
|
|
config PostgresImporterConfig
|
|
}
|
|
|
|
// clearStatement 是导入前置清理语句（文档级或库级 wiki）。
|
|
type clearStatement struct {
|
|
sql string
|
|
args []any
|
|
}
|
|
|
|
func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) {
|
|
if db == nil || len(options) > 1 {
|
|
return nil, errors.New("invalid PostgreSQL importer configuration")
|
|
}
|
|
var config PostgresImporterConfig
|
|
if len(options) == 1 {
|
|
config = options[0]
|
|
}
|
|
config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID)
|
|
config.TargetProvider = strings.TrimSpace(config.TargetProvider)
|
|
config.TargetBucket = strings.TrimSpace(config.TargetBucket)
|
|
if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") ||
|
|
(config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) {
|
|
return nil, errors.New("invalid target storage backend configuration")
|
|
}
|
|
if config.TargetBucket != "" && !validSearchIdentifier(config.TargetBucket) {
|
|
return nil, errors.New("invalid target storage bucket configuration")
|
|
}
|
|
return &PostgresImporter{db: db, config: config}, nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
|
|
prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := importer.db.Begin(ctx)
|
|
if err != nil {
|
|
return errors.New("begin PostgreSQL document import")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var version int
|
|
var dirty bool
|
|
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
|
return errors.New("target WeKnora schema contract mismatch")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
|
return errors.New("lock target knowledge")
|
|
}
|
|
_, hasWiki := wikiSnapshotRows(prepared)
|
|
if hasWiki {
|
|
// 库级 wiki 替换加知识库维度的锁（命名空间 1），与文档维度的锁区分开。
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
|
|
return errors.New("lock target wiki knowledge base")
|
|
}
|
|
}
|
|
var crossScope int
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT COUNT(*)::INTEGER
|
|
FROM knowledges
|
|
WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2)
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target knowledge ID conflicts with another scope")
|
|
}
|
|
for _, pair := range resourcePairs {
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT COUNT(*)::INTEGER
|
|
FROM resources
|
|
WHERE (id = $1 OR handle = $2)
|
|
AND NOT (id = $1 AND handle = $2 AND tenant_id = $3)
|
|
`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target resource identity conflict")
|
|
}
|
|
}
|
|
for _, raw := range prepared.Table("knowledge_tags").Rows {
|
|
var tag struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if json.Unmarshal(raw, &tag) != nil {
|
|
return errors.New("invalid target knowledge tag")
|
|
}
|
|
var crossScope int
|
|
if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target knowledge tag identity conflict")
|
|
}
|
|
}
|
|
clearStatements := []clearStatement{
|
|
{`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}},
|
|
{`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}},
|
|
{`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}},
|
|
}
|
|
for _, statement := range clearStatements {
|
|
if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil {
|
|
return errors.New("clear prior target document rows")
|
|
}
|
|
}
|
|
insertOrder := []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"}
|
|
for _, tableName := range insertOrder {
|
|
table := prepared.Table(tableName)
|
|
if err := insertSnapshotRows(ctx, tx, table); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// wiki 是库级快照，但不能盲删整库：先按 source_refs 判定归属，只替换本包
|
|
// 拥有的页面，其余（其他文档、agent 手写页）一行不动。
|
|
if hasWiki {
|
|
if _, err := importer.replaceWiki(ctx, tx, identity, prepared); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil {
|
|
return errors.New("advance target chunk sequence")
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return errors.New("commit PostgreSQL document import")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ReplaceWiki 只应用库级 wiki 快照（撤权 / 回滚到旧版时使用）。
|
|
// 撤权路径上文档行只做 disable（不能把文档行重新灌回去），而 wiki 必须整体
|
|
// 替换成撤权后的快照，否则生产 wiki 会一直留着已撤权文档的页面与引用。
|
|
func (importer *PostgresImporter) ReplaceWiki(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
|
|
return errors.New("invalid PostgreSQL wiki import identity")
|
|
}
|
|
tables, hasWiki := wikiSnapshotRows(snapshot)
|
|
if !hasWiki {
|
|
// 老包 / 未启用 wiki 的库：不动生产 wiki。
|
|
return nil
|
|
}
|
|
for _, table := range tables {
|
|
if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
|
|
return errors.New("PostgreSQL wiki import snapshot is invalid")
|
|
}
|
|
}
|
|
for _, row := range snapshot.Table("wiki_pages").Rows {
|
|
if !matchesWikiPageRow(row, identity) {
|
|
return errors.New("PostgreSQL wiki page scope mismatch")
|
|
}
|
|
}
|
|
for _, row := range snapshot.Table("wiki_folders").Rows {
|
|
if !matchesWikiFolderRow(row, identity) {
|
|
return errors.New("PostgreSQL wiki folder scope mismatch")
|
|
}
|
|
}
|
|
tx, err := importer.db.Begin(ctx)
|
|
if err != nil {
|
|
return errors.New("begin PostgreSQL wiki import")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var version int
|
|
var dirty bool
|
|
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
|
return errors.New("target WeKnora schema contract mismatch")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil {
|
|
return errors.New("lock target wiki knowledge base")
|
|
}
|
|
// 撤权 / 回滚也走同一套归属判定：只撤本包知识拥有的页面与引用，
|
|
// 目标库里其他文档和 agent 手写的内容一行不动。
|
|
if _, err := importer.replaceWiki(ctx, tx, identity, snapshot); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return errors.New("commit PostgreSQL wiki import")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return errors.New("invalid PostgreSQL disable identity")
|
|
}
|
|
tx, err := importer.db.Begin(ctx)
|
|
if err != nil {
|
|
return errors.New("begin PostgreSQL disable")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var version int
|
|
var dirty bool
|
|
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
|
return errors.New("target WeKnora schema contract mismatch")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
|
return errors.New("lock target knowledge")
|
|
}
|
|
command, err := tx.Exec(ctx, `
|
|
UPDATE knowledges
|
|
SET enable_status = 'disabled', updated_at = NOW()
|
|
WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID)
|
|
if err != nil || command.RowsAffected() != 1 {
|
|
return errors.New("disable target knowledge")
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE chunks
|
|
SET is_enabled = FALSE, updated_at = NOW()
|
|
WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil {
|
|
return errors.New("disable target chunks")
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return errors.New("commit PostgreSQL disable")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type resourceIdentityPair struct{ id, handle string }
|
|
|
|
func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity")
|
|
}
|
|
if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete")
|
|
}
|
|
prepared := cloneSnapshot(snapshot)
|
|
known := map[string]bool{}
|
|
for index, table := range prepared.Tables {
|
|
if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid")
|
|
}
|
|
known[table.Name] = true
|
|
prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
|
}
|
|
if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch")
|
|
}
|
|
chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount)
|
|
chunkRows := prepared.Table("chunks").Rows
|
|
for index, row := range chunkRows {
|
|
id, ok := matchesChunkRow(row, identity)
|
|
if !ok {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
|
}
|
|
chunkIDs = append(chunkIDs, id)
|
|
// chunks.seq_id is a per-database global sequence value, not document
|
|
// content (WeKnora retrieval and Elasticsearch never reference it).
|
|
// Carrying a source value across environments collides with rows the
|
|
// target has already written, so let the target sequence assign it and
|
|
// advance the sequence after the transaction below.
|
|
var fields map[string]any
|
|
if err := json.Unmarshal(row, &fields); err != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
|
}
|
|
if _, present := fields["seq_id"]; present {
|
|
delete(fields, "seq_id")
|
|
rewritten, err := json.Marshal(fields)
|
|
if err != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row")
|
|
}
|
|
chunkRows[index] = rewritten
|
|
}
|
|
}
|
|
chunks := newTableSnapshot("chunks", chunkRows)
|
|
for index := range prepared.Tables {
|
|
if prepared.Tables[index].Name == "chunks" {
|
|
prepared.Tables[index] = chunks
|
|
}
|
|
}
|
|
tagIDs := map[string]bool{}
|
|
for _, row := range prepared.Table("knowledge_tags").Rows {
|
|
if !matchesKnowledgeTagRow(row, identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
|
}
|
|
var tag struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if json.Unmarshal(row, &tag) != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
|
}
|
|
tagIDs[tag.ID] = true
|
|
}
|
|
for _, row := range prepared.Table("knowledge_tag_relations").Rows {
|
|
var relation struct {
|
|
TagID string `json:"tag_id"`
|
|
}
|
|
if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch")
|
|
}
|
|
}
|
|
resources := prepared.Table("resources")
|
|
resourceIDs := make([]string, 0, len(resources.Rows))
|
|
resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows))
|
|
for rowIndex, row := range resources.Rows {
|
|
rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID)
|
|
if err != nil {
|
|
return PostgresSnapshot{}, nil, nil, err
|
|
}
|
|
resources.Rows[rowIndex] = rewritten
|
|
resourceIDs = append(resourceIDs, pair.id)
|
|
resourcePairs = append(resourcePairs, pair)
|
|
}
|
|
resources = newTableSnapshot("resources", resources.Rows)
|
|
for index := range prepared.Tables {
|
|
if prepared.Tables[index].Name == "resources" {
|
|
prepared.Tables[index] = resources
|
|
}
|
|
}
|
|
for _, row := range prepared.Table("resource_bindings").Rows {
|
|
if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch")
|
|
}
|
|
}
|
|
// wiki 是库级快照：行必须落在同一个 tenant + knowledge base 里。
|
|
for _, row := range prepared.Table("wiki_pages").Rows {
|
|
if !matchesWikiPageRow(row, identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki page scope mismatch")
|
|
}
|
|
}
|
|
for _, row := range prepared.Table("wiki_folders").Rows {
|
|
if !matchesWikiFolderRow(row, identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki folder scope mismatch")
|
|
}
|
|
}
|
|
return prepared, chunkIDs, resourcePairs, nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) {
|
|
var row map[string]any
|
|
if json.Unmarshal(raw, &row) != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource")
|
|
}
|
|
id, _ := row["id"].(string)
|
|
handle, _ := row["handle"].(string)
|
|
if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id {
|
|
return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch")
|
|
}
|
|
sourceBackendID, _ := row["storage_backend_id"].(string)
|
|
physicalPath, _ := row["physical_path"].(string)
|
|
if importer.config.TargetStorageBackendID == "" {
|
|
if sourceBackendID != "" {
|
|
return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required")
|
|
}
|
|
return raw, resourceIdentityPair{id: id, handle: handle}, nil
|
|
}
|
|
providerPath := physicalPath
|
|
if strings.HasPrefix(physicalPath, "storage://") {
|
|
rest := strings.TrimPrefix(physicalPath, "storage://")
|
|
parts := strings.SplitN(rest, "/", 2)
|
|
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
|
}
|
|
providerPath = parts[1]
|
|
} else if !validObjectLocator(physicalPath) {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
|
}
|
|
// WeKnora's official file service parses the bucket out of the nested
|
|
// provider locator (oss://<bucket>/<key>) and uses it directly for
|
|
// GetObject. Carrying the source bucket across environments makes every
|
|
// target-side preview read the wrong bucket, so rewrite it to the target
|
|
// bucket whenever the locator embeds one and a target bucket is configured.
|
|
if inner := strings.Index(providerPath, "://"); inner >= 0 && importer.config.TargetBucket != "" {
|
|
rest := providerPath[inner+3:]
|
|
if boundary := strings.Index(rest, "/"); boundary > 0 && rest[:boundary] != importer.config.TargetBucket {
|
|
providerPath = providerPath[:inner+3] + importer.config.TargetBucket + rest[boundary:]
|
|
}
|
|
}
|
|
targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath
|
|
digest := sha256.Sum256([]byte(targetPath))
|
|
row["storage_backend_id"] = importer.config.TargetStorageBackendID
|
|
row["provider"] = importer.config.TargetProvider
|
|
row["physical_path"] = targetPath
|
|
row["location_hash"] = hex.EncodeToString(digest[:])
|
|
encoded, err := json.Marshal(row)
|
|
if err != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
|
}
|
|
canonical, err := canonicalJSON(encoded)
|
|
if err != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
|
}
|
|
return canonical, resourceIdentityPair{id: id, handle: handle}, nil
|
|
}
|
|
|
|
func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error {
|
|
if len(table.Rows) == 0 {
|
|
return nil
|
|
}
|
|
columns, err := tableColumns(ctx, tx, table.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if table.Name == "chunks" {
|
|
// seq_id is assigned by the target sequence; it must not appear in the
|
|
// explicit column list (an explicit NULL would violate NOT NULL) and it
|
|
// is removed from snapshot rows during prepare.
|
|
filtered := columns[:0]
|
|
for _, column := range columns {
|
|
if column != "seq_id" {
|
|
filtered = append(filtered, column)
|
|
}
|
|
}
|
|
columns = filtered
|
|
}
|
|
quotedTable := pgx.Identifier{table.Name}.Sanitize()
|
|
quotedColumns := make([]string, len(columns))
|
|
for index, column := range columns {
|
|
quotedColumns[index] = pgx.Identifier{column}.Sanitize()
|
|
}
|
|
columnList := strings.Join(quotedColumns, ", ")
|
|
selectList := make([]string, len(quotedColumns))
|
|
for index, column := range quotedColumns {
|
|
selectList[index] = "record_value." + column
|
|
}
|
|
query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable)
|
|
if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" {
|
|
updates := make([]string, 0, len(quotedColumns)-1)
|
|
for index, column := range columns {
|
|
if column == "id" {
|
|
continue
|
|
}
|
|
updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index])
|
|
}
|
|
query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ")
|
|
}
|
|
for _, row := range table.Rows {
|
|
if _, err := tx.Exec(ctx, query, string(row)); err != nil {
|
|
return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) {
|
|
if !validReleaseTable(tableName) {
|
|
return nil, errors.New("invalid PostgreSQL import table")
|
|
}
|
|
rows, err := tx.Query(ctx, `
|
|
SELECT column_name
|
|
FROM information_schema.columns
|
|
WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER'
|
|
ORDER BY ordinal_position
|
|
`, tableName)
|
|
if err != nil {
|
|
return nil, errors.New("inspect target PostgreSQL table")
|
|
}
|
|
defer rows.Close()
|
|
columns := make([]string, 0)
|
|
for rows.Next() {
|
|
var column string
|
|
if rows.Scan(&column) != nil || !validSQLIdentifier(column) {
|
|
return nil, errors.New("invalid target PostgreSQL column")
|
|
}
|
|
columns = append(columns, column)
|
|
}
|
|
if rows.Err() != nil || len(columns) == 0 {
|
|
return nil, errors.New("inspect target PostgreSQL table")
|
|
}
|
|
return columns, nil
|
|
}
|
|
|
|
func validSQLIdentifier(value string) bool {
|
|
if value == "" || len(value) > 63 {
|
|
return false
|
|
}
|
|
for index, char := range value {
|
|
if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func validObjectLocator(value string) bool {
|
|
if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") {
|
|
return false
|
|
}
|
|
if separator := strings.Index(value, "://"); separator >= 0 {
|
|
if !validSearchIdentifier(value[:separator]) {
|
|
return false
|
|
}
|
|
value = value[separator+3:]
|
|
}
|
|
cleaned := path.Clean(value)
|
|
return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../")
|
|
}
|
|
|
|
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
|
|
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))}
|
|
for index, table := range snapshot.Tables {
|
|
result.Tables[index] = table
|
|
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
|
}
|
|
return result
|
|
}
|