package v071 import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "path" "strings" "github.com/jackc/pgx/v5" ) type postgresTransactionalDB interface { Begin(ctx context.Context) (pgx.Tx, error) } type PostgresImporterConfig struct { TargetStorageBackendID string TargetProvider string // TargetBucket rewrites the bucket embedded in nested provider locators // (oss:///). WeKnora resolves GetFile from the path's inner // bucket, so a source-env bucket leaking into the target row makes every // preview/download read the wrong bucket with target credentials. TargetBucket string } type PostgresImporter struct { db postgresTransactionalDB config PostgresImporterConfig } // clearStatement 是导入前置清理语句(文档级或库级 wiki)。 type clearStatement struct { sql string args []any } func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) { if db == nil || len(options) > 1 { return nil, errors.New("invalid PostgreSQL importer configuration") } var config PostgresImporterConfig if len(options) == 1 { config = options[0] } config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID) config.TargetProvider = strings.TrimSpace(config.TargetProvider) config.TargetBucket = strings.TrimSpace(config.TargetBucket) if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") || (config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) { return nil, errors.New("invalid target storage backend configuration") } if config.TargetBucket != "" && !validSearchIdentifier(config.TargetBucket) { return nil, errors.New("invalid target storage bucket configuration") } return &PostgresImporter{db: db, config: config}, nil } func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error { prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot) if err != nil { return err } tx, err := importer.db.Begin(ctx) if err != nil { return errors.New("begin PostgreSQL document import") } defer tx.Rollback(ctx) var version int var dirty bool if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { return errors.New("target WeKnora schema contract mismatch") } if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil { return errors.New("lock target knowledge") } _, hasWiki := wikiSnapshotRows(prepared) if hasWiki { // 库级 wiki 替换加知识库维度的锁(命名空间 1),与文档维度的锁区分开。 if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil { return errors.New("lock target wiki knowledge base") } } var crossScope int if err := tx.QueryRow(ctx, ` SELECT COUNT(*)::INTEGER FROM knowledges WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2) `, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 { return errors.New("target knowledge ID conflicts with another scope") } for _, pair := range resourcePairs { if err := tx.QueryRow(ctx, ` SELECT COUNT(*)::INTEGER FROM resources WHERE (id = $1 OR handle = $2) AND NOT (id = $1 AND handle = $2 AND tenant_id = $3) `, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 { return errors.New("target resource identity conflict") } } for _, raw := range prepared.Table("knowledge_tags").Rows { var tag struct { ID string `json:"id"` } if json.Unmarshal(raw, &tag) != nil { return errors.New("invalid target knowledge tag") } var crossScope int if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 { return errors.New("target knowledge tag identity conflict") } } clearStatements := []clearStatement{ {`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}}, {`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}}, {`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}}, } for _, statement := range clearStatements { if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil { return errors.New("clear prior target document rows") } } insertOrder := []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"} for _, tableName := range insertOrder { table := prepared.Table(tableName) if err := insertSnapshotRows(ctx, tx, table); err != nil { return err } } // wiki 是库级快照,但不能盲删整库:先按 source_refs 判定归属,只替换本包 // 拥有的页面,其余(其他文档、agent 手写页)一行不动。 if hasWiki { if _, err := importer.replaceWiki(ctx, tx, identity, prepared); err != nil { return err } } if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil { return errors.New("advance target chunk sequence") } if err := tx.Commit(ctx); err != nil { return errors.New("commit PostgreSQL document import") } return nil } // ReplaceWiki 只应用库级 wiki 快照(撤权 / 回滚到旧版时使用)。 // 撤权路径上文档行只做 disable(不能把文档行重新灌回去),而 wiki 必须整体 // 替换成撤权后的快照,否则生产 wiki 会一直留着已撤权文档的页面与引用。 func (importer *PostgresImporter) ReplaceWiki(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error { if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) { return errors.New("invalid PostgreSQL wiki import identity") } tables, hasWiki := wikiSnapshotRows(snapshot) if !hasWiki { // 老包 / 未启用 wiki 的库:不动生产 wiki。 return nil } for _, table := range tables { if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 { return errors.New("PostgreSQL wiki import snapshot is invalid") } } for _, row := range snapshot.Table("wiki_pages").Rows { if !matchesWikiPageRow(row, identity) { return errors.New("PostgreSQL wiki page scope mismatch") } } for _, row := range snapshot.Table("wiki_folders").Rows { if !matchesWikiFolderRow(row, identity) { return errors.New("PostgreSQL wiki folder scope mismatch") } } tx, err := importer.db.Begin(ctx) if err != nil { return errors.New("begin PostgreSQL wiki import") } defer tx.Rollback(ctx) var version int var dirty bool if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { return errors.New("target WeKnora schema contract mismatch") } if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 1))`, identity.KnowledgeBaseID); err != nil { return errors.New("lock target wiki knowledge base") } // 撤权 / 回滚也走同一套归属判定:只撤本包知识拥有的页面与引用, // 目标库里其他文档和 agent 手写的内容一行不动。 if _, err := importer.replaceWiki(ctx, tx, identity, snapshot); err != nil { return err } if err := tx.Commit(ctx); err != nil { return errors.New("commit PostgreSQL wiki import") } return nil } func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error { if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { return errors.New("invalid PostgreSQL disable identity") } tx, err := importer.db.Begin(ctx) if err != nil { return errors.New("begin PostgreSQL disable") } defer tx.Rollback(ctx) var version int var dirty bool if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { return errors.New("target WeKnora schema contract mismatch") } if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil { return errors.New("lock target knowledge") } command, err := tx.Exec(ctx, ` UPDATE knowledges SET enable_status = 'disabled', updated_at = NOW() WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL `, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID) if err != nil || command.RowsAffected() != 1 { return errors.New("disable target knowledge") } if _, err := tx.Exec(ctx, ` UPDATE chunks SET is_enabled = FALSE, updated_at = NOW() WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL `, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil { return errors.New("disable target chunks") } if err := tx.Commit(ctx); err != nil { return errors.New("commit PostgreSQL disable") } return nil } type resourceIdentityPair struct{ id, handle string } func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) { if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity") } if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete") } prepared := cloneSnapshot(snapshot) known := map[string]bool{} for index, table := range prepared.Tables { if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid") } known[table.Name] = true prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...) } if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch") } chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount) chunkRows := prepared.Table("chunks").Rows for index, row := range chunkRows { id, ok := matchesChunkRow(row, identity) if !ok { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch") } chunkIDs = append(chunkIDs, id) // chunks.seq_id is a per-database global sequence value, not document // content (WeKnora retrieval and Elasticsearch never reference it). // Carrying a source value across environments collides with rows the // target has already written, so let the target sequence assign it and // advance the sequence after the transaction below. var fields map[string]any if err := json.Unmarshal(row, &fields); err != nil { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch") } if _, present := fields["seq_id"]; present { delete(fields, "seq_id") rewritten, err := json.Marshal(fields) if err != nil { return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row") } chunkRows[index] = rewritten } } chunks := newTableSnapshot("chunks", chunkRows) for index := range prepared.Tables { if prepared.Tables[index].Name == "chunks" { prepared.Tables[index] = chunks } } tagIDs := map[string]bool{} for _, row := range prepared.Table("knowledge_tags").Rows { if !matchesKnowledgeTagRow(row, identity) { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch") } var tag struct { ID string `json:"id"` } if json.Unmarshal(row, &tag) != nil { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch") } tagIDs[tag.ID] = true } for _, row := range prepared.Table("knowledge_tag_relations").Rows { var relation struct { TagID string `json:"tag_id"` } if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch") } } resources := prepared.Table("resources") resourceIDs := make([]string, 0, len(resources.Rows)) resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows)) for rowIndex, row := range resources.Rows { rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID) if err != nil { return PostgresSnapshot{}, nil, nil, err } resources.Rows[rowIndex] = rewritten resourceIDs = append(resourceIDs, pair.id) resourcePairs = append(resourcePairs, pair) } resources = newTableSnapshot("resources", resources.Rows) for index := range prepared.Tables { if prepared.Tables[index].Name == "resources" { prepared.Tables[index] = resources } } for _, row := range prepared.Table("resource_bindings").Rows { if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch") } } // wiki 是库级快照:行必须落在同一个 tenant + knowledge base 里。 for _, row := range prepared.Table("wiki_pages").Rows { if !matchesWikiPageRow(row, identity) { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki page scope mismatch") } } for _, row := range prepared.Table("wiki_folders").Rows { if !matchesWikiFolderRow(row, identity) { return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import wiki folder scope mismatch") } } return prepared, chunkIDs, resourcePairs, nil } func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) { var row map[string]any if json.Unmarshal(raw, &row) != nil { return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource") } id, _ := row["id"].(string) handle, _ := row["handle"].(string) if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id { return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch") } sourceBackendID, _ := row["storage_backend_id"].(string) physicalPath, _ := row["physical_path"].(string) if importer.config.TargetStorageBackendID == "" { if sourceBackendID != "" { return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required") } return raw, resourceIdentityPair{id: id, handle: handle}, nil } providerPath := physicalPath if strings.HasPrefix(physicalPath, "storage://") { rest := strings.TrimPrefix(physicalPath, "storage://") parts := strings.SplitN(rest, "/", 2) if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) { return nil, resourceIdentityPair{}, errors.New("invalid source storage locator") } providerPath = parts[1] } else if !validObjectLocator(physicalPath) { return nil, resourceIdentityPair{}, errors.New("invalid source storage locator") } // WeKnora's official file service parses the bucket out of the nested // provider locator (oss:///) and uses it directly for // GetObject. Carrying the source bucket across environments makes every // target-side preview read the wrong bucket, so rewrite it to the target // bucket whenever the locator embeds one and a target bucket is configured. if inner := strings.Index(providerPath, "://"); inner >= 0 && importer.config.TargetBucket != "" { rest := providerPath[inner+3:] if boundary := strings.Index(rest, "/"); boundary > 0 && rest[:boundary] != importer.config.TargetBucket { providerPath = providerPath[:inner+3] + importer.config.TargetBucket + rest[boundary:] } } targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath digest := sha256.Sum256([]byte(targetPath)) row["storage_backend_id"] = importer.config.TargetStorageBackendID row["provider"] = importer.config.TargetProvider row["physical_path"] = targetPath row["location_hash"] = hex.EncodeToString(digest[:]) encoded, err := json.Marshal(row) if err != nil { return nil, resourceIdentityPair{}, errors.New("rewrite target resource") } canonical, err := canonicalJSON(encoded) if err != nil { return nil, resourceIdentityPair{}, errors.New("rewrite target resource") } return canonical, resourceIdentityPair{id: id, handle: handle}, nil } func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error { if len(table.Rows) == 0 { return nil } columns, err := tableColumns(ctx, tx, table.Name) if err != nil { return err } if table.Name == "chunks" { // seq_id is assigned by the target sequence; it must not appear in the // explicit column list (an explicit NULL would violate NOT NULL) and it // is removed from snapshot rows during prepare. filtered := columns[:0] for _, column := range columns { if column != "seq_id" { filtered = append(filtered, column) } } columns = filtered } quotedTable := pgx.Identifier{table.Name}.Sanitize() quotedColumns := make([]string, len(columns)) for index, column := range columns { quotedColumns[index] = pgx.Identifier{column}.Sanitize() } columnList := strings.Join(quotedColumns, ", ") selectList := make([]string, len(quotedColumns)) for index, column := range quotedColumns { selectList[index] = "record_value." + column } query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable) if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" { updates := make([]string, 0, len(quotedColumns)-1) for index, column := range columns { if column == "id" { continue } updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index]) } query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ") } for _, row := range table.Rows { if _, err := tx.Exec(ctx, query, string(row)); err != nil { return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err) } } return nil } func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) { if !validReleaseTable(tableName) { return nil, errors.New("invalid PostgreSQL import table") } rows, err := tx.Query(ctx, ` SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER' ORDER BY ordinal_position `, tableName) if err != nil { return nil, errors.New("inspect target PostgreSQL table") } defer rows.Close() columns := make([]string, 0) for rows.Next() { var column string if rows.Scan(&column) != nil || !validSQLIdentifier(column) { return nil, errors.New("invalid target PostgreSQL column") } columns = append(columns, column) } if rows.Err() != nil || len(columns) == 0 { return nil, errors.New("inspect target PostgreSQL table") } return columns, nil } func validSQLIdentifier(value string) bool { if value == "" || len(value) > 63 { return false } for index, char := range value { if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') { return false } } return true } func validObjectLocator(value string) bool { if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") { return false } if separator := strings.Index(value, "://"); separator >= 0 { if !validSearchIdentifier(value[:separator]) { return false } value = value[separator+3:] } cleaned := path.Clean(value) return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../") } func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot { result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))} for index, table := range snapshot.Tables { result.Tables[index] = table result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...) } return result }