dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
417 lines
16 KiB
Go
417 lines
16 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
type postgresTransactionalDB interface {
|
|
Begin(ctx context.Context) (pgx.Tx, error)
|
|
}
|
|
|
|
type PostgresImporterConfig struct {
|
|
TargetStorageBackendID string
|
|
TargetProvider string
|
|
}
|
|
|
|
type PostgresImporter struct {
|
|
db postgresTransactionalDB
|
|
config PostgresImporterConfig
|
|
}
|
|
|
|
func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) {
|
|
if db == nil || len(options) > 1 {
|
|
return nil, errors.New("invalid PostgreSQL importer configuration")
|
|
}
|
|
var config PostgresImporterConfig
|
|
if len(options) == 1 {
|
|
config = options[0]
|
|
}
|
|
config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID)
|
|
config.TargetProvider = strings.TrimSpace(config.TargetProvider)
|
|
if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") ||
|
|
(config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) {
|
|
return nil, errors.New("invalid target storage backend configuration")
|
|
}
|
|
return &PostgresImporter{db: db, config: config}, nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error {
|
|
prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := importer.db.Begin(ctx)
|
|
if err != nil {
|
|
return errors.New("begin PostgreSQL document import")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var version int
|
|
var dirty bool
|
|
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
|
return errors.New("target WeKnora schema contract mismatch")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
|
return errors.New("lock target knowledge")
|
|
}
|
|
var crossScope int
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT COUNT(*)::INTEGER
|
|
FROM knowledges
|
|
WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2)
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target knowledge ID conflicts with another scope")
|
|
}
|
|
for _, pair := range resourcePairs {
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT COUNT(*)::INTEGER
|
|
FROM resources
|
|
WHERE (id = $1 OR handle = $2)
|
|
AND NOT (id = $1 AND handle = $2 AND tenant_id = $3)
|
|
`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target resource identity conflict")
|
|
}
|
|
}
|
|
for _, raw := range prepared.Table("knowledge_tags").Rows {
|
|
var tag struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if json.Unmarshal(raw, &tag) != nil {
|
|
return errors.New("invalid target knowledge tag")
|
|
}
|
|
var crossScope int
|
|
if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 {
|
|
return errors.New("target knowledge tag identity conflict")
|
|
}
|
|
}
|
|
for _, statement := range []struct {
|
|
sql string
|
|
args []any
|
|
}{
|
|
{`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}},
|
|
{`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}},
|
|
{`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}},
|
|
} {
|
|
if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil {
|
|
return errors.New("clear prior target document rows")
|
|
}
|
|
}
|
|
for _, tableName := range []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"} {
|
|
table := prepared.Table(tableName)
|
|
if err := insertSnapshotRows(ctx, tx, table); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil {
|
|
return errors.New("advance target chunk sequence")
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return errors.New("commit PostgreSQL document import")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return errors.New("invalid PostgreSQL disable identity")
|
|
}
|
|
tx, err := importer.db.Begin(ctx)
|
|
if err != nil {
|
|
return errors.New("begin PostgreSQL disable")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var version int
|
|
var dirty bool
|
|
if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty {
|
|
return errors.New("target WeKnora schema contract mismatch")
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil {
|
|
return errors.New("lock target knowledge")
|
|
}
|
|
command, err := tx.Exec(ctx, `
|
|
UPDATE knowledges
|
|
SET enable_status = 'disabled', updated_at = NOW()
|
|
WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID)
|
|
if err != nil || command.RowsAffected() != 1 {
|
|
return errors.New("disable target knowledge")
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE chunks
|
|
SET is_enabled = FALSE, updated_at = NOW()
|
|
WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL
|
|
`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil {
|
|
return errors.New("disable target chunks")
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return errors.New("commit PostgreSQL disable")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type resourceIdentityPair struct{ id, handle string }
|
|
|
|
func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity")
|
|
}
|
|
if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete")
|
|
}
|
|
prepared := cloneSnapshot(snapshot)
|
|
known := map[string]bool{}
|
|
for index, table := range prepared.Tables {
|
|
if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid")
|
|
}
|
|
known[table.Name] = true
|
|
prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
|
}
|
|
if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch")
|
|
}
|
|
chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount)
|
|
chunkRows := prepared.Table("chunks").Rows
|
|
for index, row := range chunkRows {
|
|
id, ok := matchesChunkRow(row, identity)
|
|
if !ok {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
|
}
|
|
chunkIDs = append(chunkIDs, id)
|
|
// chunks.seq_id is a per-database global sequence value, not document
|
|
// content (WeKnora retrieval and Elasticsearch never reference it).
|
|
// Carrying a source value across environments collides with rows the
|
|
// target has already written, so let the target sequence assign it and
|
|
// advance the sequence after the transaction below.
|
|
var fields map[string]any
|
|
if err := json.Unmarshal(row, &fields); err != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch")
|
|
}
|
|
if _, present := fields["seq_id"]; present {
|
|
delete(fields, "seq_id")
|
|
rewritten, err := json.Marshal(fields)
|
|
if err != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row")
|
|
}
|
|
chunkRows[index] = rewritten
|
|
}
|
|
}
|
|
chunks := newTableSnapshot("chunks", chunkRows)
|
|
for index := range prepared.Tables {
|
|
if prepared.Tables[index].Name == "chunks" {
|
|
prepared.Tables[index] = chunks
|
|
}
|
|
}
|
|
tagIDs := map[string]bool{}
|
|
for _, row := range prepared.Table("knowledge_tags").Rows {
|
|
if !matchesKnowledgeTagRow(row, identity) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
|
}
|
|
var tag struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if json.Unmarshal(row, &tag) != nil {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch")
|
|
}
|
|
tagIDs[tag.ID] = true
|
|
}
|
|
for _, row := range prepared.Table("knowledge_tag_relations").Rows {
|
|
var relation struct {
|
|
TagID string `json:"tag_id"`
|
|
}
|
|
if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch")
|
|
}
|
|
}
|
|
resources := prepared.Table("resources")
|
|
resourceIDs := make([]string, 0, len(resources.Rows))
|
|
resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows))
|
|
for rowIndex, row := range resources.Rows {
|
|
rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID)
|
|
if err != nil {
|
|
return PostgresSnapshot{}, nil, nil, err
|
|
}
|
|
resources.Rows[rowIndex] = rewritten
|
|
resourceIDs = append(resourceIDs, pair.id)
|
|
resourcePairs = append(resourcePairs, pair)
|
|
}
|
|
resources = newTableSnapshot("resources", resources.Rows)
|
|
for index := range prepared.Tables {
|
|
if prepared.Tables[index].Name == "resources" {
|
|
prepared.Tables[index] = resources
|
|
}
|
|
}
|
|
for _, row := range prepared.Table("resource_bindings").Rows {
|
|
if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) {
|
|
return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch")
|
|
}
|
|
}
|
|
return prepared, chunkIDs, resourcePairs, nil
|
|
}
|
|
|
|
func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) {
|
|
var row map[string]any
|
|
if json.Unmarshal(raw, &row) != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource")
|
|
}
|
|
id, _ := row["id"].(string)
|
|
handle, _ := row["handle"].(string)
|
|
if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id {
|
|
return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch")
|
|
}
|
|
sourceBackendID, _ := row["storage_backend_id"].(string)
|
|
physicalPath, _ := row["physical_path"].(string)
|
|
if importer.config.TargetStorageBackendID == "" {
|
|
if sourceBackendID != "" {
|
|
return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required")
|
|
}
|
|
return raw, resourceIdentityPair{id: id, handle: handle}, nil
|
|
}
|
|
providerPath := physicalPath
|
|
if strings.HasPrefix(physicalPath, "storage://") {
|
|
rest := strings.TrimPrefix(physicalPath, "storage://")
|
|
parts := strings.SplitN(rest, "/", 2)
|
|
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
|
}
|
|
providerPath = parts[1]
|
|
} else if !validObjectLocator(physicalPath) {
|
|
return nil, resourceIdentityPair{}, errors.New("invalid source storage locator")
|
|
}
|
|
targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath
|
|
digest := sha256.Sum256([]byte(targetPath))
|
|
row["storage_backend_id"] = importer.config.TargetStorageBackendID
|
|
row["provider"] = importer.config.TargetProvider
|
|
row["physical_path"] = targetPath
|
|
row["location_hash"] = hex.EncodeToString(digest[:])
|
|
encoded, err := json.Marshal(row)
|
|
if err != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
|
}
|
|
canonical, err := canonicalJSON(encoded)
|
|
if err != nil {
|
|
return nil, resourceIdentityPair{}, errors.New("rewrite target resource")
|
|
}
|
|
return canonical, resourceIdentityPair{id: id, handle: handle}, nil
|
|
}
|
|
|
|
func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error {
|
|
if len(table.Rows) == 0 {
|
|
return nil
|
|
}
|
|
columns, err := tableColumns(ctx, tx, table.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if table.Name == "chunks" {
|
|
// seq_id is assigned by the target sequence; it must not appear in the
|
|
// explicit column list (an explicit NULL would violate NOT NULL) and it
|
|
// is removed from snapshot rows during prepare.
|
|
filtered := columns[:0]
|
|
for _, column := range columns {
|
|
if column != "seq_id" {
|
|
filtered = append(filtered, column)
|
|
}
|
|
}
|
|
columns = filtered
|
|
}
|
|
quotedTable := pgx.Identifier{table.Name}.Sanitize()
|
|
quotedColumns := make([]string, len(columns))
|
|
for index, column := range columns {
|
|
quotedColumns[index] = pgx.Identifier{column}.Sanitize()
|
|
}
|
|
columnList := strings.Join(quotedColumns, ", ")
|
|
selectList := make([]string, len(quotedColumns))
|
|
for index, column := range quotedColumns {
|
|
selectList[index] = "record_value." + column
|
|
}
|
|
query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable)
|
|
if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" {
|
|
updates := make([]string, 0, len(quotedColumns)-1)
|
|
for index, column := range columns {
|
|
if column == "id" {
|
|
continue
|
|
}
|
|
updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index])
|
|
}
|
|
query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ")
|
|
}
|
|
for _, row := range table.Rows {
|
|
if _, err := tx.Exec(ctx, query, string(row)); err != nil {
|
|
return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) {
|
|
if !validReleaseTable(tableName) {
|
|
return nil, errors.New("invalid PostgreSQL import table")
|
|
}
|
|
rows, err := tx.Query(ctx, `
|
|
SELECT column_name
|
|
FROM information_schema.columns
|
|
WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER'
|
|
ORDER BY ordinal_position
|
|
`, tableName)
|
|
if err != nil {
|
|
return nil, errors.New("inspect target PostgreSQL table")
|
|
}
|
|
defer rows.Close()
|
|
columns := make([]string, 0)
|
|
for rows.Next() {
|
|
var column string
|
|
if rows.Scan(&column) != nil || !validSQLIdentifier(column) {
|
|
return nil, errors.New("invalid target PostgreSQL column")
|
|
}
|
|
columns = append(columns, column)
|
|
}
|
|
if rows.Err() != nil || len(columns) == 0 {
|
|
return nil, errors.New("inspect target PostgreSQL table")
|
|
}
|
|
return columns, nil
|
|
}
|
|
|
|
func validSQLIdentifier(value string) bool {
|
|
if value == "" || len(value) > 63 {
|
|
return false
|
|
}
|
|
for index, char := range value {
|
|
if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func validObjectLocator(value string) bool {
|
|
if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") {
|
|
return false
|
|
}
|
|
if separator := strings.Index(value, "://"); separator >= 0 {
|
|
if !validSearchIdentifier(value[:separator]) {
|
|
return false
|
|
}
|
|
value = value[separator+3:]
|
|
}
|
|
cleaned := path.Clean(value)
|
|
return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../")
|
|
}
|
|
|
|
func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot {
|
|
result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))}
|
|
for index, table := range snapshot.Tables {
|
|
result.Tables[index] = table
|
|
result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...)
|
|
}
|
|
return result
|
|
}
|