zheromiku
dcdc8d1ddd
全栈知识治理平台首次入库（P1-P10 + 发布链路模拟验证闭环）： - 治理核心：源目录/变更集/人工审核/发布批次/发布执行（fail-closed 闸口） - 测试通道：WeKnora 测试 KB 上传/轮询/验收；poll 完成时写入 parser_generation/index_generation，修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071（本地 source=测试角色 / 云端 target=生产角色 模拟验证通过）： * ES 导出/导入 mapping 自适应（.keyword 子字段、可选 tenant_id） * OSS locator 支持嵌套 scheme；chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换；verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id（strict 解码） - 三通道发布+回滚演练实测：publish 4/4 verified，检索命中， rollback 4/4 rolled_back，目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一，120 用例通过 - 门禁：go vet / go test 全包通过，gofmt 干净
308 lines
16 KiB
Go
308 lines
16 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
var ErrCatalogNodeNotFound = errors.New("catalog node not found")
|
|
|
|
func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) {
|
|
rows, err := s.db.Query(ctx, `
|
|
SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id,
|
|
snapshot.folder_count - 1, snapshot.file_count, head.activated_at,
|
|
'ready', COALESCE(refresh.duration_ms,0)
|
|
FROM source_catalog_heads head
|
|
JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id
|
|
JOIN source_connections connection ON connection.id=head.source_connection_id
|
|
JOIN source_catalog_nodes root ON root.snapshot_id=snapshot.id AND root.parent_node_id IS NULL
|
|
LEFT JOIN LATERAL (
|
|
SELECT (EXTRACT(EPOCH FROM (job.completed_at-COALESCE(job.started_at,job.created_at)))*1000)::BIGINT AS duration_ms
|
|
FROM jobs job
|
|
WHERE job.type='source.catalog.refresh' AND job.state='completed'
|
|
AND job.payload->>'connection_id'=connection.id
|
|
ORDER BY job.completed_at DESC
|
|
LIMIT 1
|
|
) refresh ON TRUE
|
|
ORDER BY CASE connection.source_type WHEN 'dingtalk' THEN 1 WHEN 'pp' THEN 2 WHEN 'alist' THEN 3 ELSE 4 END,
|
|
snapshot.source_label, connection.id`)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list catalog sources: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]governance.CatalogSource, 0)
|
|
for rows.Next() {
|
|
var item governance.CatalogSource
|
|
var sourceType string
|
|
if err := rows.Scan(&item.ConnectionID, &sourceType, &item.Name, &item.RootNodeID, &item.FolderCount, &item.FileCount, &item.ObservedAt, &item.SnapshotStatus, &item.LastRefreshDurationMS); err != nil {
|
|
return nil, fmt.Errorf("scan catalog source: %w", err)
|
|
}
|
|
item.SourceType = governance.SourceType(sourceType)
|
|
item.SnapshotOnly = true
|
|
if !item.SourceType.Valid() || item.FolderCount < 0 || item.FileCount < 0 {
|
|
return nil, errors.New("invalid catalog source projection")
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("iterate catalog sources: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogNodeFilter) (governance.CatalogNodePage, error) {
|
|
if filter.ConnectionID == "" || len(filter.ConnectionID) > 36 || filter.ParentNodeID == "" || len(filter.ParentNodeID) > 36 || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 {
|
|
return governance.CatalogNodePage{}, errors.New("invalid catalog node filter")
|
|
}
|
|
page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset}
|
|
var parentID *string
|
|
err := s.db.QueryRow(ctx, `
|
|
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count,
|
|
node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at
|
|
FROM source_catalog_heads head
|
|
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
|
WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan(
|
|
&page.Current.NodeID, &parentID, &page.Current.NodeType, &page.Current.Name, &page.Current.DirectChildCount,
|
|
&page.Current.SubtreeFileCount, &page.Current.SubtreeBytes, &page.Current.SizeBytes, &page.Current.ModifiedAt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return governance.CatalogNodePage{}, ErrCatalogNodeNotFound
|
|
}
|
|
if err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("get catalog parent: %w", err)
|
|
}
|
|
if parentID != nil {
|
|
page.Current.ParentNodeID = *parentID
|
|
}
|
|
page.Current.Visibility = "unknown"
|
|
|
|
breadcrumbs, err := s.db.Query(ctx, `
|
|
WITH RECURSIVE trail AS (
|
|
SELECT node.snapshot_id,node.node_id,node.parent_node_id,node.name,0 AS depth
|
|
FROM source_catalog_heads head
|
|
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
|
WHERE head.source_connection_id=$1 AND node.node_id=$2
|
|
UNION ALL
|
|
SELECT parent.snapshot_id,parent.node_id,parent.parent_node_id,parent.name,trail.depth+1
|
|
FROM trail JOIN source_catalog_nodes parent
|
|
ON parent.snapshot_id=trail.snapshot_id AND parent.node_id=trail.parent_node_id
|
|
)
|
|
SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter.ParentNodeID)
|
|
if err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("list catalog breadcrumbs: %w", err)
|
|
}
|
|
for breadcrumbs.Next() {
|
|
var item governance.CatalogBreadcrumb
|
|
if err := breadcrumbs.Scan(&item.NodeID, &item.Name); err != nil {
|
|
breadcrumbs.Close()
|
|
return governance.CatalogNodePage{}, fmt.Errorf("scan catalog breadcrumb: %w", err)
|
|
}
|
|
page.Breadcrumbs = append(page.Breadcrumbs, item)
|
|
}
|
|
if err := breadcrumbs.Err(); err != nil {
|
|
breadcrumbs.Close()
|
|
return governance.CatalogNodePage{}, fmt.Errorf("iterate catalog breadcrumbs: %w", err)
|
|
}
|
|
breadcrumbs.Close()
|
|
|
|
if err := s.db.QueryRow(ctx, `
|
|
SELECT COUNT(*)::integer FROM source_catalog_heads head
|
|
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
|
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err)
|
|
}
|
|
|
|
rows, err := s.db.Query(ctx, `
|
|
WITH RECURSIVE current_head AS (
|
|
SELECT snapshot_id FROM source_catalog_heads WHERE source_connection_id=$1
|
|
), selected AS MATERIALIZED (
|
|
SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
|
WHERE node.parent_node_id=$2
|
|
ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id
|
|
LIMIT $3 OFFSET $4
|
|
), descendants(ancestor_id,snapshot_id,node_id) AS (
|
|
SELECT node_id,snapshot_id,node_id FROM selected
|
|
UNION ALL
|
|
SELECT descendants.ancestor_id,child.snapshot_id,child.node_id
|
|
FROM descendants JOIN source_catalog_nodes child
|
|
ON child.snapshot_id=descendants.snapshot_id AND child.parent_node_id=descendants.node_id
|
|
), file_state AS (
|
|
SELECT descendants.ancestor_id,leaf.source_item_id,
|
|
COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility,
|
|
publication.id AS publication_id,publication.ingest_state,publication.processing_state,
|
|
current_version.content_state AS version_content_state,content.content_state AS object_content_state,
|
|
content.local_spool_ref
|
|
FROM descendants
|
|
JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file'
|
|
LEFT JOIN source_items item ON item.id=leaf.source_item_id
|
|
LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision
|
|
LEFT JOIN content_objects content ON content.id=current_version.content_object_id
|
|
LEFT JOIN source_item_scope_overrides item_scope ON item_scope.source_item_id=item.id
|
|
LEFT JOIN LATERAL (
|
|
SELECT rule.visibility FROM directory_scope_rules rule
|
|
WHERE rule.source_connection_id=$1
|
|
AND (rule.directory_path='' OR leaf.relative_path=rule.directory_path
|
|
OR LEFT(leaf.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
|
|
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1
|
|
) directory_scope ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT publication.id,publication.ingest_state,publication.processing_state
|
|
FROM test_publications publication
|
|
JOIN source_item_versions version ON version.id=publication.source_item_version_id
|
|
WHERE version.source_item_id=item.id
|
|
ORDER BY publication.updated_at DESC,publication.id LIMIT 1
|
|
) publication ON TRUE
|
|
), stats AS (
|
|
SELECT ancestor_id,
|
|
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count,
|
|
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count,
|
|
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count,
|
|
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='failed' OR processing_state IN ('failed','stalled')))::integer AS failed_count,
|
|
COUNT(DISTINCT source_item_id) FILTER (WHERE version_content_state='verified' AND object_content_state='verified' AND local_spool_ref<>'')::integer AS routable_count,
|
|
CASE WHEN COUNT(DISTINCT effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL)=0 THEN 'unknown'
|
|
WHEN COUNT(DISTINCT effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL)=1 THEN MIN(effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL)
|
|
ELSE 'mixed' END AS visibility
|
|
FROM file_state GROUP BY ancestor_id
|
|
)
|
|
SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name,
|
|
selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes,
|
|
selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'),
|
|
COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0),
|
|
COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0)
|
|
FROM selected LEFT JOIN stats ON stats.ancestor_id=selected.node_id
|
|
ORDER BY CASE selected.node_type WHEN 'folder' THEN 0 ELSE 1 END,selected.sort_order,lower(selected.name),selected.node_id`, filter.ConnectionID, filter.ParentNodeID, filter.Limit, filter.Offset)
|
|
if err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("list catalog nodes: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var item governance.CatalogNode
|
|
var parent *string
|
|
if err := rows.Scan(&item.NodeID, &parent, &item.NodeType, &item.Name, &item.DirectChildCount, &item.SubtreeFileCount, &item.SubtreeBytes, &item.SizeBytes, &item.ModifiedAt, &item.ContentSHA256, &item.Visibility, &item.ImportedCount, &item.ParsedCount, &item.ActiveCount, &item.FailedCount, &item.RoutableCount); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("scan catalog node: %w", err)
|
|
}
|
|
if parent != nil {
|
|
item.ParentNodeID = *parent
|
|
}
|
|
page.Items = append(page.Items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("iterate catalog nodes: %w", err)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
var ErrCatalogSnapshotNotFound = errors.New("source catalog snapshot not found")
|
|
|
|
// ListCatalogSnapshots returns the persisted snapshot history for one configured
|
|
// connection. The head snapshot is returned first (current view), followed by
|
|
// older snapshots newest-first. It never returns locator hashes or snapshot hashes.
|
|
func (s *Store) ListCatalogSnapshots(ctx context.Context, connectionID string, limit, offset int) (governance.CatalogSnapshotPage, error) {
|
|
if connectionID == "" || len(connectionID) > 36 || limit < 1 || limit > 100 || offset < 0 || offset > 1000000 {
|
|
return governance.CatalogSnapshotPage{}, errors.New("invalid catalog snapshot filter")
|
|
}
|
|
var exists bool
|
|
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, connectionID).Scan(&exists); err != nil {
|
|
return governance.CatalogSnapshotPage{}, fmt.Errorf("inspect snapshot connection: %w", err)
|
|
}
|
|
if !exists {
|
|
return governance.CatalogSnapshotPage{}, ErrCatalogSnapshotNotFound
|
|
}
|
|
page := governance.CatalogSnapshotPage{Items: []governance.CatalogSnapshot{}, Limit: limit, Offset: offset}
|
|
if err := s.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_snapshots WHERE source_connection_id=$1`, connectionID).Scan(&page.Total); err != nil {
|
|
return governance.CatalogSnapshotPage{}, fmt.Errorf("count catalog snapshots: %w", err)
|
|
}
|
|
rows, err := s.db.Query(ctx, `
|
|
SELECT snapshot.id,snapshot.source_connection_id,root.node_id,snapshot.source_label,snapshot.node_count,
|
|
snapshot.folder_count,snapshot.file_count,snapshot.observed_at,snapshot.created_at,
|
|
(head.snapshot_id IS NOT NULL) AS is_head
|
|
FROM source_catalog_snapshots snapshot
|
|
JOIN source_catalog_nodes root ON root.snapshot_id=snapshot.id AND root.parent_node_id IS NULL
|
|
LEFT JOIN source_catalog_heads head ON head.source_connection_id=snapshot.source_connection_id AND head.snapshot_id=snapshot.id
|
|
WHERE snapshot.source_connection_id=$1
|
|
ORDER BY is_head DESC,snapshot.observed_at DESC,snapshot.id DESC
|
|
LIMIT $2 OFFSET $3`, connectionID, limit, offset)
|
|
if err != nil {
|
|
return governance.CatalogSnapshotPage{}, fmt.Errorf("list catalog snapshots: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var item governance.CatalogSnapshot
|
|
if err := rows.Scan(&item.SnapshotID, &item.ConnectionID, &item.RootNodeID, &item.SourceName, &item.NodeCount, &item.FolderCount, &item.FileCount, &item.ObservedAt, &item.CreatedAt, &item.IsHead); err != nil {
|
|
return governance.CatalogSnapshotPage{}, fmt.Errorf("scan catalog snapshot: %w", err)
|
|
}
|
|
page.Items = append(page.Items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.CatalogSnapshotPage{}, fmt.Errorf("iterate catalog snapshots: %w", err)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
// ListCatalogNodesInSnapshot is a read-only view of one persisted snapshot. It never
|
|
// mutates the current head. content_sha256 is returned only when actually recorded;
|
|
// unknown files return null so the caller can render "内容Hash未获取".
|
|
func (s *Store) ListCatalogNodesInSnapshot(ctx context.Context, filter governance.CatalogSnapshotNodeFilter) (governance.CatalogNodePage, error) {
|
|
if filter.ConnectionID == "" || len(filter.ConnectionID) > 36 || filter.SnapshotID == "" || len(filter.SnapshotID) > 36 ||
|
|
filter.ParentNodeID == "" || len(filter.ParentNodeID) > 36 || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 {
|
|
return governance.CatalogNodePage{}, errors.New("invalid catalog snapshot node filter")
|
|
}
|
|
var parent governance.CatalogNode
|
|
var parentID *string
|
|
err := s.db.QueryRow(ctx, `
|
|
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count,
|
|
node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at,node.content_sha256
|
|
FROM source_catalog_snapshots snapshot
|
|
JOIN source_catalog_nodes node ON node.snapshot_id=snapshot.id
|
|
WHERE snapshot.id=$1 AND snapshot.source_connection_id=$2 AND node.node_id=$3 AND node.node_type='folder'`,
|
|
filter.SnapshotID, filter.ConnectionID, filter.ParentNodeID).Scan(
|
|
&parent.NodeID, &parentID, &parent.NodeType, &parent.Name, &parent.DirectChildCount,
|
|
&parent.SubtreeFileCount, &parent.SubtreeBytes, &parent.SizeBytes, &parent.ModifiedAt, &parent.ContentSHA256)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return governance.CatalogNodePage{}, ErrCatalogSnapshotNotFound
|
|
}
|
|
if err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("get snapshot parent: %w", err)
|
|
}
|
|
page := governance.CatalogNodePage{Current: parent, Limit: filter.Limit, Offset: filter.Offset}
|
|
if parentID != nil {
|
|
page.Current.ParentNodeID = *parentID
|
|
}
|
|
page.Current.Visibility = "unknown"
|
|
if err := s.db.QueryRow(ctx, `
|
|
SELECT COUNT(*)::integer FROM source_catalog_nodes
|
|
WHERE snapshot_id=$1 AND parent_node_id=$2`, filter.SnapshotID, filter.ParentNodeID).Scan(&page.Total); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("count snapshot nodes: %w", err)
|
|
}
|
|
rows, err := s.db.Query(ctx, `
|
|
SELECT node_id,parent_node_id,node_type,name,direct_child_count,subtree_file_count,subtree_bytes,size_bytes,modified_at,content_sha256
|
|
FROM source_catalog_nodes
|
|
WHERE snapshot_id=$1 AND parent_node_id=$2
|
|
ORDER BY CASE node_type WHEN 'folder' THEN 0 ELSE 1 END,sort_order,lower(name),node_id
|
|
LIMIT $3 OFFSET $4`, filter.SnapshotID, filter.ParentNodeID, filter.Limit, filter.Offset)
|
|
if err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("list snapshot nodes: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var item governance.CatalogNode
|
|
var parentPtr *string
|
|
if err := rows.Scan(&item.NodeID, &parentPtr, &item.NodeType, &item.Name, &item.DirectChildCount, &item.SubtreeFileCount, &item.SubtreeBytes, &item.SizeBytes, &item.ModifiedAt, &item.ContentSHA256); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("scan snapshot node: %w", err)
|
|
}
|
|
if parentPtr != nil {
|
|
item.ParentNodeID = *parentPtr
|
|
}
|
|
item.Visibility = "unknown"
|
|
page.Items = append(page.Items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.CatalogNodePage{}, fmt.Errorf("iterate snapshot nodes: %w", err)
|
|
}
|
|
return page, nil
|
|
}
|