Files
knowledge-governance/internal/platform/postgres/catalog_routing.go
T
姚章浩 b3c9331966 feat(ingest): 源与目录文件名过滤及钉钉文档导出
- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
2026-09-30 10:54:07 +08:00

197 lines
7.5 KiB
Go

package postgres
import (
"context"
"errors"
"fmt"
"strings"
"unicode"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) RouteCatalogFiles(ctx context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
if err := validateCatalogRouteRequest(request); err != nil {
return governance.CatalogRouteResult{}, err
}
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return governance.CatalogRouteResult{}, errors.New("catalog route transactions are unavailable")
}
tx, err := beginner.Begin(ctx)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("begin catalog route: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
result, err := routeCatalogFilesTx(ctx, tx, request)
if err != nil {
return governance.CatalogRouteResult{}, err
}
if err := tx.Commit(ctx); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("commit catalog route: %w", err)
}
return result, nil
}
func validateCatalogRouteRequest(request governance.CatalogRouteRequest) error {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
if !validRouteID(request.ConnectionID) || len(request.NodeIDs) == 0 || len(request.NodeIDs) > 100 ||
!validCatalogPolicy(request.Visibility, targetKnowledgeBaseIDs) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 ||
strings.TrimSpace(request.Reason) == "" || len(request.Reason) > 500 || request.Now.IsZero() {
return errors.New("invalid catalog route request")
}
seen := map[string]bool{}
for _, id := range request.NodeIDs {
if !validRouteID(id) || seen[id] {
return errors.New("invalid catalog route nodes")
}
seen[id] = true
}
return nil
}
func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
nodeIDs := append([]string(nil), request.NodeIDs...)
selectedRows, err := tx.Query(ctx, `SELECT node.node_id,node.node_type,node.relative_path
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
ORDER BY node.node_id
FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route nodes: %w", err)
}
selectedCount := 0
directoryPaths := make([]string, 0)
for selectedRows.Next() {
var nodeID, nodeType, directoryPath string
if err := selectedRows.Scan(&nodeID, &nodeType, &directoryPath); err != nil {
selectedRows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route node: %w", err)
}
selectedCount++
if nodeType == "folder" {
directoryPaths = append(directoryPaths, directoryPath)
}
}
if err := selectedRows.Err(); err != nil {
selectedRows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route nodes: %w", err)
}
selectedRows.Close()
if selectedCount != len(nodeIDs) {
return governance.CatalogRouteResult{}, errors.New("catalog route contains missing nodes")
}
for _, directoryPath := range directoryPaths {
if err := setCatalogDirectoryPolicy(ctx, tx, request, directoryPath); err != nil {
return governance.CatalogRouteResult{}, err
}
}
rows, err := tx.Query(ctx, `WITH RECURSIVE selected AS (
SELECT node.snapshot_id,node.node_id,node.node_type
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
), descendants AS (
SELECT snapshot_id,node_id,node_type FROM selected
UNION
SELECT child.snapshot_id,child.node_id,child.node_type
FROM descendants parent
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
WHERE parent.node_type='folder'
)
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
AND NOT filename_is_filtered($1,node.relative_path,node.name)
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
}
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
for rows.Next() {
var file governance.CatalogRouteFile
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
}
files = append(files, file)
}
if err := rows.Err(); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
}
rows.Close()
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
}
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
if _, err := tx.Exec(ctx, `INSERT INTO directory_scope_rules(source_connection_id,directory_path,visibility,updated_by,reason,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$6,$6)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return fmt.Errorf("set routed directory scope: %w", err)
}
eventID, err := newOpaqueID("se-")
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_connection_id,directory_path,action,decided_visibility,operator_ref,reason,created_at)
VALUES($1,'directory',$2,$3,'set',$4,$5,$6,$7)`, eventID, request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return fmt.Errorf("audit routed directory scope: %w", err)
}
return nil
}
func validCatalogPolicy(policy governance.Visibility, targetKnowledgeBaseIDs []string) bool {
expectedTargets := 1
switch policy {
case governance.VisibilityInternal, governance.VisibilityExternal:
case governance.VisibilityBoth:
expectedTargets = 2
case governance.VisibilityBlocked:
expectedTargets = 0
default:
return false
}
if len(targetKnowledgeBaseIDs) != expectedTargets {
return false
}
seen := make(map[string]bool, len(targetKnowledgeBaseIDs))
for _, target := range targetKnowledgeBaseIDs {
if !validRouteID(target) || seen[target] {
return false
}
seen[target] = true
}
return true
}
func validRouteID(value string) bool {
if value == "" || len(value) > 128 {
return false
}
for _, character := range value {
if unicode.IsLetter(character) || unicode.IsDigit(character) || character == '-' || character == '_' {
continue
}
return false
}
return true
}