feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
@@ -20,19 +20,30 @@ func loadArtifactFetchManifest(ctx context.Context, db interface {
QueryRow(context.Context, string, ...any) pgx.Row
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
var manifest artifactfetch.Manifest
var fetchState string
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
intent.size_bytes,intent.modified_at,intent.locator_hash
intent.size_bytes,intent.modified_at,intent.locator_hash,intent.fetch_state
FROM catalog_import_intents intent
JOIN source_connections connection ON connection.id=intent.source_connection_id
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash, &fetchState)
if errors.Is(err, pgx.ErrNoRows) {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
}
if err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
}
if fetchState == "filtered" {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
}
var filtered bool
if err := db.QueryRow(ctx, `SELECT filename_is_filtered($1,$2,$3)`, connectionID, manifest.RelativePath, manifest.Name).Scan(&filtered); err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("evaluate artifact filename filter: %w", err)
}
if filtered {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
}
manifest.ConnectionID = connectionID
manifest.NodeID = nodeID
return manifest, nil
@@ -117,6 +117,7 @@ SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.m
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
AND NOT filename_is_filtered($1,node.relative_path,node.name)
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
@@ -0,0 +1,163 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error) {
if connectionID == "" || len(connectionID) > 36 {
return nil, governance.ErrInvalidFilenameFilter
}
rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at
FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID)
if err != nil {
return nil, fmt.Errorf("list filename filters: %w", err)
}
defer rows.Close()
result := make([]governance.FilenameFilterRule, 0)
for rows.Next() {
var rule governance.FilenameFilterRule
if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan filename filter: %w", err)
}
result = append(result, rule)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate filename filters: %w", err)
}
return result, nil
}
func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
if err := request.Validate(); err != nil {
return governance.FilenameFilterRule{}, false, err
}
fingerprint, err := operationFingerprint(struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
ExcludedExtensions []string `json:"excluded_extensions"`
OperatorRef string `json:"operator_ref"`
Clear bool `json:"clear"`
}{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear})
if err != nil {
return governance.FilenameFilterRule{}, false, err
}
payload, replayed, err := s.executeOperation(ctx, "filename.filter.mutate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
rule, err := mutateFilenameFilterRuleTx(ctx, tx, request)
if err != nil {
return nil, err
}
return json.Marshal(rule)
})
if err != nil {
return governance.FilenameFilterRule{}, false, err
}
var rule governance.FilenameFilterRule
if err := json.Unmarshal(payload, &rule); err != nil {
return governance.FilenameFilterRule{}, false, fmt.Errorf("decode filename filter result: %w", err)
}
return rule, replayed, nil
}
func mutateFilenameFilterRuleTx(ctx context.Context, tx pgx.Tx, request governance.FilenameFilterMutation) (governance.FilenameFilterRule, error) {
var exists bool
if request.DirectoryPath == "" {
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, request.ConnectionID).Scan(&exists); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter source: %w", err)
}
} else {
if err := tx.QueryRow(ctx, `SELECT EXISTS(
SELECT 1 FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative_path=$2)`, request.ConnectionID, request.DirectoryPath).Scan(&exists); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter directory: %w", err)
}
}
if !exists {
return governance.FilenameFilterRule{}, governance.ErrFilenameFilterTargetNotFound
}
if request.Clear {
if _, err := tx.Exec(ctx, `DELETE FROM filename_filter_rules WHERE source_connection_id=$1 AND directory_path=$2`, request.ConnectionID, request.DirectoryPath); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err)
}
} else {
if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err)
}
}
if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil {
return governance.FilenameFilterRule{}, err
}
return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil
}
func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error {
// Restore entries first, while fetch_state still identifies a previous filter.
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state=intent.visibility,updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' AND intent.source_item_id=item.id
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered items: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=NULL,updated_at=$2
FROM source_item_versions version, catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
AND publication.processing_state='planned'`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered publications: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='pending',attempt=0,available_at=$2,lease_owner=NULL,lease_until=NULL,last_error='',last_error_code='',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('cancelled','failed','retry')`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered fetch jobs: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET
fetch_state=CASE WHEN EXISTS(
SELECT 1 FROM source_items item JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
WHERE item.id=intent.source_item_id AND version.content_state='verified'
) THEN 'ready' ELSE 'pending' END,last_error='',updated_at=$2
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered intents: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=COALESCE(superseded_at,$2),updated_at=$2
FROM source_item_versions version, catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
AND publication.processing_state='planned' AND publication.superseded_at IS NULL`, connectionID, now); err != nil {
return fmt.Errorf("retire filename-filtered publications: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state='blocked',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND intent.source_item_id=item.id`, connectionID, now); err != nil {
return fmt.Errorf("block filename-filtered items: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='cancelled',lease_owner=NULL,lease_until=NULL,last_error='filtered by filename rule',last_error_code='',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('pending','retry','failed')`, connectionID, now); err != nil {
return fmt.Errorf("cancel filename-filtered fetch jobs: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET fetch_state='filtered',last_error='',updated_at=$2
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND intent.fetch_state<>'ready'`, connectionID, now); err != nil {
return fmt.Errorf("mark filename-filtered intents: %w", err)
}
return nil
}
@@ -0,0 +1,128 @@
package postgres
import (
"context"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
func TestFilenameFilterRuleHidesAndStopsMatchingArtifactsThenRestoresThem(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-filter','dingtalk','过滤测试',true);
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin,policy_state)
VALUES('item-zip','conn-filter','zip',repeat('1',64),'固件.ZIP','file','r1','internal','configured_scope','internal'),
('item-pdf','conn-filter','pdf',repeat('2',64),'手册.pdf','file','r1','internal','configured_scope','internal');
INSERT INTO source_item_versions(id,source_item_id,revision,content_state)
VALUES('version-zip','item-zip','r1','unknown'),('version-pdf','item-pdf','r1','unknown');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-filter','conn-filter',repeat('a',64),'过滤测试',4,2,2,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-filter','root-filter',NULL,'folder','过滤测试','',repeat('a',64),NULL,NULL,-1,1,2,30),
('snapshot-filter','folder-filter','root-filter','folder','固件','固件',repeat('b',64),NULL,NULL,0,2,2,30),
('snapshot-filter','node-zip','folder-filter','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),'item-zip',10,0,0,1,10),
('snapshot-filter','node-pdf','folder-filter','file','手册.pdf','固件/手册.pdf',repeat('d',64),'item-pdf',20,1,0,1,20);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-filter','snapshot-filter',NOW());
INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,source_item_id,decided_by,relative_path,name,locator_hash,fetch_state)
VALUES('conn-filter','node-zip','internal','item-zip','admin','固件/固件.ZIP','固件.ZIP',repeat('c',64),'pending'),
('conn-filter','node-pdf','internal','item-pdf','admin','固件/手册.pdf','手册.pdf',repeat('d',64),'pending');
INSERT INTO jobs(id,type,payload,state,idempotency_key)
VALUES('job-filter-zip','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-zip","version_id":"version-zip"}','retry','fetch-zip'),
('job-filter-pdf','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-pdf","version_id":"version-pdf"}','pending','fetch-pdf');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope)
VALUES('pub-filter-zip','version-zip','kb-test','planned','internal');`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-filter", DirectoryPath: "", ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin", Now: now,
}, "op-filter-set")
if err != nil || replayed || len(rule.ExcludedExtensions) != 3 {
t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err)
}
var zipState, pdfState, zipJob, pdfJob, zipPolicy string
var superseded *time.Time
if err := pool.QueryRow(ctx, `SELECT
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-pdf'),
(SELECT state FROM jobs WHERE id='job-filter-zip'),
(SELECT state FROM jobs WHERE id='job-filter-pdf'),
(SELECT policy_state FROM source_items WHERE id='item-zip'),
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &pdfState, &zipJob, &pdfJob, &zipPolicy, &superseded); err != nil {
t.Fatal(err)
}
if zipState != "filtered" || pdfState != "pending" || zipJob != "cancelled" || pdfJob != "pending" || zipPolicy != "blocked" || superseded == nil {
t.Fatalf("filtered states zip=%s pdf=%s jobs=%s/%s policy=%s superseded=%v", zipState, pdfState, zipJob, pdfJob, zipPolicy, superseded)
}
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-filter", ParentNodeID: "folder-filter", Limit: 50})
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].Name != "手册.pdf" || page.Current.SubtreeFileCount != 1 || page.Current.SubtreeBytes != 20 {
t.Fatalf("filtered page=%#v err=%v", page, err)
}
routed, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
ConnectionID: "conn-filter", NodeIDs: []string{"folder-filter"}, Visibility: governance.VisibilityInternal,
TargetKnowledgeBaseID: "kb-route", OperatorRef: "admin", Reason: "过滤后路由", Now: now,
})
if err != nil || routed.SelectedFiles != 1 {
t.Fatalf("filtered route=%#v err=%v", routed, err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-filter-old','conn-filter',repeat('e',64),'过滤测试',2,1,1,NOW()-INTERVAL '1 hour');
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-filter-old','old-root',NULL,'folder','过滤测试','',repeat('e',64),-1,1,1,5),
('snapshot-filter-old','old-zip','old-root','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),0,0,1,5);
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
VALUES('change-filter','conn-filter','snapshot-filter-old','snapshot-filter',NOW());
SELECT compare_catalog_change_set('change-filter');`)
if err != nil {
t.Fatal(err)
}
var changes, adds, updates int
var onlyTitle string
if err := pool.QueryRow(ctx, `SELECT change_set.add_count,change_set.update_count,COUNT(item.*),COALESCE(MAX(item.title),'')
FROM source_catalog_change_sets change_set LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
WHERE change_set.id='change-filter' GROUP BY change_set.add_count,change_set.update_count`).Scan(&adds, &updates, &changes, &onlyTitle); err != nil {
t.Fatal(err)
}
if changes != 1 || adds != 1 || updates != 0 || onlyTitle != "手册.pdf" {
t.Fatalf("filtered diff changes=%d adds=%d updates=%d title=%q", changes, adds, updates, onlyTitle)
}
_, _, err = store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-filter", DirectoryPath: "", OperatorRef: "admin", Clear: true, Now: now.Add(time.Minute),
}, "op-filter-clear")
if err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
(SELECT state FROM jobs WHERE id='job-filter-zip'),
(SELECT policy_state FROM source_items WHERE id='item-zip'),
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &zipJob, &zipPolicy, &superseded); err != nil {
t.Fatal(err)
}
if zipState != "pending" || zipJob != "pending" || zipPolicy != "internal" || superseded != nil {
t.Fatalf("restored states fetch=%s job=%s policy=%s superseded=%v", zipState, zipJob, zipPolicy, superseded)
}
}
@@ -0,0 +1,13 @@
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
ALTER FUNCTION compare_catalog_change_set_unfiltered(VARCHAR) RENAME TO compare_catalog_change_set;
UPDATE catalog_import_intents SET fetch_state='missing_local' WHERE fetch_state='filtered';
ALTER TABLE catalog_import_intents
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
ALTER TABLE catalog_import_intents
ADD CONSTRAINT catalog_import_intents_fetch_state_check
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed'));
DROP FUNCTION IF EXISTS filename_is_filtered(VARCHAR, TEXT, TEXT);
DROP FUNCTION IF EXISTS filename_extension(TEXT);
DROP TABLE IF EXISTS filename_filter_rules;
@@ -0,0 +1,69 @@
-- Source/directory filename filters. Rules are additive: the root path is the
-- source default and every matching ancestor directory contributes exclusions.
CREATE TABLE IF NOT EXISTS filename_filter_rules (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE CASCADE,
directory_path TEXT NOT NULL DEFAULT '',
excluded_extensions TEXT[] NOT NULL,
updated_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, directory_path),
CHECK (cardinality(excluded_extensions) BETWEEN 1 AND 100)
);
ALTER TABLE catalog_import_intents
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
ALTER TABLE catalog_import_intents
ADD CONSTRAINT catalog_import_intents_fetch_state_check
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed', 'filtered'));
CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$
SELECT CASE
WHEN COALESCE(p_name, '') ~ '\.[^.]+$'
THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$'))
ELSE ''
END
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
CREATE OR REPLACE FUNCTION filename_is_filtered(
p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT
) RETURNS BOOLEAN AS $$
SELECT EXISTS (
SELECT 1
FROM filename_filter_rules rule
CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension
WHERE rule.source_connection_id = p_connection_id
AND (rule.directory_path = ''
OR p_relative_path LIKE rule.directory_path || '/%')
AND extension = filename_extension(p_name)
)
$$ LANGUAGE SQL STABLE PARALLEL SAFE;
-- Keep the existing complete classifier as the implementation, then remove
-- filtered names before recounting. This preserves all move/scope semantics.
ALTER FUNCTION compare_catalog_change_set(VARCHAR) RENAME TO compare_catalog_change_set_unfiltered;
CREATE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
PERFORM compare_catalog_change_set_unfiltered(p_change_set_id);
DELETE FROM source_catalog_change_items item
USING source_catalog_change_sets change_set
WHERE item.change_set_id = p_change_set_id
AND change_set.id = item.change_set_id
AND filename_is_filtered(
change_set.source_connection_id,
COALESCE(NULLIF(item.new_path, ''), item.old_path),
item.title
);
UPDATE source_catalog_change_sets change_set SET
add_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='add'),
update_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='update'),
move_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move'),
move_scope_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move_scope'),
delete_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='delete'),
no_change_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='no_change')
WHERE change_set.id = p_change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 34 {
t.Fatalf("migration count = %d, want 34", len(migrations))
if len(migrations) != 35 {
t.Fatalf("migration count = %d, want 35", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -101,6 +101,9 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if migrations[31].Version != 32 || !strings.Contains(migrations[31].UpSQL, "CREATE TABLE IF NOT EXISTS users") || !strings.Contains(migrations[31].UpSQL, "user_sessions") {
t.Fatalf("migration 32 = %#v", migrations[31])
}
if migrations[34].Version != 35 || !strings.Contains(migrations[34].UpSQL, "CREATE TABLE IF NOT EXISTS filename_filter_rules") || !strings.Contains(migrations[34].UpSQL, "'filtered'") {
t.Fatalf("migration 35 = %#v", migrations[34])
}
}
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
@@ -620,6 +623,7 @@ func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) {
"case when attempt >= max_attempts then 'failed'",
"else 'retry'",
"lease_owner = null",
"select id from updated",
},
},
}
+4 -1
View File
@@ -92,7 +92,7 @@ WITH updated AS (
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt,
state
)
), projected_failure AS (
INSERT INTO job_failures (
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
business_change_set_id, business_version_id, business_action, business_target_kb_id,
@@ -117,6 +117,9 @@ SET publication_id = EXCLUDED.publication_id,
resolution_actor_ref = '',
resolution_reason = '',
created_at = EXCLUDED.created_at
RETURNING job_id
)
SELECT id FROM updated
`
const listJobsSQL = `
+31 -6
View File
@@ -15,7 +15,11 @@ var ErrCatalogNodeNotFound = errors.New("catalog node not found")
func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) {
rows, err := s.db.Query(ctx, `
SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id,
snapshot.folder_count - 1, snapshot.file_count, head.activated_at,
snapshot.folder_count - 1,
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
WHERE file.snapshot_id=snapshot.id AND file.node_type='file'
AND NOT filename_is_filtered(connection.id,file.relative_path,file.name)),
head.activated_at,
'ready', COALESCE(refresh.duration_ms,0)
FROM source_catalog_heads head
JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id
@@ -62,8 +66,19 @@ func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogN
page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset}
var parentID *string
err := s.db.QueryRow(ctx, `
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count,
node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,
(SELECT COUNT(*)::integer FROM source_catalog_nodes child
WHERE child.snapshot_id=node.snapshot_id AND child.parent_node_id=node.node_id
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))),
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
(SELECT COALESCE(SUM(COALESCE(file.size_bytes,0)),0)::bigint FROM source_catalog_nodes file
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
node.size_bytes,node.modified_at
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan(
@@ -112,7 +127,8 @@ SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter
if err := s.db.QueryRow(ctx, `
SELECT COUNT(*)::integer FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err)
}
@@ -122,6 +138,7 @@ WITH RECURSIVE current_head AS (
), selected AS MATERIALIZED (
SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE node.parent_node_id=$2
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))
ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id
LIMIT $3 OFFSET $4
), descendants(ancestor_id,snapshot_id,node_id) AS (
@@ -135,9 +152,10 @@ WITH RECURSIVE current_head AS (
COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility,
publication.id AS publication_id,publication.ingest_state,publication.processing_state,
current_version.content_state AS version_content_state,content.content_state AS object_content_state,
content.local_spool_ref
content.local_spool_ref,leaf.size_bytes
FROM descendants
JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file'
AND NOT filename_is_filtered($1,leaf.relative_path,leaf.name)
LEFT JOIN source_items item ON item.id=leaf.source_item_id
LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision
LEFT JOIN content_objects content ON content.id=current_version.content_object_id
@@ -158,6 +176,8 @@ WITH RECURSIVE current_head AS (
) publication ON TRUE
), stats AS (
SELECT ancestor_id,
COUNT(*)::integer AS eligible_file_count,
COALESCE(SUM(COALESCE(size_bytes,0)),0)::bigint AS eligible_bytes,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count,
@@ -169,7 +189,12 @@ WITH RECURSIVE current_head AS (
FROM file_state GROUP BY ancestor_id
)
SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name,
selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes,
CASE WHEN selected.node_type='file' THEN 0 ELSE (
SELECT COUNT(*)::integer FROM source_catalog_nodes child
WHERE child.snapshot_id=selected.snapshot_id AND child.parent_node_id=selected.node_id
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))
) END,
COALESCE(stats.eligible_file_count,0),COALESCE(stats.eligible_bytes,0),
selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'),
COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0),
COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0)