feat(ingest): 源与目录文件名过滤及钉钉文档导出
- filename_filter_rules table (000035) with source-root and per-directory rows - additive inheritance: source default + every ancestor directory contributes - filename_is_filtered() SQL predicate applied to catalog browsing, subtree counts, change-set diffing, routing fan-out and artifact fetch manifests - fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch jobs, retires planned publications and blocks items when rules change - GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized extensions) + Settings and Catalog filter editors - failJobSQL no longer swallows the claimed row when the failure projection upserts (fixes spurious 'lease lost' retries) - DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403 from file/download fixed), header-signature downloads supported
This commit is contained in:
@@ -20,19 +20,30 @@ func loadArtifactFetchManifest(ctx context.Context, db interface {
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
|
||||
var manifest artifactfetch.Manifest
|
||||
var fetchState string
|
||||
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
|
||||
intent.size_bytes,intent.modified_at,intent.locator_hash
|
||||
intent.size_bytes,intent.modified_at,intent.locator_hash,intent.fetch_state
|
||||
FROM catalog_import_intents intent
|
||||
JOIN source_connections connection ON connection.id=intent.source_connection_id
|
||||
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
|
||||
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
|
||||
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
|
||||
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash, &fetchState)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
|
||||
}
|
||||
if err != nil {
|
||||
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
|
||||
}
|
||||
if fetchState == "filtered" {
|
||||
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
|
||||
}
|
||||
var filtered bool
|
||||
if err := db.QueryRow(ctx, `SELECT filename_is_filtered($1,$2,$3)`, connectionID, manifest.RelativePath, manifest.Name).Scan(&filtered); err != nil {
|
||||
return artifactfetch.Manifest{}, fmt.Errorf("evaluate artifact filename filter: %w", err)
|
||||
}
|
||||
if filtered {
|
||||
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
|
||||
}
|
||||
manifest.ConnectionID = connectionID
|
||||
manifest.NodeID = nodeID
|
||||
return manifest, nil
|
||||
|
||||
@@ -117,6 +117,7 @@ SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.m
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
|
||||
WHERE node.node_type='file'
|
||||
AND NOT filename_is_filtered($1,node.relative_path,node.name)
|
||||
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error) {
|
||||
if connectionID == "" || len(connectionID) > 36 {
|
||||
return nil, governance.ErrInvalidFilenameFilter
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at
|
||||
FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list filename filters: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]governance.FilenameFilterRule, 0)
|
||||
for rows.Next() {
|
||||
var rule governance.FilenameFilterRule
|
||||
if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan filename filter: %w", err)
|
||||
}
|
||||
result = append(result, rule)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate filename filters: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
|
||||
if err := request.Validate(); err != nil {
|
||||
return governance.FilenameFilterRule{}, false, err
|
||||
}
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
DirectoryPath string `json:"directory_path"`
|
||||
ExcludedExtensions []string `json:"excluded_extensions"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Clear bool `json:"clear"`
|
||||
}{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear})
|
||||
if err != nil {
|
||||
return governance.FilenameFilterRule{}, false, err
|
||||
}
|
||||
payload, replayed, err := s.executeOperation(ctx, "filename.filter.mutate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
rule, err := mutateFilenameFilterRuleTx(ctx, tx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(rule)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.FilenameFilterRule{}, false, err
|
||||
}
|
||||
var rule governance.FilenameFilterRule
|
||||
if err := json.Unmarshal(payload, &rule); err != nil {
|
||||
return governance.FilenameFilterRule{}, false, fmt.Errorf("decode filename filter result: %w", err)
|
||||
}
|
||||
return rule, replayed, nil
|
||||
}
|
||||
|
||||
func mutateFilenameFilterRuleTx(ctx context.Context, tx pgx.Tx, request governance.FilenameFilterMutation) (governance.FilenameFilterRule, error) {
|
||||
var exists bool
|
||||
if request.DirectoryPath == "" {
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, request.ConnectionID).Scan(&exists); err != nil {
|
||||
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter source: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS(
|
||||
SELECT 1 FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative_path=$2)`, request.ConnectionID, request.DirectoryPath).Scan(&exists); err != nil {
|
||||
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter directory: %w", err)
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
return governance.FilenameFilterRule{}, governance.ErrFilenameFilterTargetNotFound
|
||||
}
|
||||
if request.Clear {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM filename_filter_rules WHERE source_connection_id=$1 AND directory_path=$2`, request.ConnectionID, request.DirectoryPath); err != nil {
|
||||
return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err)
|
||||
}
|
||||
} else {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$5)
|
||||
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`,
|
||||
request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil {
|
||||
return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err)
|
||||
}
|
||||
}
|
||||
if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil {
|
||||
return governance.FilenameFilterRule{}, err
|
||||
}
|
||||
return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil
|
||||
}
|
||||
|
||||
func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error {
|
||||
// Restore entries first, while fetch_state still identifies a previous filter.
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state=intent.visibility,updated_at=$2
|
||||
FROM catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' AND intent.source_item_id=item.id
|
||||
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("restore filename-filtered items: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=NULL,updated_at=$2
|
||||
FROM source_item_versions version, catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
|
||||
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
|
||||
AND publication.processing_state='planned'`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("restore filename-filtered publications: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='pending',attempt=0,available_at=$2,lease_owner=NULL,lease_until=NULL,last_error='',last_error_code='',updated_at=$2
|
||||
FROM catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
|
||||
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
|
||||
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('cancelled','failed','retry')`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("restore filename-filtered fetch jobs: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET
|
||||
fetch_state=CASE WHEN EXISTS(
|
||||
SELECT 1 FROM source_items item JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
WHERE item.id=intent.source_item_id AND version.content_state='verified'
|
||||
) THEN 'ready' ELSE 'pending' END,last_error='',updated_at=$2
|
||||
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
|
||||
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("restore filename-filtered intents: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=COALESCE(superseded_at,$2),updated_at=$2
|
||||
FROM source_item_versions version, catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
|
||||
AND publication.processing_state='planned' AND publication.superseded_at IS NULL`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("retire filename-filtered publications: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state='blocked',updated_at=$2
|
||||
FROM catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND intent.source_item_id=item.id`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("block filename-filtered items: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='cancelled',lease_owner=NULL,lease_until=NULL,last_error='filtered by filename rule',last_error_code='',updated_at=$2
|
||||
FROM catalog_import_intents intent
|
||||
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
|
||||
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('pending','retry','failed')`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("cancel filename-filtered fetch jobs: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET fetch_state='filtered',last_error='',updated_at=$2
|
||||
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
|
||||
AND intent.fetch_state<>'ready'`, connectionID, now); err != nil {
|
||||
return fmt.Errorf("mark filename-filtered intents: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestFilenameFilterRuleHidesAndStopsMatchingArtifactsThenRestoresThem(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-filter','dingtalk','过滤测试',true);
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin,policy_state)
|
||||
VALUES('item-zip','conn-filter','zip',repeat('1',64),'固件.ZIP','file','r1','internal','configured_scope','internal'),
|
||||
('item-pdf','conn-filter','pdf',repeat('2',64),'手册.pdf','file','r1','internal','configured_scope','internal');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_state)
|
||||
VALUES('version-zip','item-zip','r1','unknown'),('version-pdf','item-pdf','r1','unknown');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-filter','conn-filter',repeat('a',64),'过滤测试',4,2,2,NOW());
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES('snapshot-filter','root-filter',NULL,'folder','过滤测试','',repeat('a',64),NULL,NULL,-1,1,2,30),
|
||||
('snapshot-filter','folder-filter','root-filter','folder','固件','固件',repeat('b',64),NULL,NULL,0,2,2,30),
|
||||
('snapshot-filter','node-zip','folder-filter','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),'item-zip',10,0,0,1,10),
|
||||
('snapshot-filter','node-pdf','folder-filter','file','手册.pdf','固件/手册.pdf',repeat('d',64),'item-pdf',20,1,0,1,20);
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-filter','snapshot-filter',NOW());
|
||||
INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,source_item_id,decided_by,relative_path,name,locator_hash,fetch_state)
|
||||
VALUES('conn-filter','node-zip','internal','item-zip','admin','固件/固件.ZIP','固件.ZIP',repeat('c',64),'pending'),
|
||||
('conn-filter','node-pdf','internal','item-pdf','admin','固件/手册.pdf','手册.pdf',repeat('d',64),'pending');
|
||||
INSERT INTO jobs(id,type,payload,state,idempotency_key)
|
||||
VALUES('job-filter-zip','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-zip","version_id":"version-zip"}','retry','fetch-zip'),
|
||||
('job-filter-pdf','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-pdf","version_id":"version-pdf"}','pending','fetch-pdf');
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope)
|
||||
VALUES('pub-filter-zip','version-zip','kb-test','planned','internal');`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
|
||||
rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
|
||||
ConnectionID: "conn-filter", DirectoryPath: "", ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin", Now: now,
|
||||
}, "op-filter-set")
|
||||
if err != nil || replayed || len(rule.ExcludedExtensions) != 3 {
|
||||
t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err)
|
||||
}
|
||||
|
||||
var zipState, pdfState, zipJob, pdfJob, zipPolicy string
|
||||
var superseded *time.Time
|
||||
if err := pool.QueryRow(ctx, `SELECT
|
||||
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
|
||||
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-pdf'),
|
||||
(SELECT state FROM jobs WHERE id='job-filter-zip'),
|
||||
(SELECT state FROM jobs WHERE id='job-filter-pdf'),
|
||||
(SELECT policy_state FROM source_items WHERE id='item-zip'),
|
||||
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &pdfState, &zipJob, &pdfJob, &zipPolicy, &superseded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if zipState != "filtered" || pdfState != "pending" || zipJob != "cancelled" || pdfJob != "pending" || zipPolicy != "blocked" || superseded == nil {
|
||||
t.Fatalf("filtered states zip=%s pdf=%s jobs=%s/%s policy=%s superseded=%v", zipState, pdfState, zipJob, pdfJob, zipPolicy, superseded)
|
||||
}
|
||||
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-filter", ParentNodeID: "folder-filter", Limit: 50})
|
||||
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].Name != "手册.pdf" || page.Current.SubtreeFileCount != 1 || page.Current.SubtreeBytes != 20 {
|
||||
t.Fatalf("filtered page=%#v err=%v", page, err)
|
||||
}
|
||||
routed, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: "conn-filter", NodeIDs: []string{"folder-filter"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: "kb-route", OperatorRef: "admin", Reason: "过滤后路由", Now: now,
|
||||
})
|
||||
if err != nil || routed.SelectedFiles != 1 {
|
||||
t.Fatalf("filtered route=%#v err=%v", routed, err)
|
||||
}
|
||||
_, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-filter-old','conn-filter',repeat('e',64),'过滤测试',2,1,1,NOW()-INTERVAL '1 hour');
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES('snapshot-filter-old','old-root',NULL,'folder','过滤测试','',repeat('e',64),-1,1,1,5),
|
||||
('snapshot-filter-old','old-zip','old-root','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),0,0,1,5);
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
|
||||
VALUES('change-filter','conn-filter','snapshot-filter-old','snapshot-filter',NOW());
|
||||
SELECT compare_catalog_change_set('change-filter');`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var changes, adds, updates int
|
||||
var onlyTitle string
|
||||
if err := pool.QueryRow(ctx, `SELECT change_set.add_count,change_set.update_count,COUNT(item.*),COALESCE(MAX(item.title),'')
|
||||
FROM source_catalog_change_sets change_set LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
|
||||
WHERE change_set.id='change-filter' GROUP BY change_set.add_count,change_set.update_count`).Scan(&adds, &updates, &changes, &onlyTitle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changes != 1 || adds != 1 || updates != 0 || onlyTitle != "手册.pdf" {
|
||||
t.Fatalf("filtered diff changes=%d adds=%d updates=%d title=%q", changes, adds, updates, onlyTitle)
|
||||
}
|
||||
|
||||
_, _, err = store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
|
||||
ConnectionID: "conn-filter", DirectoryPath: "", OperatorRef: "admin", Clear: true, Now: now.Add(time.Minute),
|
||||
}, "op-filter-clear")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT
|
||||
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
|
||||
(SELECT state FROM jobs WHERE id='job-filter-zip'),
|
||||
(SELECT policy_state FROM source_items WHERE id='item-zip'),
|
||||
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &zipJob, &zipPolicy, &superseded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if zipState != "pending" || zipJob != "pending" || zipPolicy != "internal" || superseded != nil {
|
||||
t.Fatalf("restored states fetch=%s job=%s policy=%s superseded=%v", zipState, zipJob, zipPolicy, superseded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
|
||||
ALTER FUNCTION compare_catalog_change_set_unfiltered(VARCHAR) RENAME TO compare_catalog_change_set;
|
||||
|
||||
UPDATE catalog_import_intents SET fetch_state='missing_local' WHERE fetch_state='filtered';
|
||||
ALTER TABLE catalog_import_intents
|
||||
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
|
||||
ALTER TABLE catalog_import_intents
|
||||
ADD CONSTRAINT catalog_import_intents_fetch_state_check
|
||||
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed'));
|
||||
|
||||
DROP FUNCTION IF EXISTS filename_is_filtered(VARCHAR, TEXT, TEXT);
|
||||
DROP FUNCTION IF EXISTS filename_extension(TEXT);
|
||||
DROP TABLE IF EXISTS filename_filter_rules;
|
||||
@@ -0,0 +1,69 @@
|
||||
-- Source/directory filename filters. Rules are additive: the root path is the
|
||||
-- source default and every matching ancestor directory contributes exclusions.
|
||||
CREATE TABLE IF NOT EXISTS filename_filter_rules (
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE CASCADE,
|
||||
directory_path TEXT NOT NULL DEFAULT '',
|
||||
excluded_extensions TEXT[] NOT NULL,
|
||||
updated_by VARCHAR(128) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (source_connection_id, directory_path),
|
||||
CHECK (cardinality(excluded_extensions) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
ALTER TABLE catalog_import_intents
|
||||
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
|
||||
ALTER TABLE catalog_import_intents
|
||||
ADD CONSTRAINT catalog_import_intents_fetch_state_check
|
||||
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed', 'filtered'));
|
||||
|
||||
CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$
|
||||
SELECT CASE
|
||||
WHEN COALESCE(p_name, '') ~ '\.[^.]+$'
|
||||
THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$'))
|
||||
ELSE ''
|
||||
END
|
||||
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION filename_is_filtered(
|
||||
p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT
|
||||
) RETURNS BOOLEAN AS $$
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM filename_filter_rules rule
|
||||
CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension
|
||||
WHERE rule.source_connection_id = p_connection_id
|
||||
AND (rule.directory_path = ''
|
||||
OR p_relative_path LIKE rule.directory_path || '/%')
|
||||
AND extension = filename_extension(p_name)
|
||||
)
|
||||
$$ LANGUAGE SQL STABLE PARALLEL SAFE;
|
||||
|
||||
-- Keep the existing complete classifier as the implementation, then remove
|
||||
-- filtered names before recounting. This preserves all move/scope semantics.
|
||||
ALTER FUNCTION compare_catalog_change_set(VARCHAR) RENAME TO compare_catalog_change_set_unfiltered;
|
||||
CREATE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
|
||||
BEGIN
|
||||
PERFORM compare_catalog_change_set_unfiltered(p_change_set_id);
|
||||
|
||||
DELETE FROM source_catalog_change_items item
|
||||
USING source_catalog_change_sets change_set
|
||||
WHERE item.change_set_id = p_change_set_id
|
||||
AND change_set.id = item.change_set_id
|
||||
AND filename_is_filtered(
|
||||
change_set.source_connection_id,
|
||||
COALESCE(NULLIF(item.new_path, ''), item.old_path),
|
||||
item.title
|
||||
);
|
||||
|
||||
UPDATE source_catalog_change_sets change_set SET
|
||||
add_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='add'),
|
||||
update_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='update'),
|
||||
move_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move'),
|
||||
move_scope_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move_scope'),
|
||||
delete_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='delete'),
|
||||
no_change_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='no_change')
|
||||
WHERE change_set.id = p_change_set_id;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 34 {
|
||||
t.Fatalf("migration count = %d, want 34", len(migrations))
|
||||
if len(migrations) != 35 {
|
||||
t.Fatalf("migration count = %d, want 35", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
@@ -101,6 +101,9 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if migrations[31].Version != 32 || !strings.Contains(migrations[31].UpSQL, "CREATE TABLE IF NOT EXISTS users") || !strings.Contains(migrations[31].UpSQL, "user_sessions") {
|
||||
t.Fatalf("migration 32 = %#v", migrations[31])
|
||||
}
|
||||
if migrations[34].Version != 35 || !strings.Contains(migrations[34].UpSQL, "CREATE TABLE IF NOT EXISTS filename_filter_rules") || !strings.Contains(migrations[34].UpSQL, "'filtered'") {
|
||||
t.Fatalf("migration 35 = %#v", migrations[34])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
|
||||
@@ -620,6 +623,7 @@ func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) {
|
||||
"case when attempt >= max_attempts then 'failed'",
|
||||
"else 'retry'",
|
||||
"lease_owner = null",
|
||||
"select id from updated",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ WITH updated AS (
|
||||
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
|
||||
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt,
|
||||
state
|
||||
)
|
||||
), projected_failure AS (
|
||||
INSERT INTO job_failures (
|
||||
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
|
||||
business_change_set_id, business_version_id, business_action, business_target_kb_id,
|
||||
@@ -117,6 +117,9 @@ SET publication_id = EXCLUDED.publication_id,
|
||||
resolution_actor_ref = '',
|
||||
resolution_reason = '',
|
||||
created_at = EXCLUDED.created_at
|
||||
RETURNING job_id
|
||||
)
|
||||
SELECT id FROM updated
|
||||
`
|
||||
|
||||
const listJobsSQL = `
|
||||
|
||||
@@ -15,7 +15,11 @@ var ErrCatalogNodeNotFound = errors.New("catalog node not found")
|
||||
func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id,
|
||||
snapshot.folder_count - 1, snapshot.file_count, head.activated_at,
|
||||
snapshot.folder_count - 1,
|
||||
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
|
||||
WHERE file.snapshot_id=snapshot.id AND file.node_type='file'
|
||||
AND NOT filename_is_filtered(connection.id,file.relative_path,file.name)),
|
||||
head.activated_at,
|
||||
'ready', COALESCE(refresh.duration_ms,0)
|
||||
FROM source_catalog_heads head
|
||||
JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id
|
||||
@@ -62,8 +66,19 @@ func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogN
|
||||
page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset}
|
||||
var parentID *string
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count,
|
||||
node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at
|
||||
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,
|
||||
(SELECT COUNT(*)::integer FROM source_catalog_nodes child
|
||||
WHERE child.snapshot_id=node.snapshot_id AND child.parent_node_id=node.node_id
|
||||
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))),
|
||||
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
|
||||
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
|
||||
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
|
||||
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
|
||||
(SELECT COALESCE(SUM(COALESCE(file.size_bytes,0)),0)::bigint FROM source_catalog_nodes file
|
||||
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
|
||||
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
|
||||
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
|
||||
node.size_bytes,node.modified_at
|
||||
FROM source_catalog_heads head
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan(
|
||||
@@ -112,7 +127,8 @@ SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*)::integer FROM source_catalog_heads head
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
|
||||
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2
|
||||
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
|
||||
return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err)
|
||||
}
|
||||
|
||||
@@ -122,6 +138,7 @@ WITH RECURSIVE current_head AS (
|
||||
), selected AS MATERIALIZED (
|
||||
SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE node.parent_node_id=$2
|
||||
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))
|
||||
ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id
|
||||
LIMIT $3 OFFSET $4
|
||||
), descendants(ancestor_id,snapshot_id,node_id) AS (
|
||||
@@ -135,9 +152,10 @@ WITH RECURSIVE current_head AS (
|
||||
COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility,
|
||||
publication.id AS publication_id,publication.ingest_state,publication.processing_state,
|
||||
current_version.content_state AS version_content_state,content.content_state AS object_content_state,
|
||||
content.local_spool_ref
|
||||
content.local_spool_ref,leaf.size_bytes
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file'
|
||||
AND NOT filename_is_filtered($1,leaf.relative_path,leaf.name)
|
||||
LEFT JOIN source_items item ON item.id=leaf.source_item_id
|
||||
LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=current_version.content_object_id
|
||||
@@ -158,6 +176,8 @@ WITH RECURSIVE current_head AS (
|
||||
) publication ON TRUE
|
||||
), stats AS (
|
||||
SELECT ancestor_id,
|
||||
COUNT(*)::integer AS eligible_file_count,
|
||||
COALESCE(SUM(COALESCE(size_bytes,0)),0)::bigint AS eligible_bytes,
|
||||
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count,
|
||||
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count,
|
||||
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count,
|
||||
@@ -169,7 +189,12 @@ WITH RECURSIVE current_head AS (
|
||||
FROM file_state GROUP BY ancestor_id
|
||||
)
|
||||
SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name,
|
||||
selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes,
|
||||
CASE WHEN selected.node_type='file' THEN 0 ELSE (
|
||||
SELECT COUNT(*)::integer FROM source_catalog_nodes child
|
||||
WHERE child.snapshot_id=selected.snapshot_id AND child.parent_node_id=selected.node_id
|
||||
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))
|
||||
) END,
|
||||
COALESCE(stats.eligible_file_count,0),COALESCE(stats.eligible_bytes,0),
|
||||
selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'),
|
||||
COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0),
|
||||
COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0)
|
||||
|
||||
Reference in New Issue
Block a user