From e6ef5a46934195fa35ecb915ef1c491638e5a901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 15 Sep 2026 10:31:55 +0800 Subject: [PATCH] feat(filters): allowlist mode for filename filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - filter rules gain mode ('exclude'|'allow'); nearest allow rule governs a directory subtree, extensions outside it are filtered before fetch - filename_extension() now requires a well-formed suffix, so dated names like 'CRA_V3.1手册_2024-07-31_en' no longer fake an extension; extensionless files are never filtered - API accepts {mode, extensions}; UI editor becomes grouped format chips with keep/exclude modes (keep-list default: docs/text/scripts/images) - migration 000036 + integration tests for allow, nesting and restore --- internal/governance/filename_filter.go | 18 +++ internal/governance/filename_filter_test.go | 17 +++ internal/httpapi/filename_filters.go | 13 ++- internal/httpapi/server_test.go | 8 ++ .../platform/postgres/filename_filters.go | 20 ++-- .../postgres/filename_filters_allow_test.go | 98 ++++++++++++++++ .../000036_filename_filter_modes.down.sql | 27 +++++ .../000036_filename_filter_modes.up.sql | 51 ++++++++ internal/platform/postgres/migrations_test.go | 4 +- web/src/FilenameFilterEditor.test.tsx | 25 ++-- web/src/FilenameFilterEditor.tsx | 110 +++++++++++++----- web/src/filenameFilters.ts | 18 ++- web/src/systemSettings.css | 5 + 13 files changed, 360 insertions(+), 54 deletions(-) create mode 100644 internal/platform/postgres/filename_filters_allow_test.go create mode 100644 internal/platform/postgres/migrations/000036_filename_filter_modes.down.sql create mode 100644 internal/platform/postgres/migrations/000036_filename_filter_modes.up.sql diff --git a/internal/governance/filename_filter.go b/internal/governance/filename_filter.go index 0e55975..60b5cff 100644 --- a/internal/governance/filename_filter.go +++ b/internal/governance/filename_filter.go @@ -13,11 +13,23 @@ var ( ErrFilenameFilterTargetNotFound = errors.New("filename filter target not found") ) +// Filter modes. Exclude (legacy) blocks the listed extensions; Allow turns the +// rule into a whitelist: matching ancestors contribute allowed extensions and +// a file whose extension is in none of them is filtered. When no allow rule +// matches, the connection allows everything the exclude rules do not block. +const ( + FilenameFilterExclude = "exclude" + FilenameFilterAllow = "allow" +) + +func FilenameFilterIsWhitelist(mode string) bool { return mode == FilenameFilterAllow } + // FilenameFilterRule is additive: the source rule uses an empty directory path; // a directory inherits that rule plus every matching ancestor rule. type FilenameFilterRule struct { ConnectionID string `json:"connection_id"` DirectoryPath string `json:"directory_path"` + Mode string `json:"mode"` ExcludedExtensions []string `json:"excluded_extensions"` UpdatedBy string `json:"updated_by"` UpdatedAt time.Time `json:"updated_at"` @@ -26,6 +38,7 @@ type FilenameFilterRule struct { type FilenameFilterMutation struct { ConnectionID string DirectoryPath string + Mode string ExcludedExtensions []string OperatorRef string Clear bool @@ -63,6 +76,11 @@ func (request FilenameFilterMutation) Validate() error { if strings.TrimSpace(request.ConnectionID) == "" || len(request.ConnectionID) > 36 || !validScopePath(request.DirectoryPath) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 || request.Now.IsZero() { return errors.New("filename filter target is invalid") } + switch request.Mode { + case "", FilenameFilterExclude, FilenameFilterAllow: + default: + return errors.New("filename filter mode is invalid") + } if request.Clear { if len(request.ExcludedExtensions) != 0 { return errors.New("clear filename filter must not include extensions") diff --git a/internal/governance/filename_filter_test.go b/internal/governance/filename_filter_test.go index ccf1b14..c2ae5e9 100644 --- a/internal/governance/filename_filter_test.go +++ b/internal/governance/filename_filter_test.go @@ -47,3 +47,20 @@ func TestFilenameFilterMutationRejectsUnsafePathAndUnnormalizedExtension(t *test } } } + +func TestFilenameFilterMutationAcceptsAllowMode(t *testing.T) { + request := FilenameFilterMutation{ConnectionID: "conn-1", Mode: FilenameFilterAllow, ExcludedExtensions: []string{"docx", "pdf"}, OperatorRef: "admin", Now: time.Now()} + if err := request.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } + if !FilenameFilterIsWhitelist(FilenameFilterAllow) || FilenameFilterIsWhitelist(FilenameFilterExclude) { + t.Fatal("mode helpers disagree") + } +} + +func TestFilenameFilterMutationRejectsUnknownMode(t *testing.T) { + request := FilenameFilterMutation{ConnectionID: "conn-1", Mode: "whitelist", ExcludedExtensions: []string{"pdf"}, OperatorRef: "admin", Now: time.Now()} + if err := request.Validate(); err == nil { + t.Fatalf("Validate(%#v) succeeded", request) + } +} diff --git a/internal/httpapi/filename_filters.go b/internal/httpapi/filename_filters.go index c21f147..fb4731e 100644 --- a/internal/httpapi/filename_filters.go +++ b/internal/httpapi/filename_filters.go @@ -14,7 +14,9 @@ import ( type filenameFilterInput struct { ConnectionID string `json:"connection_id"` DirectoryPath string `json:"directory_path"` + Mode string `json:"mode"` ExcludedExtensions []string `json:"excluded_extensions"` + Extensions []string `json:"extensions"` } func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) { @@ -43,17 +45,24 @@ func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config, writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"}) return } + // "extensions" is the friendly name for both modes; excluded_extensions stays + // accepted for callers written against the first (blocklist-only) release. + list := input.Extensions + if len(list) == 0 { + list = input.ExcludedExtensions + } extensions := []string(nil) if !clear { var err error - extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions) + extensions, err = governance.NormalizeExcludedExtensions(list) if err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"}) + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename extensions"}) return } } request := governance.FilenameFilterMutation{ ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath), + Mode: strings.TrimSpace(input.Mode), ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(), } if err := request.Validate(); err != nil { diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index a79b66b..4e8b758 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -275,6 +275,14 @@ func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) { !strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) { t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String()) } + allow := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"","mode":"allow","extensions":["PDF","docx"]}`)) + request.Header.Set("Idempotency-Key", "op-filter-allow") + handler.ServeHTTP(allow, request) + if allow.Code != http.StatusOK || store.filenameFilterRequest.Mode != "allow" || + !slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"docx", "pdf"}) { + t.Fatalf("allow status=%d request=%#v", allow.Code, store.filenameFilterRequest) + } } func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) { diff --git a/internal/platform/postgres/filename_filters.go b/internal/platform/postgres/filename_filters.go index 84eb903..e78fdff 100644 --- a/internal/platform/postgres/filename_filters.go +++ b/internal/platform/postgres/filename_filters.go @@ -15,7 +15,7 @@ func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string if connectionID == "" || len(connectionID) > 36 { return nil, governance.ErrInvalidFilenameFilter } - rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at + rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,mode,excluded_extensions,updated_by,updated_at FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID) if err != nil { return nil, fmt.Errorf("list filename filters: %w", err) @@ -24,7 +24,7 @@ FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directo result := make([]governance.FilenameFilterRule, 0) for rows.Next() { var rule governance.FilenameFilterRule - if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil { + if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.Mode, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil { return nil, fmt.Errorf("scan filename filter: %w", err) } result = append(result, rule) @@ -39,13 +39,17 @@ func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, reque if err := request.Validate(); err != nil { return governance.FilenameFilterRule{}, false, err } + if request.Mode == "" { + request.Mode = governance.FilenameFilterExclude + } fingerprint, err := operationFingerprint(struct { ConnectionID string `json:"connection_id"` DirectoryPath string `json:"directory_path"` + Mode string `json:"mode"` ExcludedExtensions []string `json:"excluded_extensions"` OperatorRef string `json:"operator_ref"` Clear bool `json:"clear"` - }{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear}) + }{request.ConnectionID, request.DirectoryPath, request.Mode, request.ExcludedExtensions, request.OperatorRef, request.Clear}) if err != nil { return governance.FilenameFilterRule{}, false, err } @@ -87,17 +91,17 @@ WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err) } } else { - if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at) -VALUES($1,$2,$3,$4,$5,$5) -ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`, - request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil { + if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,mode,excluded_extensions,updated_by,created_at,updated_at) +VALUES($1,$2,$5,$3,$4,$6,$6) +ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET mode=EXCLUDED.mode,excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`, + request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Mode, request.Now); err != nil { return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err) } } if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil { return governance.FilenameFilterRule{}, err } - return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil + return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, Mode: request.Mode, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil } func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error { diff --git a/internal/platform/postgres/filename_filters_allow_test.go b/internal/platform/postgres/filename_filters_allow_test.go new file mode 100644 index 0000000..8cbc11f --- /dev/null +++ b/internal/platform/postgres/filename_filters_allow_test.go @@ -0,0 +1,98 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// Whitelist mode: the nearest allow rule governs a file; unlisted extensions +// are filtered while extensionless and fake-extension names always pass. +func TestFilenameFilterAllowModeKeepsOnlySelectedExtensions(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-allow','dingtalk','白名单测试',true); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-allow','conn-allow',repeat('a',64),'白名单测试',6,2,4,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes) +VALUES('snapshot-allow','root-allow',NULL,'folder','白名单测试','',repeat('a',64),NULL,-1,1,4,40), + ('snapshot-allow','folder-allow','root-allow','folder','资料','资料',repeat('b',64),NULL,0,4,4,40), + ('snapshot-allow','allow-pdf','folder-allow','file','手册.pdf','资料/手册.pdf',repeat('c',64),10,0,0,1,10), + ('snapshot-allow','allow-zip','folder-allow','file','固件.zip','资料/固件.zip',repeat('d',64),10,1,0,1,10), + ('snapshot-allow','allow-readme','folder-allow','file','README','资料/README',repeat('e',64),10,2,0,1,10), + ('snapshot-allow','allow-versioned','folder-allow','file','CRA_V3.1手册','资料/CRA_V3.1手册',repeat('f',64),10,3,0,1,10); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-allow','snapshot-allow',NOW());`) + if err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + now := time.Date(2026, 9, 15, 3, 0, 0, 0, time.UTC) + rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{ + ConnectionID: "conn-allow", Mode: governance.FilenameFilterAllow, ExcludedExtensions: []string{"pdf"}, OperatorRef: "admin", Now: now, + }, "op-allow-set") + if err != nil || replayed || rule.Mode != "allow" || len(rule.ExcludedExtensions) != 1 { + t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err) + } + page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50}) + if err != nil || page.Total != 3 { + t.Fatalf("allow page=%#v err=%v", page, err) + } + names := map[string]bool{} + for _, item := range page.Items { + names[item.Name] = true + } + if !names["手册.pdf"] || !names["README"] || !names["CRA_V3.1手册"] || names["固件.zip"] { + t.Fatalf("allow page names=%v", names) + } + + // A deeper directory rule overrides the source-wide whitelist only below it. + if _, _, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{ + ConnectionID: "conn-allow", DirectoryPath: "资料", Mode: governance.FilenameFilterAllow, ExcludedExtensions: []string{"pdf", "zip"}, OperatorRef: "admin", Now: now.Add(time.Minute), + }, "op-allow-nested"); err != nil { + t.Fatal(err) + } + page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50}) + if err != nil || page.Total != 4 { + t.Fatalf("nested allow page=%#v err=%v", page, err) + } + { + found := false + for _, item := range page.Items { + if item.Name == "固件.zip" { + found = true + } + } + if !found { + t.Fatalf("nested allow must include zip page=%#v", page) + } + } + + // Clearing the nested rule restores nearest-wins to the source whitelist. + if _, _, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{ + ConnectionID: "conn-allow", DirectoryPath: "资料", Mode: governance.FilenameFilterAllow, OperatorRef: "admin", Clear: true, Now: now.Add(2 * time.Minute), + }, "op-allow-clear"); err != nil { + t.Fatal(err) + } + page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50}) + if err != nil || page.Total != 3 { + t.Fatalf("cleared page=%#v err=%v", page, err) + } +} diff --git a/internal/platform/postgres/migrations/000036_filename_filter_modes.down.sql b/internal/platform/postgres/migrations/000036_filename_filter_modes.down.sql new file mode 100644 index 0000000..3ed4ac8 --- /dev/null +++ b/internal/platform/postgres/migrations/000036_filename_filter_modes.down.sql @@ -0,0 +1,27 @@ +-- Restore the 000035 exclude-only evaluator and loose extension parser. +-- Allow rules cannot be expressed as a blocklist, so they are dropped. +DELETE FROM filename_filter_rules WHERE mode = 'allow'; + +CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$ + SELECT CASE + WHEN COALESCE(p_name, '') ~ '\.[^.]+$' + THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$')) + ELSE '' + END +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION filename_is_filtered( + p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT +) RETURNS BOOLEAN AS $$ + SELECT EXISTS ( + SELECT 1 + FROM filename_filter_rules rule + CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension + WHERE rule.source_connection_id = p_connection_id + AND (rule.directory_path = '' + OR p_relative_path LIKE rule.directory_path || '/%') + AND extension = filename_extension(p_name) + ) +$$ LANGUAGE SQL STABLE PARALLEL SAFE; + +ALTER TABLE filename_filter_rules DROP COLUMN IF EXISTS mode; diff --git a/internal/platform/postgres/migrations/000036_filename_filter_modes.up.sql b/internal/platform/postgres/migrations/000036_filename_filter_modes.up.sql new file mode 100644 index 0000000..aa97e94 --- /dev/null +++ b/internal/platform/postgres/migrations/000036_filename_filter_modes.up.sql @@ -0,0 +1,51 @@ +-- Filename filters gain a mode: 'allow' turns a rule into a whitelist (the +-- nearest matching allow rule governs a file and only listed extensions may +-- be ingested), 'exclude' keeps the legacy additive blocklist. Both modes can +-- coexist; a file must pass both. Files with no recognizable extension are +-- never filtered (extensionless names, dates-with-dots, "手册 V2.1" style). +ALTER TABLE filename_filter_rules + ADD COLUMN mode VARCHAR(10) NOT NULL DEFAULT 'exclude' + CHECK (mode IN ('exclude', 'allow')); + +-- Only well-formed suffixes count as extensions; 'CRA_V3.1手册_2024-07-31_en' +-- style names with stray dots no longer produce fake extensions. +CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$ + SELECT CASE + WHEN COALESCE(p_name, '') ~ '\.[^.]+$' + AND LOWER(SUBSTRING(p_name FROM '\.([^.]+)$')) ~ '^[a-z0-9][a-z0-9_-]{0,15}$' + THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$')) + ELSE '' + END +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION filename_is_filtered( + p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT +) RETURNS BOOLEAN AS $$ + WITH nearest_allow AS ( + SELECT rule.excluded_extensions AS extensions + FROM filename_filter_rules rule + WHERE rule.source_connection_id = p_connection_id + AND rule.mode = 'allow' + AND (rule.directory_path = '' + OR p_relative_path LIKE rule.directory_path || '/%') + ORDER BY LENGTH(rule.directory_path) DESC + LIMIT 1 + ) + SELECT CASE + WHEN filename_extension(p_name) = '' THEN FALSE + WHEN EXISTS ( + SELECT 1 + FROM filename_filter_rules rule + CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension + WHERE rule.source_connection_id = p_connection_id + AND rule.mode = 'exclude' + AND (rule.directory_path = '' + OR p_relative_path LIKE rule.directory_path || '/%') + AND extension = filename_extension(p_name) + ) THEN TRUE + WHEN EXISTS (SELECT 1 FROM nearest_allow) + THEN NOT EXISTS (SELECT 1 FROM nearest_allow + WHERE filename_extension(p_name) = ANY(extensions)) + ELSE FALSE + END +$$ LANGUAGE SQL STABLE PARALLEL SAFE; diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 95dd0f6..5932fcb 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 35 { - t.Fatalf("migration count = %d, want 35", len(migrations)) + if len(migrations) != 36 { + t.Fatalf("migration count = %d, want 36", len(migrations)) } if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") { t.Fatalf("migration 1 = %#v", migrations[0]) diff --git a/web/src/FilenameFilterEditor.test.tsx b/web/src/FilenameFilterEditor.test.tsx index 6be4d58..2039e22 100644 --- a/web/src/FilenameFilterEditor.test.tsx +++ b/web/src/FilenameFilterEditor.test.tsx @@ -4,23 +4,26 @@ import { describe, expect, it } from 'vitest' import { FilenameFilterEditorView } from './FilenameFilterEditor' describe('FilenameFilterEditorView', () => { - it('shows source exclusions and inherited directory exclusions', () => { + it('renders format groups with the allow-mode preselection', () => { const html = renderToStaticMarkup( undefined} + options={['pdf', 'zip', 'mp4']} + selected={['pdf']} + onToggle={() => undefined} busy={false} - message={null} + hasOwnRule={false} + message="已保留 .pdf" error={null} - onDraft={() => undefined} onSave={() => undefined} onClear={() => undefined} />) - expect(html).toContain('文件名筛选') - expect(html).toContain('.zip') - expect(html).toContain('.rar') - expect(html).toContain('.mp4') - expect(html).toContain('继承排除') + expect(html).toContain('文件格式筛选') + expect(html).toContain('勾选保留') + expect(html).toContain('勾选排除') + expect(html).toContain('.pdf') + expect(html).toContain('aria-pressed="true"') + expect(html).toContain('已保留 .pdf') }) }) diff --git a/web/src/FilenameFilterEditor.tsx b/web/src/FilenameFilterEditor.tsx index e9bc124..888b587 100644 --- a/web/src/FilenameFilterEditor.tsx +++ b/web/src/FilenameFilterEditor.tsx @@ -1,35 +1,74 @@ import { useEffect, useMemo, useState } from 'react' -import { clearFilenameFilter, DEFAULT_EXCLUDED_EXTENSIONS, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters' +import { clearFilenameFilter, DEFAULT_ALLOWED_EXTENSIONS, DEFAULT_EXCLUDED_EXTENSIONS, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters' import type { FilenameFilterRule } from './filenameFilters' import './systemSettings.css' +type FilterMode = 'allow' | 'exclude' + interface ViewProps { label: string - directExtensions: string[] - inheritedExtensions: string[] - draft: string + mode: FilterMode + onModeChange: (mode: FilterMode) => void + options: string[] + selected: string[] + onToggle: (extension: string) => void busy: boolean + hasOwnRule: boolean message: string | null error: string | null - onDraft: (value: string) => void onSave: () => void onClear: () => void } -const tags = (values: string[]) => values.map((value) => .{value}) +// Known knowledge-base suffixes not tied to a single family; kept out of the +// curated groups so they stay one-click selectable. +const EXTRA_FORMATS = ['eds', 'qm', 'sdp2', 'chm', 'wyc', 'dns', 'pdb', 'pwf', 'tvf', 'tvx', 'tvd', 'tid'] -export function FilenameFilterEditorView({ label, directExtensions, inheritedExtensions, draft, busy, message, error, onDraft, onSave, onClear }: ViewProps) { - return
-
文件名筛选{label} · 按扩展名排除,大小写不敏感
- {inheritedExtensions.length ?
继承排除{tags(inheritedExtensions)}
: null} - {directExtensions.length ?
本级排除{tags(directExtensions)}
: 本级尚未设置;只继承上级规则。} -
- onDraft(event.target.value)} aria-label={`${label}排除扩展名`} /> - - {directExtensions.length ? : null} +const FORMAT_GROUPS: { label: string; extensions: string[] }[] = [ + { label: '在线文档', extensions: ['adoc', 'able', 'axls'] }, + { label: '办公文档', extensions: ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'csv'] }, + { label: '文本', extensions: ['txt', 'md', 'json', 'xml', 'html', 'yaml', 'conf', 'config', 'ini', 'log'] }, + { label: '脚本源码', extensions: ['lua', 'py', 'sh', 'bat', 'qml'] }, + { label: '图片', extensions: ['png', 'jpg', 'jpeg', 'gif', 'bmp', 'webp', 'heic', 'tif'] }, + { label: '其他', extensions: EXTRA_FORMATS }, +] + +const ALL_OPTIONS = [...new Set(FORMAT_GROUPS.flatMap((group) => group.extensions))].sort() + +function groupOf(extension: string): string { + return DEFAULT_EXCLUDED_EXTENSIONS.includes(extension) ? '排除项' : '其他' +} + +export function FilenameFilterEditorView({ label, mode, onModeChange, options, selected, onToggle, busy, hasOwnRule, message, error, onSave, onClear }: ViewProps) { + const selectedSet = useMemo(() => new Set(selected), [selected]) + const unknown = selected.filter((extension) => !options.includes(extension)) + return
+
文件格式筛选{label} · 选择要入库的文件格式,未勾选的一律不取件
+
+ + +
- 保存后立即停止匹配文件的新取件与入库;原始来源文件和快照审计记录保留。 +
+ {FORMAT_GROUPS.map((group) =>
+ {group.label} +
{group.extensions.map((extension) => )}
+
)} +
+ {unknown.length ?
规则含未知格式{unknown.map((extension) => .{extension})}
: null} +
+ { + if (event.key !== 'Enter') return + event.preventDefault() + const target = event.currentTarget + for (const extension of parseExtensionDraft(target.value)) onToggle(extension) + target.value = '' + }} aria-label="手动补充格式" /> + + {hasOwnRule ? : null} +
+ {mode === 'allow' ? '白名单:无扩展名的文件不受限制;目录级规则就近覆盖上级。' : '黑名单:仅排除勾选格式,其余可入库。'} {message ?

{message}

: null} {error ?

{error}

: null}
@@ -37,7 +76,9 @@ export function FilenameFilterEditorView({ label, directExtensions, inheritedExt export function FilenameFilterEditor({ connectionID, directoryPath = '', label, onChanged }: { connectionID: string; directoryPath?: string; label: string; onChanged?: () => void }) { const [rules, setRules] = useState([]) - const [draft, setDraft] = useState(DEFAULT_EXCLUDED_EXTENSIONS.join(', ')) + const [mode, setMode] = useState('allow') + const [selected, setSelected] = useState(DEFAULT_ALLOWED_EXTENSIONS) + const [touched, setTouched] = useState(directoryPath === '') const [busy, setBusy] = useState(false) const [message, setMessage] = useState(null) const [error, setError] = useState(null) @@ -47,33 +88,46 @@ export function FilenameFilterEditor({ connectionID, directoryPath = '', label, for (const rule of rules) { if (rule.directory_path === directoryPath) continue if (rule.directory_path === '' || directoryPath.startsWith(`${rule.directory_path}/`)) { - for (const extension of rule.excluded_extensions) values.add(extension) + for (const extension of rule.excluded_extensions) values.add(`${extension}:${rule.mode || 'exclude'}`) } } - return [...values].sort() + return values }, [directoryPath, rules]) + const options = useMemo(() => [...new Set([...ALL_OPTIONS, ...selected])].sort(), [selected]) const load = async (signal?: AbortSignal) => { const next = await fetchFilenameFilters(connectionID, signal) setRules(next) const current = next.find((rule) => rule.directory_path === directoryPath) - setDraft(current?.excluded_extensions.join(', ') || DEFAULT_EXCLUDED_EXTENSIONS.join(', ')) + if (current) { + setMode(current.mode === 'exclude' ? 'exclude' : 'allow') + setSelected(current.excluded_extensions) + } else if (!touched) { + setMode('allow') + setSelected(directoryPath === '' ? DEFAULT_ALLOWED_EXTENSIONS : [...new Set([...(inheritedHas('allow') ? inheritedValues('allow') : DEFAULT_ALLOWED_EXTENSIONS)])]) + } } + const inheritedHas = (want: string) => [...inherited].some((value) => value.endsWith(`:${want}`)) + const inheritedValues = (want: string) => [...inherited].filter((value) => value.endsWith(`:${want}`)).map((value) => value.split(':')[0]) + useEffect(() => { const controller = new AbortController() - void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件名筛选失败') }) + void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件格式筛选失败') }) return () => controller.abort() }, [connectionID, directoryPath]) + const toggle = (extension: string) => { + setTouched(true) + setSelected((previous) => previous.includes(extension) ? previous.filter((value) => value !== extension) : [...previous, extension].sort()) + } const save = async () => { - const extensions = parseExtensionDraft(draft) setBusy(true); setError(null); setMessage(null) try { - await saveFilenameFilter(connectionID, directoryPath, extensions) + await saveFilenameFilter(connectionID, directoryPath, mode, selected) await load() - setMessage(`已排除 ${extensions.map((value) => `.${value}`).join('、')};目录、差异、取件和入库立即生效。`) + setMessage(mode === 'allow' ? `已保留 ${selected.map((value) => `.${value}`).join('、')};其余格式不再取件入库。` : `已排除 ${selected.map((value) => `.${value}`).join('、')}。`) onChanged?.() - } catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件名筛选失败') } + } catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件格式筛选失败') } finally { setBusy(false) } } const clear = async () => { @@ -81,10 +135,10 @@ export function FilenameFilterEditor({ connectionID, directoryPath = '', label, try { await clearFilenameFilter(connectionID, directoryPath) await load() - setMessage('已移除本级筛选,恢复继承上级规则。') + setMessage('已移除本级规则,恢复继承上级。') onChanged?.() - } catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件名筛选失败') } + } catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件格式筛选失败') } finally { setBusy(false) } } - return void save()} onClear={() => void clear()} /> + return { setTouched(true); setMode(next) }} options={options} selected={selected} onToggle={toggle} busy={busy} hasOwnRule={Boolean(direct)} message={message} error={error} onSave={() => void save()} onClear={() => void clear()} /> } diff --git a/web/src/filenameFilters.ts b/web/src/filenameFilters.ts index 1cf6f43..f152016 100644 --- a/web/src/filenameFilters.ts +++ b/web/src/filenameFilters.ts @@ -3,6 +3,7 @@ import { newOperationID } from './operationKey' export interface FilenameFilterRule { connection_id: string directory_path: string + mode: 'allow' | 'exclude' excluded_extensions: string[] updated_by: string updated_at: string @@ -24,10 +25,10 @@ export async function fetchFilenameFilters(connectionID: string, signal?: AbortS }) } -export async function saveFilenameFilter(connectionID: string, directoryPath: string, extensions: string[]): Promise { +export async function saveFilenameFilter(connectionID: string, directoryPath: string, mode: 'allow' | 'exclude', extensions: string[]): Promise { const response = await fetch('/api/v1/filename-filters', { method: 'PUT', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() }, - body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, excluded_extensions: extensions }), + body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, mode, extensions }), }) if (!response.ok) throw new Error(`保存文件名筛选失败(HTTP ${response.status})`) const envelope = await response.json() as { data?: FilenameFilterRule } @@ -44,7 +45,8 @@ export async function clearFilenameFilter(connectionID: string, directoryPath: s } // Source-wide default exclusions: archives/media/executables/CAD/design -// binaries have no RAG value for the support knowledge bases. +// binaries have no RAG value for the support knowledge bases. Kept for the +// legacy "exclude" mode editor prefill. export const DEFAULT_EXCLUDED_EXTENSIONS = [ "zip", "rar", "7z", "gz", "tgz", "tar", "xz", "iso", "crdownload", "mp4", "avi", "mkv", "mov", "wmv", "m4v", "mp3", "wav", "flac", @@ -54,6 +56,16 @@ export const DEFAULT_EXCLUDED_EXTENSIONS = [ "dlink", "atomproject", "amind", "appt", "rdk", "vcmx", "swf", "dav", ].sort() +// Whitelist defaults for the "keep selected" mode: documents, text/scripts +// and images that a RAG knowledge base can actually parse and cite. +export const DEFAULT_ALLOWED_EXTENSIONS = [ + "adoc", "able", "axls", + "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "csv", + "txt", "md", "json", "xml", "html", "yaml", "conf", "config", "ini", "log", + "lua", "py", "sh", "bat", "qml", + "png", "jpg", "jpeg", "gif", "bmp", "webp", "heic", "tif", +].sort() + export function parseExtensionDraft(value: string): string[] { return [...new Set(value.split(/[,,\s]+/).map((item) => item.trim().toLowerCase().replace(/^\./, '')).filter(Boolean))].sort() } diff --git a/web/src/systemSettings.css b/web/src/systemSettings.css index 5e97afa..6ad166a 100644 --- a/web/src/systemSettings.css +++ b/web/src/systemSettings.css @@ -40,3 +40,8 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d .filename-filter-inherited,.filename-filter-direct{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin:6px 0}.filename-filter-inherited span,.filename-filter-direct span{font-size:10px;color:var(--ink-mute)} .filename-filter-editor code{padding:2px 7px;border-radius:999px;background:#eaf2ff;color:#2456a6;font-family:inherit;font-size:11px} .filename-filter-form{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:9px 0}.filename-filter-form input{min-width:220px;flex:1;padding:7px 9px;border:1px solid var(--border-strong);border-radius:7px;background:var(--bg-panel)} +.filename-filter-modes{display:flex;gap:12px;align-self:center;font-size:11px;color:var(--ink-base)}.filename-filter-modes label{display:flex;gap:4px;align-items:center;cursor:pointer} +.filename-filter-groups{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:6px 14px;margin:8px 0}.filename-filter-group{display:grid;gap:4px}.filename-filter-group>span{color:var(--ink-mute);font-size:9px} +.filename-filter-group>div{display:flex;flex-wrap:wrap;gap:4px}.filename-filter-group button{border:1px solid var(--border-base);background:var(--bg-panel);color:var(--ink-base);border-radius:999px;padding:2px 8px;font-size:10px;font-family:inherit;cursor:pointer} +.filename-filter-group button.picked{background:var(--c-run);border-color:var(--c-run);color:var(--bg-panel);font-weight:650}.filename-filter-group button.blocked{color:var(--c-fail)} +@media(max-width:900px){.filename-filter-groups{grid-template-columns:1fr 1fr}}