feat(filters): allowlist mode for filename filters

- filter rules gain mode ('exclude'|'allow'); nearest allow rule governs a
  directory subtree, extensions outside it are filtered before fetch
- filename_extension() now requires a well-formed suffix, so dated names like
  'CRA_V3.1手册_2024-07-31_en' no longer fake an extension; extensionless
  files are never filtered
- API accepts {mode, extensions}; UI editor becomes grouped format chips with
  keep/exclude modes (keep-list default: docs/text/scripts/images)
- migration 000036 + integration tests for allow, nesting and restore
This commit is contained in:
姚章浩
2026-09-15 10:31:55 +08:00
parent f7c65571e4
commit e6ef5a4693
13 changed files with 360 additions and 54 deletions
+18
View File
@@ -13,11 +13,23 @@ var (
ErrFilenameFilterTargetNotFound = errors.New("filename filter target not found")
)
// Filter modes. Exclude (legacy) blocks the listed extensions; Allow turns the
// rule into a whitelist: matching ancestors contribute allowed extensions and
// a file whose extension is in none of them is filtered. When no allow rule
// matches, the connection allows everything the exclude rules do not block.
const (
FilenameFilterExclude = "exclude"
FilenameFilterAllow = "allow"
)
func FilenameFilterIsWhitelist(mode string) bool { return mode == FilenameFilterAllow }
// FilenameFilterRule is additive: the source rule uses an empty directory path;
// a directory inherits that rule plus every matching ancestor rule.
type FilenameFilterRule struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
Mode string `json:"mode"`
ExcludedExtensions []string `json:"excluded_extensions"`
UpdatedBy string `json:"updated_by"`
UpdatedAt time.Time `json:"updated_at"`
@@ -26,6 +38,7 @@ type FilenameFilterRule struct {
type FilenameFilterMutation struct {
ConnectionID string
DirectoryPath string
Mode string
ExcludedExtensions []string
OperatorRef string
Clear bool
@@ -63,6 +76,11 @@ func (request FilenameFilterMutation) Validate() error {
if strings.TrimSpace(request.ConnectionID) == "" || len(request.ConnectionID) > 36 || !validScopePath(request.DirectoryPath) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 || request.Now.IsZero() {
return errors.New("filename filter target is invalid")
}
switch request.Mode {
case "", FilenameFilterExclude, FilenameFilterAllow:
default:
return errors.New("filename filter mode is invalid")
}
if request.Clear {
if len(request.ExcludedExtensions) != 0 {
return errors.New("clear filename filter must not include extensions")
@@ -47,3 +47,20 @@ func TestFilenameFilterMutationRejectsUnsafePathAndUnnormalizedExtension(t *test
}
}
}
func TestFilenameFilterMutationAcceptsAllowMode(t *testing.T) {
request := FilenameFilterMutation{ConnectionID: "conn-1", Mode: FilenameFilterAllow, ExcludedExtensions: []string{"docx", "pdf"}, OperatorRef: "admin", Now: time.Now()}
if err := request.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
if !FilenameFilterIsWhitelist(FilenameFilterAllow) || FilenameFilterIsWhitelist(FilenameFilterExclude) {
t.Fatal("mode helpers disagree")
}
}
func TestFilenameFilterMutationRejectsUnknownMode(t *testing.T) {
request := FilenameFilterMutation{ConnectionID: "conn-1", Mode: "whitelist", ExcludedExtensions: []string{"pdf"}, OperatorRef: "admin", Now: time.Now()}
if err := request.Validate(); err == nil {
t.Fatalf("Validate(%#v) succeeded", request)
}
}
+11 -2
View File
@@ -14,7 +14,9 @@ import (
type filenameFilterInput struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
Mode string `json:"mode"`
ExcludedExtensions []string `json:"excluded_extensions"`
Extensions []string `json:"extensions"`
}
func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) {
@@ -43,17 +45,24 @@ func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config,
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
return
}
// "extensions" is the friendly name for both modes; excluded_extensions stays
// accepted for callers written against the first (blocklist-only) release.
list := input.Extensions
if len(list) == 0 {
list = input.ExcludedExtensions
}
extensions := []string(nil)
if !clear {
var err error
extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions)
extensions, err = governance.NormalizeExcludedExtensions(list)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"})
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename extensions"})
return
}
}
request := governance.FilenameFilterMutation{
ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath),
Mode: strings.TrimSpace(input.Mode),
ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(),
}
if err := request.Validate(); err != nil {
+8
View File
@@ -275,6 +275,14 @@ func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) {
!strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String())
}
allow := httptest.NewRecorder()
request = httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"","mode":"allow","extensions":["PDF","docx"]}`))
request.Header.Set("Idempotency-Key", "op-filter-allow")
handler.ServeHTTP(allow, request)
if allow.Code != http.StatusOK || store.filenameFilterRequest.Mode != "allow" ||
!slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"docx", "pdf"}) {
t.Fatalf("allow status=%d request=%#v", allow.Code, store.filenameFilterRequest)
}
}
func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) {
+12 -8
View File
@@ -15,7 +15,7 @@ func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string
if connectionID == "" || len(connectionID) > 36 {
return nil, governance.ErrInvalidFilenameFilter
}
rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at
rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,mode,excluded_extensions,updated_by,updated_at
FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID)
if err != nil {
return nil, fmt.Errorf("list filename filters: %w", err)
@@ -24,7 +24,7 @@ FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directo
result := make([]governance.FilenameFilterRule, 0)
for rows.Next() {
var rule governance.FilenameFilterRule
if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil {
if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.Mode, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan filename filter: %w", err)
}
result = append(result, rule)
@@ -39,13 +39,17 @@ func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, reque
if err := request.Validate(); err != nil {
return governance.FilenameFilterRule{}, false, err
}
if request.Mode == "" {
request.Mode = governance.FilenameFilterExclude
}
fingerprint, err := operationFingerprint(struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
Mode string `json:"mode"`
ExcludedExtensions []string `json:"excluded_extensions"`
OperatorRef string `json:"operator_ref"`
Clear bool `json:"clear"`
}{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear})
}{request.ConnectionID, request.DirectoryPath, request.Mode, request.ExcludedExtensions, request.OperatorRef, request.Clear})
if err != nil {
return governance.FilenameFilterRule{}, false, err
}
@@ -87,17 +91,17 @@ WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative
return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err)
}
} else {
if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil {
if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,mode,excluded_extensions,updated_by,created_at,updated_at)
VALUES($1,$2,$5,$3,$4,$6,$6)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET mode=EXCLUDED.mode,excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Mode, request.Now); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err)
}
}
if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil {
return governance.FilenameFilterRule{}, err
}
return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil
return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, Mode: request.Mode, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil
}
func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error {
@@ -0,0 +1,98 @@
package postgres
import (
"context"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
// Whitelist mode: the nearest allow rule governs a file; unlisted extensions
// are filtered while extensionless and fake-extension names always pass.
func TestFilenameFilterAllowModeKeepsOnlySelectedExtensions(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-allow','dingtalk','白名单测试',true);
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-allow','conn-allow',repeat('a',64),'白名单测试',6,2,4,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-allow','root-allow',NULL,'folder','白名单测试','',repeat('a',64),NULL,-1,1,4,40),
('snapshot-allow','folder-allow','root-allow','folder','资料','资料',repeat('b',64),NULL,0,4,4,40),
('snapshot-allow','allow-pdf','folder-allow','file','手册.pdf','资料/手册.pdf',repeat('c',64),10,0,0,1,10),
('snapshot-allow','allow-zip','folder-allow','file','固件.zip','资料/固件.zip',repeat('d',64),10,1,0,1,10),
('snapshot-allow','allow-readme','folder-allow','file','README','资料/README',repeat('e',64),10,2,0,1,10),
('snapshot-allow','allow-versioned','folder-allow','file','CRA_V3.1手册','资料/CRA_V3.1手册',repeat('f',64),10,3,0,1,10);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-allow','snapshot-allow',NOW());`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 15, 3, 0, 0, 0, time.UTC)
rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-allow", Mode: governance.FilenameFilterAllow, ExcludedExtensions: []string{"pdf"}, OperatorRef: "admin", Now: now,
}, "op-allow-set")
if err != nil || replayed || rule.Mode != "allow" || len(rule.ExcludedExtensions) != 1 {
t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err)
}
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50})
if err != nil || page.Total != 3 {
t.Fatalf("allow page=%#v err=%v", page, err)
}
names := map[string]bool{}
for _, item := range page.Items {
names[item.Name] = true
}
if !names["手册.pdf"] || !names["README"] || !names["CRA_V3.1手册"] || names["固件.zip"] {
t.Fatalf("allow page names=%v", names)
}
// A deeper directory rule overrides the source-wide whitelist only below it.
if _, _, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-allow", DirectoryPath: "资料", Mode: governance.FilenameFilterAllow, ExcludedExtensions: []string{"pdf", "zip"}, OperatorRef: "admin", Now: now.Add(time.Minute),
}, "op-allow-nested"); err != nil {
t.Fatal(err)
}
page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50})
if err != nil || page.Total != 4 {
t.Fatalf("nested allow page=%#v err=%v", page, err)
}
{
found := false
for _, item := range page.Items {
if item.Name == "固件.zip" {
found = true
}
}
if !found {
t.Fatalf("nested allow must include zip page=%#v", page)
}
}
// Clearing the nested rule restores nearest-wins to the source whitelist.
if _, _, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-allow", DirectoryPath: "资料", Mode: governance.FilenameFilterAllow, OperatorRef: "admin", Clear: true, Now: now.Add(2 * time.Minute),
}, "op-allow-clear"); err != nil {
t.Fatal(err)
}
page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-allow", ParentNodeID: "folder-allow", Limit: 50})
if err != nil || page.Total != 3 {
t.Fatalf("cleared page=%#v err=%v", page, err)
}
}
@@ -0,0 +1,27 @@
-- Restore the 000035 exclude-only evaluator and loose extension parser.
-- Allow rules cannot be expressed as a blocklist, so they are dropped.
DELETE FROM filename_filter_rules WHERE mode = 'allow';
CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$
SELECT CASE
WHEN COALESCE(p_name, '') ~ '\.[^.]+$'
THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$'))
ELSE ''
END
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
CREATE OR REPLACE FUNCTION filename_is_filtered(
p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT
) RETURNS BOOLEAN AS $$
SELECT EXISTS (
SELECT 1
FROM filename_filter_rules rule
CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension
WHERE rule.source_connection_id = p_connection_id
AND (rule.directory_path = ''
OR p_relative_path LIKE rule.directory_path || '/%')
AND extension = filename_extension(p_name)
)
$$ LANGUAGE SQL STABLE PARALLEL SAFE;
ALTER TABLE filename_filter_rules DROP COLUMN IF EXISTS mode;
@@ -0,0 +1,51 @@
-- Filename filters gain a mode: 'allow' turns a rule into a whitelist (the
-- nearest matching allow rule governs a file and only listed extensions may
-- be ingested), 'exclude' keeps the legacy additive blocklist. Both modes can
-- coexist; a file must pass both. Files with no recognizable extension are
-- never filtered (extensionless names, dates-with-dots, "手册 V2.1" style).
ALTER TABLE filename_filter_rules
ADD COLUMN mode VARCHAR(10) NOT NULL DEFAULT 'exclude'
CHECK (mode IN ('exclude', 'allow'));
-- Only well-formed suffixes count as extensions; 'CRA_V3.1手册_2024-07-31_en'
-- style names with stray dots no longer produce fake extensions.
CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$
SELECT CASE
WHEN COALESCE(p_name, '') ~ '\.[^.]+$'
AND LOWER(SUBSTRING(p_name FROM '\.([^.]+)$')) ~ '^[a-z0-9][a-z0-9_-]{0,15}$'
THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$'))
ELSE ''
END
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
CREATE OR REPLACE FUNCTION filename_is_filtered(
p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT
) RETURNS BOOLEAN AS $$
WITH nearest_allow AS (
SELECT rule.excluded_extensions AS extensions
FROM filename_filter_rules rule
WHERE rule.source_connection_id = p_connection_id
AND rule.mode = 'allow'
AND (rule.directory_path = ''
OR p_relative_path LIKE rule.directory_path || '/%')
ORDER BY LENGTH(rule.directory_path) DESC
LIMIT 1
)
SELECT CASE
WHEN filename_extension(p_name) = '' THEN FALSE
WHEN EXISTS (
SELECT 1
FROM filename_filter_rules rule
CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension
WHERE rule.source_connection_id = p_connection_id
AND rule.mode = 'exclude'
AND (rule.directory_path = ''
OR p_relative_path LIKE rule.directory_path || '/%')
AND extension = filename_extension(p_name)
) THEN TRUE
WHEN EXISTS (SELECT 1 FROM nearest_allow)
THEN NOT EXISTS (SELECT 1 FROM nearest_allow
WHERE filename_extension(p_name) = ANY(extensions))
ELSE FALSE
END
$$ LANGUAGE SQL STABLE PARALLEL SAFE;
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 35 {
t.Fatalf("migration count = %d, want 35", len(migrations))
if len(migrations) != 36 {
t.Fatalf("migration count = %d, want 36", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
+14 -11
View File
@@ -4,23 +4,26 @@ import { describe, expect, it } from 'vitest'
import { FilenameFilterEditorView } from './FilenameFilterEditor'
describe('FilenameFilterEditorView', () => {
it('shows source exclusions and inherited directory exclusions', () => {
it('renders format groups with the allow-mode preselection', () => {
const html = renderToStaticMarkup(<FilenameFilterEditorView
label="当前目录"
directExtensions={['mp4']}
inheritedExtensions={['rar', 'zip']}
draft="mp4"
mode="allow"
onModeChange={() => undefined}
options={['pdf', 'zip', 'mp4']}
selected={['pdf']}
onToggle={() => undefined}
busy={false}
message={null}
hasOwnRule={false}
message="已保留 .pdf"
error={null}
onDraft={() => undefined}
onSave={() => undefined}
onClear={() => undefined}
/>)
expect(html).toContain('文件名筛选')
expect(html).toContain('.zip')
expect(html).toContain('.rar')
expect(html).toContain('.mp4')
expect(html).toContain('继承排除')
expect(html).toContain('文件格式筛选')
expect(html).toContain('勾选保留')
expect(html).toContain('勾选排除')
expect(html).toContain('.pdf')
expect(html).toContain('aria-pressed="true"')
expect(html).toContain('已保留 .pdf')
})
})
+82 -28
View File
@@ -1,35 +1,74 @@
import { useEffect, useMemo, useState } from 'react'
import { clearFilenameFilter, DEFAULT_EXCLUDED_EXTENSIONS, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters'
import { clearFilenameFilter, DEFAULT_ALLOWED_EXTENSIONS, DEFAULT_EXCLUDED_EXTENSIONS, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters'
import type { FilenameFilterRule } from './filenameFilters'
import './systemSettings.css'
type FilterMode = 'allow' | 'exclude'
interface ViewProps {
label: string
directExtensions: string[]
inheritedExtensions: string[]
draft: string
mode: FilterMode
onModeChange: (mode: FilterMode) => void
options: string[]
selected: string[]
onToggle: (extension: string) => void
busy: boolean
hasOwnRule: boolean
message: string | null
error: string | null
onDraft: (value: string) => void
onSave: () => void
onClear: () => void
}
const tags = (values: string[]) => values.map((value) => <code key={value}>.{value}</code>)
// Known knowledge-base suffixes not tied to a single family; kept out of the
// curated groups so they stay one-click selectable.
const EXTRA_FORMATS = ['eds', 'qm', 'sdp2', 'chm', 'wyc', 'dns', 'pdb', 'pwf', 'tvf', 'tvx', 'tvd', 'tid']
export function FilenameFilterEditorView({ label, directExtensions, inheritedExtensions, draft, busy, message, error, onDraft, onSave, onClear }: ViewProps) {
return <section className="filename-filter-editor" aria-label={`${label}文件名筛选`}>
<div className="filename-filter-heading"><div><strong>文件名筛选</strong><small>{label} · 按扩展名排除,大小写不敏感</small></div></div>
{inheritedExtensions.length ? <div className="filename-filter-inherited"><span>继承排除</span>{tags(inheritedExtensions)}</div> : null}
{directExtensions.length ? <div className="filename-filter-direct"><span>本级排除</span>{tags(directExtensions)}</div> : <small>本级尚未设置;只继承上级规则。</small>}
<div className="filename-filter-form">
<input value={draft} placeholder="zip, rar, mp4" onChange={(event) => onDraft(event.target.value)} aria-label={`${label}排除扩展名`} />
<button className="secondary" disabled={busy || parseExtensionDraft(draft).length === 0} onClick={onSave}>{busy ? '保存中…' : '保存筛选'}</button>
{directExtensions.length ? <button className="link" disabled={busy} onClick={onClear}>移除本级筛选</button> : null}
const FORMAT_GROUPS: { label: string; extensions: string[] }[] = [
{ label: '在线文档', extensions: ['adoc', 'able', 'axls'] },
{ label: '办公文档', extensions: ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'csv'] },
{ label: '文本', extensions: ['txt', 'md', 'json', 'xml', 'html', 'yaml', 'conf', 'config', 'ini', 'log'] },
{ label: '脚本源码', extensions: ['lua', 'py', 'sh', 'bat', 'qml'] },
{ label: '图片', extensions: ['png', 'jpg', 'jpeg', 'gif', 'bmp', 'webp', 'heic', 'tif'] },
{ label: '其他', extensions: EXTRA_FORMATS },
]
const ALL_OPTIONS = [...new Set(FORMAT_GROUPS.flatMap((group) => group.extensions))].sort()
function groupOf(extension: string): string {
return DEFAULT_EXCLUDED_EXTENSIONS.includes(extension) ? '排除项' : '其他'
}
export function FilenameFilterEditorView({ label, mode, onModeChange, options, selected, onToggle, busy, hasOwnRule, message, error, onSave, onClear }: ViewProps) {
const selectedSet = useMemo(() => new Set(selected), [selected])
const unknown = selected.filter((extension) => !options.includes(extension))
return <section className="filename-filter-editor" aria-label={`${label}文件格式筛选`}>
<div className="filename-filter-heading"><div><strong>文件格式筛选</strong><small>{label} · 选择要入库的文件格式,未勾选的一律不取件</small></div>
<div className="filename-filter-modes" role="radiogroup" aria-label="筛选模式">
<label><input type="radio" name={`mode-${label}`} checked={mode === 'allow'} onChange={() => onModeChange('allow')} />勾选保留</label>
<label><input type="radio" name={`mode-${label}`} checked={mode === 'exclude'} onChange={() => onModeChange('exclude')} />勾选排除</label>
</div>
</div>
<small>保存后立即停止匹配文件的新取件与入库;原始来源文件和快照审计记录保留。</small>
<div className="filename-filter-groups">
{FORMAT_GROUPS.map((group) => <div className="filename-filter-group" key={group.label}>
<span>{group.label}</span>
<div>{group.extensions.map((extension) => <button key={extension} type="button" className={selectedSet.has(extension) ? 'picked' : groupOf(extension) === '排除项' ? 'blocked' : ''} aria-pressed={selectedSet.has(extension)} onClick={() => onToggle(extension)}>.{extension}</button>)}</div>
</div>)}
</div>
{unknown.length ? <div className="filename-filter-direct"><span>规则含未知格式</span>{unknown.map((extension) => <code key={extension}>.{extension}</code>)}</div> : null}
<div className="filename-filter-form">
<input placeholder={mode === 'allow' ? '手动补充格式:csv, lua' : '手动补充排除:zip, exe'} onKeyDown={(event) => {
if (event.key !== 'Enter') return
event.preventDefault()
const target = event.currentTarget
for (const extension of parseExtensionDraft(target.value)) onToggle(extension)
target.value = ''
}} aria-label="手动补充格式" />
<button className="secondary" disabled={busy || selected.length === 0} onClick={onSave}>{busy ? '保存中…' : mode === 'allow' ? '保存保留格式' : '保存排除格式'}</button>
{hasOwnRule ? <button className="link" disabled={busy} onClick={onClear}>恢复继承上级规则</button> : null}
</div>
<small>{mode === 'allow' ? '白名单:无扩展名的文件不受限制;目录级规则就近覆盖上级。' : '黑名单:仅排除勾选格式,其余可入库。'}</small>
{message ? <p className="success-message" role="status">{message}</p> : null}
{error ? <p className="error-message" role="alert">{error}</p> : null}
</section>
@@ -37,7 +76,9 @@ export function FilenameFilterEditorView({ label, directExtensions, inheritedExt
export function FilenameFilterEditor({ connectionID, directoryPath = '', label, onChanged }: { connectionID: string; directoryPath?: string; label: string; onChanged?: () => void }) {
const [rules, setRules] = useState<FilenameFilterRule[]>([])
const [draft, setDraft] = useState(DEFAULT_EXCLUDED_EXTENSIONS.join(', '))
const [mode, setMode] = useState<FilterMode>('allow')
const [selected, setSelected] = useState<string[]>(DEFAULT_ALLOWED_EXTENSIONS)
const [touched, setTouched] = useState(directoryPath === '')
const [busy, setBusy] = useState(false)
const [message, setMessage] = useState<string | null>(null)
const [error, setError] = useState<string | null>(null)
@@ -47,33 +88,46 @@ export function FilenameFilterEditor({ connectionID, directoryPath = '', label,
for (const rule of rules) {
if (rule.directory_path === directoryPath) continue
if (rule.directory_path === '' || directoryPath.startsWith(`${rule.directory_path}/`)) {
for (const extension of rule.excluded_extensions) values.add(extension)
for (const extension of rule.excluded_extensions) values.add(`${extension}:${rule.mode || 'exclude'}`)
}
}
return [...values].sort()
return values
}, [directoryPath, rules])
const options = useMemo(() => [...new Set([...ALL_OPTIONS, ...selected])].sort(), [selected])
const load = async (signal?: AbortSignal) => {
const next = await fetchFilenameFilters(connectionID, signal)
setRules(next)
const current = next.find((rule) => rule.directory_path === directoryPath)
setDraft(current?.excluded_extensions.join(', ') || DEFAULT_EXCLUDED_EXTENSIONS.join(', '))
if (current) {
setMode(current.mode === 'exclude' ? 'exclude' : 'allow')
setSelected(current.excluded_extensions)
} else if (!touched) {
setMode('allow')
setSelected(directoryPath === '' ? DEFAULT_ALLOWED_EXTENSIONS : [...new Set([...(inheritedHas('allow') ? inheritedValues('allow') : DEFAULT_ALLOWED_EXTENSIONS)])])
}
}
const inheritedHas = (want: string) => [...inherited].some((value) => value.endsWith(`:${want}`))
const inheritedValues = (want: string) => [...inherited].filter((value) => value.endsWith(`:${want}`)).map((value) => value.split(':')[0])
useEffect(() => {
const controller = new AbortController()
void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件名筛选失败') })
void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件格式筛选失败') })
return () => controller.abort()
}, [connectionID, directoryPath])
const toggle = (extension: string) => {
setTouched(true)
setSelected((previous) => previous.includes(extension) ? previous.filter((value) => value !== extension) : [...previous, extension].sort())
}
const save = async () => {
const extensions = parseExtensionDraft(draft)
setBusy(true); setError(null); setMessage(null)
try {
await saveFilenameFilter(connectionID, directoryPath, extensions)
await saveFilenameFilter(connectionID, directoryPath, mode, selected)
await load()
setMessage(`已排除 ${extensions.map((value) => `.${value}`).join('、')};目录、差异、取件和入库立即生效。`)
setMessage(mode === 'allow' ? `已保留 ${selected.map((value) => `.${value}`).join('、')};其余格式不再取件入库。` : `已排除 ${selected.map((value) => `.${value}`).join('、')}。`)
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件名筛选失败') }
} catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件格式筛选失败') }
finally { setBusy(false) }
}
const clear = async () => {
@@ -81,10 +135,10 @@ export function FilenameFilterEditor({ connectionID, directoryPath = '', label,
try {
await clearFilenameFilter(connectionID, directoryPath)
await load()
setMessage('已移除本级筛选,恢复继承上级规则。')
setMessage('已移除本级规则,恢复继承上级。')
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件名筛选失败') }
} catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件格式筛选失败') }
finally { setBusy(false) }
}
return <FilenameFilterEditorView label={label} directExtensions={direct?.excluded_extensions || []} inheritedExtensions={inherited} draft={draft} busy={busy} message={message} error={error} onDraft={setDraft} onSave={() => void save()} onClear={() => void clear()} />
return <FilenameFilterEditorView label={label} mode={mode} onModeChange={(next) => { setTouched(true); setMode(next) }} options={options} selected={selected} onToggle={toggle} busy={busy} hasOwnRule={Boolean(direct)} message={message} error={error} onSave={() => void save()} onClear={() => void clear()} />
}
+15 -3
View File
@@ -3,6 +3,7 @@ import { newOperationID } from './operationKey'
export interface FilenameFilterRule {
connection_id: string
directory_path: string
mode: 'allow' | 'exclude'
excluded_extensions: string[]
updated_by: string
updated_at: string
@@ -24,10 +25,10 @@ export async function fetchFilenameFilters(connectionID: string, signal?: AbortS
})
}
export async function saveFilenameFilter(connectionID: string, directoryPath: string, extensions: string[]): Promise<FilenameFilterRule> {
export async function saveFilenameFilter(connectionID: string, directoryPath: string, mode: 'allow' | 'exclude', extensions: string[]): Promise<FilenameFilterRule> {
const response = await fetch('/api/v1/filename-filters', {
method: 'PUT', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, excluded_extensions: extensions }),
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, mode, extensions }),
})
if (!response.ok) throw new Error(`保存文件名筛选失败(HTTP ${response.status})`)
const envelope = await response.json() as { data?: FilenameFilterRule }
@@ -44,7 +45,8 @@ export async function clearFilenameFilter(connectionID: string, directoryPath: s
}
// Source-wide default exclusions: archives/media/executables/CAD/design
// binaries have no RAG value for the support knowledge bases.
// binaries have no RAG value for the support knowledge bases. Kept for the
// legacy "exclude" mode editor prefill.
export const DEFAULT_EXCLUDED_EXTENSIONS = [
"zip", "rar", "7z", "gz", "tgz", "tar", "xz", "iso", "crdownload",
"mp4", "avi", "mkv", "mov", "wmv", "m4v", "mp3", "wav", "flac",
@@ -54,6 +56,16 @@ export const DEFAULT_EXCLUDED_EXTENSIONS = [
"dlink", "atomproject", "amind", "appt", "rdk", "vcmx", "swf", "dav",
].sort()
// Whitelist defaults for the "keep selected" mode: documents, text/scripts
// and images that a RAG knowledge base can actually parse and cite.
export const DEFAULT_ALLOWED_EXTENSIONS = [
"adoc", "able", "axls",
"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "csv",
"txt", "md", "json", "xml", "html", "yaml", "conf", "config", "ini", "log",
"lua", "py", "sh", "bat", "qml",
"png", "jpg", "jpeg", "gif", "bmp", "webp", "heic", "tif",
].sort()
export function parseExtensionDraft(value: string): string[] {
return [...new Set(value.split(/[,,\s]+/).map((item) => item.trim().toLowerCase().replace(/^\./, '')).filter(Boolean))].sort()
}
+5
View File
@@ -40,3 +40,8 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d
.filename-filter-inherited,.filename-filter-direct{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin:6px 0}.filename-filter-inherited span,.filename-filter-direct span{font-size:10px;color:var(--ink-mute)}
.filename-filter-editor code{padding:2px 7px;border-radius:999px;background:#eaf2ff;color:#2456a6;font-family:inherit;font-size:11px}
.filename-filter-form{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:9px 0}.filename-filter-form input{min-width:220px;flex:1;padding:7px 9px;border:1px solid var(--border-strong);border-radius:7px;background:var(--bg-panel)}
.filename-filter-modes{display:flex;gap:12px;align-self:center;font-size:11px;color:var(--ink-base)}.filename-filter-modes label{display:flex;gap:4px;align-items:center;cursor:pointer}
.filename-filter-groups{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:6px 14px;margin:8px 0}.filename-filter-group{display:grid;gap:4px}.filename-filter-group>span{color:var(--ink-mute);font-size:9px}
.filename-filter-group>div{display:flex;flex-wrap:wrap;gap:4px}.filename-filter-group button{border:1px solid var(--border-base);background:var(--bg-panel);color:var(--ink-base);border-radius:999px;padding:2px 8px;font-size:10px;font-family:inherit;cursor:pointer}
.filename-filter-group button.picked{background:var(--c-run);border-color:var(--c-run);color:var(--bg-panel);font-weight:650}.filename-filter-group button.blocked{color:var(--c-fail)}
@media(max-width:900px){.filename-filter-groups{grid-template-columns:1fr 1fr}}