feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
+26
View File
@@ -0,0 +1,26 @@
import { renderToStaticMarkup } from 'react-dom/server'
import { describe, expect, it } from 'vitest'
import { FilenameFilterEditorView } from './FilenameFilterEditor'
describe('FilenameFilterEditorView', () => {
it('shows source exclusions and inherited directory exclusions', () => {
const html = renderToStaticMarkup(<FilenameFilterEditorView
label="当前目录"
directExtensions={['mp4']}
inheritedExtensions={['rar', 'zip']}
draft="mp4"
busy={false}
message={null}
error={null}
onDraft={() => undefined}
onSave={() => undefined}
onClear={() => undefined}
/>)
expect(html).toContain('文件名筛选')
expect(html).toContain('.zip')
expect(html).toContain('.rar')
expect(html).toContain('.mp4')
expect(html).toContain('继承排除')
})
})
+90
View File
@@ -0,0 +1,90 @@
import { useEffect, useMemo, useState } from 'react'
import { clearFilenameFilter, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters'
import type { FilenameFilterRule } from './filenameFilters'
import './systemSettings.css'
interface ViewProps {
label: string
directExtensions: string[]
inheritedExtensions: string[]
draft: string
busy: boolean
message: string | null
error: string | null
onDraft: (value: string) => void
onSave: () => void
onClear: () => void
}
const tags = (values: string[]) => values.map((value) => <code key={value}>.{value}</code>)
export function FilenameFilterEditorView({ label, directExtensions, inheritedExtensions, draft, busy, message, error, onDraft, onSave, onClear }: ViewProps) {
return <section className="filename-filter-editor" aria-label={`${label}文件名筛选`}>
<div className="filename-filter-heading"><div><strong>文件名筛选</strong><small>{label} · 按扩展名排除,大小写不敏感</small></div></div>
{inheritedExtensions.length ? <div className="filename-filter-inherited"><span>继承排除</span>{tags(inheritedExtensions)}</div> : null}
{directExtensions.length ? <div className="filename-filter-direct"><span>本级排除</span>{tags(directExtensions)}</div> : <small>本级尚未设置;只继承上级规则。</small>}
<div className="filename-filter-form">
<input value={draft} placeholder="zip, rar, mp4" onChange={(event) => onDraft(event.target.value)} aria-label={`${label}排除扩展名`} />
<button className="secondary" disabled={busy || parseExtensionDraft(draft).length === 0} onClick={onSave}>{busy ? '保存中…' : '保存筛选'}</button>
{directExtensions.length ? <button className="link" disabled={busy} onClick={onClear}>移除本级筛选</button> : null}
</div>
<small>保存后立即停止匹配文件的新取件与入库;原始来源文件和快照审计记录保留。</small>
{message ? <p className="success-message" role="status">{message}</p> : null}
{error ? <p className="error-message" role="alert">{error}</p> : null}
</section>
}
export function FilenameFilterEditor({ connectionID, directoryPath = '', label, onChanged }: { connectionID: string; directoryPath?: string; label: string; onChanged?: () => void }) {
const [rules, setRules] = useState<FilenameFilterRule[]>([])
const [draft, setDraft] = useState('zip, rar, mp4')
const [busy, setBusy] = useState(false)
const [message, setMessage] = useState<string | null>(null)
const [error, setError] = useState<string | null>(null)
const direct = rules.find((rule) => rule.directory_path === directoryPath)
const inherited = useMemo(() => {
const values = new Set<string>()
for (const rule of rules) {
if (rule.directory_path === directoryPath) continue
if (rule.directory_path === '' || directoryPath.startsWith(`${rule.directory_path}/`)) {
for (const extension of rule.excluded_extensions) values.add(extension)
}
}
return [...values].sort()
}, [directoryPath, rules])
const load = async (signal?: AbortSignal) => {
const next = await fetchFilenameFilters(connectionID, signal)
setRules(next)
const current = next.find((rule) => rule.directory_path === directoryPath)
setDraft(current?.excluded_extensions.join(', ') || 'zip, rar, mp4')
}
useEffect(() => {
const controller = new AbortController()
void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件名筛选失败') })
return () => controller.abort()
}, [connectionID, directoryPath])
const save = async () => {
const extensions = parseExtensionDraft(draft)
setBusy(true); setError(null); setMessage(null)
try {
await saveFilenameFilter(connectionID, directoryPath, extensions)
await load()
setMessage(`已排除 ${extensions.map((value) => `.${value}`).join('、')};目录、差异、取件和入库立即生效。`)
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件名筛选失败') }
finally { setBusy(false) }
}
const clear = async () => {
setBusy(true); setError(null); setMessage(null)
try {
await clearFilenameFilter(connectionID, directoryPath)
await load()
setMessage('已移除本级筛选,恢复继承上级规则。')
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件名筛选失败') }
finally { setBusy(false) }
}
return <FilenameFilterEditorView label={label} directExtensions={direct?.excluded_extensions || []} inheritedExtensions={inherited} draft={draft} busy={busy} message={message} error={error} onDraft={setDraft} onSave={() => void save()} onClear={() => void clear()} />
}
+5 -3
View File
@@ -6,6 +6,7 @@ import { triggerCatalogRefresh } from './sourceCatalog'
import { fetchJobs } from './jobs'
import type { Job } from './jobs'
import { WorkspaceContextTabsSetter } from './workspaceContext'
import { FilenameFilterEditor } from './FilenameFilterEditor'
import './sourceCatalogBrowser.css'
const sourceLabels:Record<string,string>={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'}
@@ -20,9 +21,9 @@ interface ViewProps{
refreshJob?:Job|null
onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise<boolean>
history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void}
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void;onFiltersChanged?:()=>void
}
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden}:ViewProps){
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden,onFiltersChanged}:ViewProps){
const [selectedIDs,setSelectedIDs]=useState<string[]>([])
const [appliedPolicies,setAppliedPolicies]=useState<Record<string,CatalogPolicy>>({})
const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1)
@@ -65,6 +66,7 @@ export function SourceCatalogBrowserView({sources,selectedSource,page,loading,er
<nav aria-label="目录面包屑">{page.breadcrumbs.map((crumb,index)=><span key={crumb.node_id}>{index>0?<i aria-hidden="true">/</i>:null}<button type="button" className={index===page.breadcrumbs.length-1?'current':''} disabled={index===page.breadcrumbs.length-1} onClick={()=>onNavigateTo(crumb.node_id)}>{crumb.name}</button></span>)}</nav>
</div>
<div className="catalog-summary"><div><strong>{page.current.name}</strong><small>{selectedSource.snapshot_only?'本地只读快照,不代表实时连接':'实时目录'}</small></div><span>{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}</span><span>含所有子目录文件 {page.current.subtree_file_count}</span><span>总大小 {bytes(page.current.subtree_bytes)}</span></div>
<FilenameFilterEditor connectionID={selectedSource.connection_id} directoryPath={page.breadcrumbs.slice(1).map((item)=>item.name).join('/')} label={atSourceRoot?'整个来源':'当前目录'} onChanged={onFiltersChanged} />
{visibleItems.length?<section className="catalog-routing">
<div className="routing-actions">
@@ -107,5 +109,5 @@ export function SourceCatalogBrowser(){
const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)}
const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise<boolean>=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,reason:`目录页策略:${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份;已记入意图 ${result.intents_recorded} 份;排队从源头取件 ${result.fetch_queued} 份;列入待入库 ${result.planned_publications} 份;标记禁止 ${result.blocked_items} 份;目录外无文件可标注 ${result.unbound_files} 份。到“解析监控 → 待入库”点开始处理即可。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}}
const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}}
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} />
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} onFiltersChanged={()=>setRevision(value=>value+1)} />
}
+2
View File
@@ -3,6 +3,7 @@ import { useCallback, useEffect, useState } from 'react'
import { MembersSection } from './MembersSection'
import { DingtalkSessionControl } from './DingtalkSessionControl'
import { DingtalkSpaceRegistrar } from './DingtalkSpaceRegistrar'
import { FilenameFilterEditor } from './FilenameFilterEditor'
import { RoutingBindingsSection } from './RoutingBindingsSection'
import { useOptionalAuth } from './auth'
import { fetchMonitorOverview } from './monitor'
@@ -122,6 +123,7 @@ function SourceSettingsCard({ connection, onSourcesChanged }: { connection: Sour
<button className="danger" disabled={!canApply} onClick={() => setPendingDisable(true)}>停用来源</button></div>
: <div className="source-schedule-actions"><button className="secondary" disabled={!canApply} onClick={() => void submit(true)}>保存并启用</button></div>}
{connection.source_type === "dingtalk" && isAdmin ? <DingtalkSessionControl /> : null}
<FilenameFilterEditor connectionID={connection.id} label="整个来源" onChanged={onSourcesChanged} />
<button className="link" onClick={() => void loadHistory()}>{showHistory ? '收起修订历史' : '查看修订历史'}</button>
</div>
{message ? <p className="success-message" role="status">{message}</p> : null}
+48
View File
@@ -0,0 +1,48 @@
import { newOperationID } from './operationKey'
export interface FilenameFilterRule {
connection_id: string
directory_path: string
excluded_extensions: string[]
updated_by: string
updated_at: string
}
const record = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
export async function fetchFilenameFilters(connectionID: string, signal?: AbortSignal): Promise<FilenameFilterRule[]> {
const query = new URLSearchParams({ connection_id: connectionID })
const response = await fetch(`/api/v1/filename-filters?${query}`, { headers: { Accept: 'application/json' }, signal })
if (!response.ok) throw new Error(`加载文件名筛选失败(HTTP ${response.status})`)
const envelope = await response.json() as unknown
if (!record(envelope) || !Array.isArray(envelope.data)) throw new Error('文件名筛选响应无效')
return envelope.data.map((value) => {
if (!record(value) || typeof value.connection_id !== 'string' || typeof value.directory_path !== 'string' ||
!Array.isArray(value.excluded_extensions) || !value.excluded_extensions.every((item) => typeof item === 'string') ||
typeof value.updated_by !== 'string' || typeof value.updated_at !== 'string') throw new Error('文件名筛选条目无效')
return value as unknown as FilenameFilterRule
})
}
export async function saveFilenameFilter(connectionID: string, directoryPath: string, extensions: string[]): Promise<FilenameFilterRule> {
const response = await fetch('/api/v1/filename-filters', {
method: 'PUT', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, excluded_extensions: extensions }),
})
if (!response.ok) throw new Error(`保存文件名筛选失败(HTTP ${response.status})`)
const envelope = await response.json() as { data?: FilenameFilterRule }
if (!envelope.data) throw new Error('文件名筛选响应无效')
return envelope.data
}
export async function clearFilenameFilter(connectionID: string, directoryPath: string): Promise<void> {
const response = await fetch('/api/v1/filename-filters', {
method: 'DELETE', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath }),
})
if (!response.ok) throw new Error(`清除文件名筛选失败(HTTP ${response.status})`)
}
export function parseExtensionDraft(value: string): string[] {
return [...new Set(value.split(/[,,\s]+/).map((item) => item.trim().toLowerCase().replace(/^\./, '')).filter(Boolean))].sort()
}
+5
View File
@@ -35,3 +35,8 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d
.dingtalk-qr-loading span{width:42px;height:42px;border:4px solid #dbe4ee;border-top-color:var(--brand);border-radius:50%;animation:dingtalk-qr-spin .8s linear infinite}
@keyframes dingtalk-qr-spin{to{transform:rotate(360deg)}}
.dingtalk-qr-card .modal-actions{justify-content:center}
.filename-filter-editor{flex-basis:100%;margin-top:10px;padding:12px;border:1px solid var(--border-base);border-radius:10px;background:var(--bg-soft)}
.filename-filter-heading{display:flex;justify-content:space-between;gap:10px;margin-bottom:7px}.filename-filter-heading div{display:grid;gap:3px}.filename-filter-heading small{color:var(--ink-mute)}
.filename-filter-inherited,.filename-filter-direct{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin:6px 0}.filename-filter-inherited span,.filename-filter-direct span{font-size:10px;color:var(--ink-mute)}
.filename-filter-editor code{padding:2px 7px;border-radius:999px;background:#eaf2ff;color:#2456a6;font-family:inherit;font-size:11px}
.filename-filter-form{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:9px 0}.filename-filter-form input{min-width:220px;flex:1;padding:7px 9px;border:1px solid var(--border-strong);border-radius:7px;background:var(--bg-panel)}