Files
knowledge-governance/web/src/SourceCatalogBrowser.tsx
T
姚章浩 b3c9331966 feat(ingest): 源与目录文件名过滤及钉钉文档导出
- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
2026-09-30 10:54:07 +08:00

114 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useContext,useEffect,useState } from 'react'
import { fetchCatalogNodes,fetchCatalogRoutingCapabilities,fetchCatalogSnapshotNodes,fetchCatalogSnapshots,fetchCatalogSources,routeCatalogFiles } from './sourceCatalog'
import type { CatalogNode,CatalogNodePage,CatalogPolicy,CatalogRoutingCapabilities,CatalogSnapshot,CatalogSource } from './sourceCatalog'
import { SnapshotHistoryView } from './SnapshotHistory'
import { triggerCatalogRefresh } from './sourceCatalog'
import { fetchJobs } from './jobs'
import type { Job } from './jobs'
import { WorkspaceContextTabsSetter } from './workspaceContext'
import { FilenameFilterEditor } from './FilenameFilterEditor'
import './sourceCatalogBrowser.css'
const sourceLabels:Record<string,string>={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'}
const bytes=(value:number)=>value>=1073741824?`${(value/1073741824).toFixed(1)} GB`:value>=1048576?`${(value/1048576).toFixed(1)} MB`:value>=1024?`${(value/1024).toFixed(1)} KB`:`${value} B`
const duration=(milliseconds:number)=>milliseconds<=0?'未记录':milliseconds<1000?`${milliseconds}毫秒`:milliseconds<60000?`${Math.round(milliseconds/1000)}秒`:`${Math.floor(milliseconds/60000)}分${Math.round(milliseconds%60000/1000)}秒`
const scope=(value:string)=>value==='internal'?'对内':value==='external'?'对外':value==='both'?'对内 + 对外':value==='blocked'?'禁止导入':value==='mixed'?'混合':'未选择'
const progress=(node:CatalogNode)=>node.imported_count?`已解析 ${node.parsed_count} / 已导入 ${node.imported_count}`:'尚未导入'
interface ViewProps{
sources:CatalogSource[];selectedSource:CatalogSource|null;page:CatalogNodePage|null;loading:boolean;error:string|null
routingCapabilities?:CatalogRoutingCapabilities;routingMessage?:string;routingLoading?:boolean
refreshJob?:Job|null
onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise<boolean>
history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void}
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void;onFiltersChanged?:()=>void
}
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden,onFiltersChanged}:ViewProps){
const [selectedIDs,setSelectedIDs]=useState<string[]>([])
const [appliedPolicies,setAppliedPolicies]=useState<Record<string,CatalogPolicy>>({})
const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1)
const visibleItems=page?.items||[]
useEffect(()=>{setSelectedIDs([]);setAppliedPolicies({})},[selectedSource?.connection_id,page?.current.node_id])
useEffect(()=>{if(routingMessage)setSelectedIDs([])},[routingMessage])
const selectableNodes=visibleItems
const selectedNodeIDs=selectedIDs
const selectedCount=selectedIDs.length
const routeReady=!routingLoading
const policyEnabled=(policy:CatalogPolicy)=>policy==='blocked'||policy==='both'?(policy==='blocked'||routingCapabilities.internal_enabled&&routingCapabilities.external_enabled):policy==='internal'?routingCapabilities.internal_enabled:routingCapabilities.external_enabled
const toggleNode=(id:string)=>setSelectedIDs(current=>current.includes(id)?current.filter(value=>value!==id):[...current,id])
const applyPolicy=async(policy:CatalogPolicy)=>{if(!selectedNodeIDs.length)return;const changed=await onRoute(selectedNodeIDs,policy);if(changed)setAppliedPolicies(current=>({...current,...Object.fromEntries(selectedNodeIDs.map(id=>[id,policy]))}))}
const refreshPayload=refreshJob?.payload&&typeof refreshJob.payload==='object'?refreshJob.payload as Record<string,unknown>:{}
const refreshProgress=typeof refreshPayload.progress==='number'?refreshPayload.progress:0
const refreshMessage=typeof refreshPayload.message==='string'?refreshPayload.message:'等待后台刷新'
const refreshDirectory=typeof refreshPayload.current_directory==='string'&&refreshPayload.current_directory?refreshPayload.current_directory:''
const refreshFiles=typeof refreshPayload.scanned_files==='number'&&refreshPayload.scanned_files>0?refreshPayload.scanned_files:0
const refreshEstimate=refreshPayload.total_is_estimate===true
const stageLabels:Record<string,string>={queued:'排队中',connecting:'连接来源',scanning:'扫描目录',building:'构建快照',activating:'写入快照',completed:'完成',failed:'失败'}
const refreshStageLabel=typeof refreshPayload.stage==='string'?stageLabels[refreshPayload.stage]||refreshPayload.stage:'排队中'
const pillSource=typeof refreshPayload.source_name==='string'?refreshPayload.source_name:''
const refreshPillSource=pillSource||selectedSource?.name||''
const refreshActive=refreshJob?.state==='pending'||refreshJob?.state==='running'||refreshJob?.state==='retry'
return <main className="page-shell catalog-browser">
<header className="hero compact"><div><h1>来源与目录</h1><p className="subtitle">各个知识源头的资料总览与标注</p></div><button onClick={onRefresh} disabled={loading||refreshActive}>{refreshActive?'实时刷新中…':'实时刷新目录'}</button></header>
{error?<div className="error-banner" role="alert">{error}</div>:null}
{refreshJob&&refreshActive&&refreshHidden?<button type="button" className="catalog-refresh-pill" aria-label="刷新进行中,点击展开" onClick={onToggleRefreshHidden}><span className={`pill-dot ${refreshJob.state}`} aria-hidden="true"/>后台刷新中 · {refreshPillSource} · {refreshProgress}%{refreshEstimate?'(估算)':''}</button>:null}
{refreshJob&&!(refreshActive&&refreshHidden)?<><section className={`catalog-refresh-progress ${refreshJob.state}`} aria-label="实时快照进度"><div><strong>{refreshActive?'实时刷新中':refreshJob.state==='completed'?'实时快照已完成':'实时刷新失败'}</strong><span>{selectedSource?selectedSource.name+' · ':''}{refreshStageLabel}</span><span>{refreshMessage}</span></div><progress max="100" value={refreshProgress}/><small>{refreshProgress}%{refreshEstimate?'(估算)':''} · 后台运行,可以切换页面</small>{refreshActive&&onToggleRefreshHidden?<button type="button" className="catalog-refresh-hide" onClick={onToggleRefreshHidden}>隐藏到后台</button>:null}</section>{refreshActive&&(refreshDirectory||refreshFiles)?<div className="catalog-refresh-detail" aria-label="刷新明细">{refreshDirectory?<span>当前目录:{refreshDirectory}</span>:null}{refreshFiles?<span>已发现 {refreshFiles} 份文件</span>:null}</div>:null}{refreshJob.state==='failed'?<p className="catalog-refresh-failed-note" role="status">刷新失败;旧快照仍然保留,目录浏览与标注不受影响。</p>:null}</>:null}
<div className="catalog-layout">
<aside className="catalog-sources" aria-label="内容来源">
<h2>内容来源</h2>
{sources.map(source=><button key={source.connection_id} className={selectedSource?.connection_id===source.connection_id?'catalog-source active':'catalog-source'} onClick={()=>onSelectSource(source)}><span><b>{source.name}</b><small>{sourceLabels[source.source_type]} · 本地只读快照</small></span><strong>{source.file_count}</strong></button>)}
{!loading&&!error&&!sources.length?<div className="empty-state">尚无来源目录。</div>:null}
</aside>
<section className="catalog-content">
{selectedSource&&page?<>
<div className="catalog-pathbar" aria-label="当前目录路径">
<button type="button" className="catalog-back-button" aria-label="返回上一级" disabled={page.breadcrumbs.length<2} onClick={onBack}>←</button>
<nav aria-label="目录面包屑">{page.breadcrumbs.map((crumb,index)=><span key={crumb.node_id}>{index>0?<i aria-hidden="true">/</i>:null}<button type="button" className={index===page.breadcrumbs.length-1?'current':''} disabled={index===page.breadcrumbs.length-1} onClick={()=>onNavigateTo(crumb.node_id)}>{crumb.name}</button></span>)}</nav>
</div>
<div className="catalog-summary"><div><strong>{page.current.name}</strong><small>{selectedSource.snapshot_only?'本地只读快照,不代表实时连接':'实时目录'}</small></div><span>{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}</span><span>含所有子目录文件 {page.current.subtree_file_count}</span><span>总大小 {bytes(page.current.subtree_bytes)}</span></div>
<FilenameFilterEditor connectionID={selectedSource.connection_id} directoryPath={page.breadcrumbs.slice(1).map((item)=>item.name).join('/')} label={atSourceRoot?'整个来源':'当前目录'} onChanged={onFiltersChanged} />
{visibleItems.length?<section className="catalog-routing">
<div className="routing-actions">
<label className="routing-check"><input type="checkbox" checked={selectableNodes.length>0&&selectableNodes.every(node=>selectedIDs.includes(node.node_id))} disabled={!selectableNodes.length} onChange={event=>setSelectedIDs(event.target.checked?selectableNodes.map(node=>node.node_id):[])} />全选当前页</label>
<div className="catalog-policy-actions"><button className="internal-policy-button" disabled={!routeReady||!selectedNodeIDs.length||!policyEnabled('internal')} onClick={()=>void applyPolicy('internal')}>对内</button><button className="external-policy-button" disabled={!routeReady||!selectedNodeIDs.length||!policyEnabled('external')} onClick={()=>void applyPolicy('external')}>对外</button><div className="blocked-policy-stack"><button className="blocked-policy-button" disabled={!routeReady||!selectedNodeIDs.length} onClick={()=>void applyPolicy('blocked')}>禁止导入</button><small className="selected-count">已选 {selectedCount} 项</small></div></div>
</div>
<small className="blocked-policy-note">禁止导入即生效:待入库条目自动撤销;仍在知识库里的文件会出现在“解析监控 → 需下架”,在知识清单停用检索,不删除来源文件,也不操作生产知识库。</small>
{routingMessage?<p role="status">{routingMessage}</p>:null}
</section>:null}
{visibleItems.length?<div className="catalog-table-wrap"><table className="catalog-table"><thead><tr><th>名称</th><th>类型</th><th>处理策略</th><th>文件数量</th><th>导入与解析</th><th>大小</th><th>更新时间</th></tr></thead><tbody>{visibleItems.map(node=>{const visibility=appliedPolicies[node.node_id]||node.visibility;const size=node.node_type==='folder'?(node.subtree_bytes>0?`已知合计 ${bytes(node.subtree_bytes)}`:'大小未提供'):typeof node.size_bytes==='number'?bytes(node.size_bytes):'大小未提供';return <tr key={node.node_id}><td>{node.node_type==='folder'?<div className="catalog-node-control"><input type="checkbox" aria-label={`选择${node.name}`} checked={selectedIDs.includes(node.node_id)} onChange={()=>toggleNode(node.node_id)} /><button className="catalog-name" onClick={()=>onOpenFolder(node)}><span aria-hidden="true">📁</span>{node.name}</button></div>:<label className="catalog-file"><input type="checkbox" aria-label={`选择${node.name}`} checked={selectedIDs.includes(node.node_id)} onChange={()=>toggleNode(node.node_id)} /><span aria-hidden="true">📄</span>{node.name}</label>}</td><td>{node.node_type==='folder'?'文件夹':'文件'}</td><td><div className="catalog-policy-cell"><span className={`scope-badge ${visibility}`}>{scope(visibility)}</span></div></td><td>{node.node_type==='folder'?`${node.subtree_file_count} 份`:'—'}</td><td><span>{progress(node)}</span>{node.active_count?<small>处理中 {node.active_count}</small>:null}{node.failed_count?<small className="problem-text">失败 {node.failed_count}</small>:null}</td><td>{size}</td><td>{node.modified_at?new Date(node.modified_at).toLocaleString('zh-CN',{hour12:false}):'—'}</td></tr>})}</tbody></table></div>:<div className="empty-state">当前目录为空。</div>}
{page.total>page.limit?<div className="pagination"><span>第 {Math.floor(page.offset/page.limit)+1} / {Math.max(1,Math.ceil(page.total/page.limit))} 页</span><div><button disabled={page.offset===0||loading} onClick={()=>onPageChange(Math.max(0,page.offset-page.limit))}>上一页</button><button disabled={page.offset+page.limit>=page.total||loading} onClick={()=>onPageChange(page.offset+page.limit)}>下一页</button></div></div>:null}
<div className="snapshot-timestamp"><span>快照就绪</span><span>快照时间:{new Date(selectedSource.observed_at).toLocaleString('zh-CN',{hour12:false})}</span><span>上次刷新耗时 {duration(selectedSource.last_refresh_duration_ms)}</span>{history?<button type="button" className="catalog-history-button" onClick={history.onOpen}>{history.open?'返回当前目录':'快照历史'}</button>:null}</div>
{history&&history.open?<SnapshotHistoryView snapshots={history.snapshots} active={history.active} page={history.page} loading={history.loading} error={history.error} onClose={history.onClose} onOpenSnapshot={history.onOpenSnapshot} onOpenFolder={history.onOpenFolder} />:null}
</>:loading?<div className="empty-state">正在读取第一层目录…</div>:!error?<div className="empty-state">请选择一个内容来源。</div>:null}
</section>
</div>
</main>
}
export function SourceCatalogBrowser(){
const [sources,setSources]=useState<CatalogSource[]>([]);const [selected,setSelected]=useState<CatalogSource|null>(null);const [parentID,setParentID]=useState('');const [page,setPage]=useState<CatalogNodePage|null>(null);const [offset,setOffset]=useState(0);const [revision,setRevision]=useState(0);const [loading,setLoading]=useState(true);const [error,setError]=useState<string|null>(null);const setContextTabs=useContext(WorkspaceContextTabsSetter)
const [routingCapabilities,setRoutingCapabilities]=useState<CatalogRoutingCapabilities>({internal_enabled:false,external_enabled:false});const [routingMessage,setRoutingMessage]=useState('');const [routingLoading,setRoutingLoading]=useState(false)
const [refreshJob,setRefreshJob]=useState<Job|null>(null)
const [historySnapshots,setHistorySnapshots]=useState<CatalogSnapshot[]>([]);const [historyActive,setHistoryActive]=useState<CatalogSnapshot|null>(null);const [historyPage,setHistoryPage]=useState<CatalogNodePage|null>(null);const [historyLoading,setHistoryLoading]=useState(false);const [historyError,setHistoryError]=useState<string|null>(null);const [historyOpen,setHistoryOpen]=useState(false)
const [refreshHidden,setRefreshHidden]=useState(false)
const openHistory=async()=>{if(!selected||historyLoading)return;if(historyOpen){closeHistory();return}setHistoryOpen(true);setHistoryLoading(true);setHistoryError(null);try{const result=await fetchCatalogSnapshots(selected.connection_id,20,0);setHistorySnapshots(result.items);const head=result.items.find(item=>item.is_head)||result.items[0]||null;setHistoryActive(head);if(head)setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,head.snapshot_id,head.root_node_id));else setHistoryPage(null)}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载快照历史失败')}finally{setHistoryLoading(false)}}
const openHistorySnapshot=async(snapshot:CatalogSnapshot)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryActive(snapshot);setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,snapshot.root_node_id))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}}
const openHistoryFolder=async(snapshot:CatalogSnapshot,nodeID:string)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,nodeID))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}}
const closeHistory=()=>{setHistoryOpen(false);setHistoryActive(null);setHistoryPage(null);setHistoryError(null)}
useEffect(()=>{const controller=new AbortController();setLoading(true);void fetchCatalogSources(controller.signal).then(result=>{if(controller.signal.aborted)return;setSources(result);setSelected(current=>current&&result.find(source=>source.connection_id===current.connection_id)||result[0]||null);setParentID(current=>current||result[0]?.root_node_id||'');setError(null)}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载来源目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[revision])
useEffect(()=>{if(!selected||!parentID)return;const controller=new AbortController();setLoading(true);void fetchCatalogNodes(selected.connection_id,parentID,50,offset,controller.signal).then(result=>{if(!controller.signal.aborted){setPage(result);setError(null)}}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[selected,parentID,offset,revision])
useEffect(()=>{const controller=new AbortController();void fetchCatalogRoutingCapabilities(controller.signal).then(setRoutingCapabilities).catch(()=>setRoutingCapabilities({internal_enabled:false,external_enabled:false}));return()=>controller.abort()},[revision])
useEffect(()=>{if(!selected)return;const controller=new AbortController();void fetchJobs(controller.signal).then(items=>{if(controller.signal.aborted)return;const match=items.find(job=>job.type==='source.catalog.refresh'&&job.payload&&typeof job.payload==='object'&&(job.payload as Record<string,unknown>).connection_id===selected.connection_id);setRefreshJob(match||null)}).catch(()=>undefined);return()=>controller.abort()},[selected])
useEffect(()=>{if(!refreshJob||!['pending','running','retry'].includes(refreshJob.state))return;const timer=window.setTimeout(()=>void fetchJobs().then(items=>{const current=items.find(job=>job.id===refreshJob.id);if(!current)return;setRefreshJob(previous=>{if(previous?.state!=='completed'&&current.state==='completed')setRevision(value=>value+1);return current})}).catch(()=>undefined),750);return()=>window.clearTimeout(timer)},[refreshJob])
useEffect(()=>{if(!selected||!page){setContextTabs([]);return}setContextTabs(page.breadcrumbs.map(item=>({key:item.node_id,label:item.name,onActivate:()=>{setParentID(item.node_id);setOffset(0)}})));return()=>setContextTabs([])},[selected,page,setContextTabs])
const selectSource=(source:CatalogSource)=>{setSelected(source);setParentID(source.root_node_id);setOffset(0);setPage(null);setRefreshJob(null);closeHistory()}
const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)}
const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise<boolean>=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,reason:`目录页策略:${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份;已记入意图 ${result.intents_recorded} 份;排队从源头取件 ${result.fetch_queued} 份;列入待入库 ${result.planned_publications} 份;标记禁止 ${result.blocked_items} 份;目录外无文件可标注 ${result.unbound_files} 份。到“解析监控 → 待入库”点开始处理即可。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}}
const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}}
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} onFiltersChanged={()=>setRevision(value=>value+1)} />
}