b3c9331966
- filename_filter_rules table (000035) with source-root and per-directory rows - additive inheritance: source default + every ancestor directory contributes - filename_is_filtered() SQL predicate applied to catalog browsing, subtree counts, change-set diffing, routing fan-out and artifact fetch manifests - fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch jobs, retires planned publications and blocks items when rules change - GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized extensions) + Settings and Catalog filter editors - failJobSQL no longer swallows the claimed row when the failure projection upserts (fixes spurious 'lease lost' retries) - DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403 from file/download fixed), header-signature downloads supported
114 lines
21 KiB
TypeScript
114 lines
21 KiB
TypeScript
import { useContext,useEffect,useState } from 'react'
|
||
import { fetchCatalogNodes,fetchCatalogRoutingCapabilities,fetchCatalogSnapshotNodes,fetchCatalogSnapshots,fetchCatalogSources,routeCatalogFiles } from './sourceCatalog'
|
||
import type { CatalogNode,CatalogNodePage,CatalogPolicy,CatalogRoutingCapabilities,CatalogSnapshot,CatalogSource } from './sourceCatalog'
|
||
import { SnapshotHistoryView } from './SnapshotHistory'
|
||
import { triggerCatalogRefresh } from './sourceCatalog'
|
||
import { fetchJobs } from './jobs'
|
||
import type { Job } from './jobs'
|
||
import { WorkspaceContextTabsSetter } from './workspaceContext'
|
||
import { FilenameFilterEditor } from './FilenameFilterEditor'
|
||
import './sourceCatalogBrowser.css'
|
||
|
||
const sourceLabels:Record<string,string>={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'}
|
||
const bytes=(value:number)=>value>=1073741824?`${(value/1073741824).toFixed(1)} GB`:value>=1048576?`${(value/1048576).toFixed(1)} MB`:value>=1024?`${(value/1024).toFixed(1)} KB`:`${value} B`
|
||
const duration=(milliseconds:number)=>milliseconds<=0?'未记录':milliseconds<1000?`${milliseconds}毫秒`:milliseconds<60000?`${Math.round(milliseconds/1000)}秒`:`${Math.floor(milliseconds/60000)}分${Math.round(milliseconds%60000/1000)}秒`
|
||
const scope=(value:string)=>value==='internal'?'对内':value==='external'?'对外':value==='both'?'对内 + 对外':value==='blocked'?'禁止导入':value==='mixed'?'混合':'未选择'
|
||
const progress=(node:CatalogNode)=>node.imported_count?`已解析 ${node.parsed_count} / 已导入 ${node.imported_count}`:'尚未导入'
|
||
|
||
interface ViewProps{
|
||
sources:CatalogSource[];selectedSource:CatalogSource|null;page:CatalogNodePage|null;loading:boolean;error:string|null
|
||
routingCapabilities?:CatalogRoutingCapabilities;routingMessage?:string;routingLoading?:boolean
|
||
refreshJob?:Job|null
|
||
onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise<boolean>
|
||
history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void}
|
||
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void;onFiltersChanged?:()=>void
|
||
}
|
||
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden,onFiltersChanged}:ViewProps){
|
||
const [selectedIDs,setSelectedIDs]=useState<string[]>([])
|
||
const [appliedPolicies,setAppliedPolicies]=useState<Record<string,CatalogPolicy>>({})
|
||
const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1)
|
||
const visibleItems=page?.items||[]
|
||
useEffect(()=>{setSelectedIDs([]);setAppliedPolicies({})},[selectedSource?.connection_id,page?.current.node_id])
|
||
useEffect(()=>{if(routingMessage)setSelectedIDs([])},[routingMessage])
|
||
const selectableNodes=visibleItems
|
||
const selectedNodeIDs=selectedIDs
|
||
const selectedCount=selectedIDs.length
|
||
const routeReady=!routingLoading
|
||
const policyEnabled=(policy:CatalogPolicy)=>policy==='blocked'||policy==='both'?(policy==='blocked'||routingCapabilities.internal_enabled&&routingCapabilities.external_enabled):policy==='internal'?routingCapabilities.internal_enabled:routingCapabilities.external_enabled
|
||
const toggleNode=(id:string)=>setSelectedIDs(current=>current.includes(id)?current.filter(value=>value!==id):[...current,id])
|
||
const applyPolicy=async(policy:CatalogPolicy)=>{if(!selectedNodeIDs.length)return;const changed=await onRoute(selectedNodeIDs,policy);if(changed)setAppliedPolicies(current=>({...current,...Object.fromEntries(selectedNodeIDs.map(id=>[id,policy]))}))}
|
||
const refreshPayload=refreshJob?.payload&&typeof refreshJob.payload==='object'?refreshJob.payload as Record<string,unknown>:{}
|
||
const refreshProgress=typeof refreshPayload.progress==='number'?refreshPayload.progress:0
|
||
const refreshMessage=typeof refreshPayload.message==='string'?refreshPayload.message:'等待后台刷新'
|
||
const refreshDirectory=typeof refreshPayload.current_directory==='string'&&refreshPayload.current_directory?refreshPayload.current_directory:''
|
||
const refreshFiles=typeof refreshPayload.scanned_files==='number'&&refreshPayload.scanned_files>0?refreshPayload.scanned_files:0
|
||
const refreshEstimate=refreshPayload.total_is_estimate===true
|
||
const stageLabels:Record<string,string>={queued:'排队中',connecting:'连接来源',scanning:'扫描目录',building:'构建快照',activating:'写入快照',completed:'完成',failed:'失败'}
|
||
const refreshStageLabel=typeof refreshPayload.stage==='string'?stageLabels[refreshPayload.stage]||refreshPayload.stage:'排队中'
|
||
const pillSource=typeof refreshPayload.source_name==='string'?refreshPayload.source_name:''
|
||
const refreshPillSource=pillSource||selectedSource?.name||''
|
||
const refreshActive=refreshJob?.state==='pending'||refreshJob?.state==='running'||refreshJob?.state==='retry'
|
||
return <main className="page-shell catalog-browser">
|
||
<header className="hero compact"><div><h1>来源与目录</h1><p className="subtitle">各个知识源头的资料总览与标注</p></div><button onClick={onRefresh} disabled={loading||refreshActive}>{refreshActive?'实时刷新中…':'实时刷新目录'}</button></header>
|
||
{error?<div className="error-banner" role="alert">{error}</div>:null}
|
||
{refreshJob&&refreshActive&&refreshHidden?<button type="button" className="catalog-refresh-pill" aria-label="刷新进行中,点击展开" onClick={onToggleRefreshHidden}><span className={`pill-dot ${refreshJob.state}`} aria-hidden="true"/>后台刷新中 · {refreshPillSource} · {refreshProgress}%{refreshEstimate?'(估算)':''}</button>:null}
|
||
{refreshJob&&!(refreshActive&&refreshHidden)?<><section className={`catalog-refresh-progress ${refreshJob.state}`} aria-label="实时快照进度"><div><strong>{refreshActive?'实时刷新中':refreshJob.state==='completed'?'实时快照已完成':'实时刷新失败'}</strong><span>{selectedSource?selectedSource.name+' · ':''}{refreshStageLabel}</span><span>{refreshMessage}</span></div><progress max="100" value={refreshProgress}/><small>{refreshProgress}%{refreshEstimate?'(估算)':''} · 后台运行,可以切换页面</small>{refreshActive&&onToggleRefreshHidden?<button type="button" className="catalog-refresh-hide" onClick={onToggleRefreshHidden}>隐藏到后台</button>:null}</section>{refreshActive&&(refreshDirectory||refreshFiles)?<div className="catalog-refresh-detail" aria-label="刷新明细">{refreshDirectory?<span>当前目录:{refreshDirectory}</span>:null}{refreshFiles?<span>已发现 {refreshFiles} 份文件</span>:null}</div>:null}{refreshJob.state==='failed'?<p className="catalog-refresh-failed-note" role="status">刷新失败;旧快照仍然保留,目录浏览与标注不受影响。</p>:null}</>:null}
|
||
<div className="catalog-layout">
|
||
<aside className="catalog-sources" aria-label="内容来源">
|
||
<h2>内容来源</h2>
|
||
{sources.map(source=><button key={source.connection_id} className={selectedSource?.connection_id===source.connection_id?'catalog-source active':'catalog-source'} onClick={()=>onSelectSource(source)}><span><b>{source.name}</b><small>{sourceLabels[source.source_type]} · 本地只读快照</small></span><strong>{source.file_count}</strong></button>)}
|
||
{!loading&&!error&&!sources.length?<div className="empty-state">尚无来源目录。</div>:null}
|
||
</aside>
|
||
<section className="catalog-content">
|
||
{selectedSource&&page?<>
|
||
<div className="catalog-pathbar" aria-label="当前目录路径">
|
||
<button type="button" className="catalog-back-button" aria-label="返回上一级" disabled={page.breadcrumbs.length<2} onClick={onBack}>←</button>
|
||
<nav aria-label="目录面包屑">{page.breadcrumbs.map((crumb,index)=><span key={crumb.node_id}>{index>0?<i aria-hidden="true">/</i>:null}<button type="button" className={index===page.breadcrumbs.length-1?'current':''} disabled={index===page.breadcrumbs.length-1} onClick={()=>onNavigateTo(crumb.node_id)}>{crumb.name}</button></span>)}</nav>
|
||
</div>
|
||
<div className="catalog-summary"><div><strong>{page.current.name}</strong><small>{selectedSource.snapshot_only?'本地只读快照,不代表实时连接':'实时目录'}</small></div><span>{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}</span><span>含所有子目录文件 {page.current.subtree_file_count}</span><span>总大小 {bytes(page.current.subtree_bytes)}</span></div>
|
||
<FilenameFilterEditor connectionID={selectedSource.connection_id} directoryPath={page.breadcrumbs.slice(1).map((item)=>item.name).join('/')} label={atSourceRoot?'整个来源':'当前目录'} onChanged={onFiltersChanged} />
|
||
|
||
{visibleItems.length?<section className="catalog-routing">
|
||
<div className="routing-actions">
|
||
<label className="routing-check"><input type="checkbox" checked={selectableNodes.length>0&&selectableNodes.every(node=>selectedIDs.includes(node.node_id))} disabled={!selectableNodes.length} onChange={event=>setSelectedIDs(event.target.checked?selectableNodes.map(node=>node.node_id):[])} />全选当前页</label>
|
||
<div className="catalog-policy-actions"><button className="internal-policy-button" disabled={!routeReady||!selectedNodeIDs.length||!policyEnabled('internal')} onClick={()=>void applyPolicy('internal')}>对内</button><button className="external-policy-button" disabled={!routeReady||!selectedNodeIDs.length||!policyEnabled('external')} onClick={()=>void applyPolicy('external')}>对外</button><div className="blocked-policy-stack"><button className="blocked-policy-button" disabled={!routeReady||!selectedNodeIDs.length} onClick={()=>void applyPolicy('blocked')}>禁止导入</button><small className="selected-count">已选 {selectedCount} 项</small></div></div>
|
||
</div>
|
||
<small className="blocked-policy-note">禁止导入即生效:待入库条目自动撤销;仍在知识库里的文件会出现在“解析监控 → 需下架”,在知识清单停用检索,不删除来源文件,也不操作生产知识库。</small>
|
||
{routingMessage?<p role="status">{routingMessage}</p>:null}
|
||
</section>:null}
|
||
{visibleItems.length?<div className="catalog-table-wrap"><table className="catalog-table"><thead><tr><th>名称</th><th>类型</th><th>处理策略</th><th>文件数量</th><th>导入与解析</th><th>大小</th><th>更新时间</th></tr></thead><tbody>{visibleItems.map(node=>{const visibility=appliedPolicies[node.node_id]||node.visibility;const size=node.node_type==='folder'?(node.subtree_bytes>0?`已知合计 ${bytes(node.subtree_bytes)}`:'大小未提供'):typeof node.size_bytes==='number'?bytes(node.size_bytes):'大小未提供';return <tr key={node.node_id}><td>{node.node_type==='folder'?<div className="catalog-node-control"><input type="checkbox" aria-label={`选择${node.name}`} checked={selectedIDs.includes(node.node_id)} onChange={()=>toggleNode(node.node_id)} /><button className="catalog-name" onClick={()=>onOpenFolder(node)}><span aria-hidden="true">📁</span>{node.name}</button></div>:<label className="catalog-file"><input type="checkbox" aria-label={`选择${node.name}`} checked={selectedIDs.includes(node.node_id)} onChange={()=>toggleNode(node.node_id)} /><span aria-hidden="true">📄</span>{node.name}</label>}</td><td>{node.node_type==='folder'?'文件夹':'文件'}</td><td><div className="catalog-policy-cell"><span className={`scope-badge ${visibility}`}>{scope(visibility)}</span></div></td><td>{node.node_type==='folder'?`${node.subtree_file_count} 份`:'—'}</td><td><span>{progress(node)}</span>{node.active_count?<small>处理中 {node.active_count}</small>:null}{node.failed_count?<small className="problem-text">失败 {node.failed_count}</small>:null}</td><td>{size}</td><td>{node.modified_at?new Date(node.modified_at).toLocaleString('zh-CN',{hour12:false}):'—'}</td></tr>})}</tbody></table></div>:<div className="empty-state">当前目录为空。</div>}
|
||
{page.total>page.limit?<div className="pagination"><span>第 {Math.floor(page.offset/page.limit)+1} / {Math.max(1,Math.ceil(page.total/page.limit))} 页</span><div><button disabled={page.offset===0||loading} onClick={()=>onPageChange(Math.max(0,page.offset-page.limit))}>上一页</button><button disabled={page.offset+page.limit>=page.total||loading} onClick={()=>onPageChange(page.offset+page.limit)}>下一页</button></div></div>:null}
|
||
<div className="snapshot-timestamp"><span>快照就绪</span><span>快照时间:{new Date(selectedSource.observed_at).toLocaleString('zh-CN',{hour12:false})}</span><span>上次刷新耗时 {duration(selectedSource.last_refresh_duration_ms)}</span>{history?<button type="button" className="catalog-history-button" onClick={history.onOpen}>{history.open?'返回当前目录':'快照历史'}</button>:null}</div>
|
||
{history&&history.open?<SnapshotHistoryView snapshots={history.snapshots} active={history.active} page={history.page} loading={history.loading} error={history.error} onClose={history.onClose} onOpenSnapshot={history.onOpenSnapshot} onOpenFolder={history.onOpenFolder} />:null}
|
||
</>:loading?<div className="empty-state">正在读取第一层目录…</div>:!error?<div className="empty-state">请选择一个内容来源。</div>:null}
|
||
</section>
|
||
</div>
|
||
</main>
|
||
}
|
||
|
||
export function SourceCatalogBrowser(){
|
||
const [sources,setSources]=useState<CatalogSource[]>([]);const [selected,setSelected]=useState<CatalogSource|null>(null);const [parentID,setParentID]=useState('');const [page,setPage]=useState<CatalogNodePage|null>(null);const [offset,setOffset]=useState(0);const [revision,setRevision]=useState(0);const [loading,setLoading]=useState(true);const [error,setError]=useState<string|null>(null);const setContextTabs=useContext(WorkspaceContextTabsSetter)
|
||
const [routingCapabilities,setRoutingCapabilities]=useState<CatalogRoutingCapabilities>({internal_enabled:false,external_enabled:false});const [routingMessage,setRoutingMessage]=useState('');const [routingLoading,setRoutingLoading]=useState(false)
|
||
const [refreshJob,setRefreshJob]=useState<Job|null>(null)
|
||
const [historySnapshots,setHistorySnapshots]=useState<CatalogSnapshot[]>([]);const [historyActive,setHistoryActive]=useState<CatalogSnapshot|null>(null);const [historyPage,setHistoryPage]=useState<CatalogNodePage|null>(null);const [historyLoading,setHistoryLoading]=useState(false);const [historyError,setHistoryError]=useState<string|null>(null);const [historyOpen,setHistoryOpen]=useState(false)
|
||
const [refreshHidden,setRefreshHidden]=useState(false)
|
||
|
||
const openHistory=async()=>{if(!selected||historyLoading)return;if(historyOpen){closeHistory();return}setHistoryOpen(true);setHistoryLoading(true);setHistoryError(null);try{const result=await fetchCatalogSnapshots(selected.connection_id,20,0);setHistorySnapshots(result.items);const head=result.items.find(item=>item.is_head)||result.items[0]||null;setHistoryActive(head);if(head)setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,head.snapshot_id,head.root_node_id));else setHistoryPage(null)}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载快照历史失败')}finally{setHistoryLoading(false)}}
|
||
const openHistorySnapshot=async(snapshot:CatalogSnapshot)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryActive(snapshot);setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,snapshot.root_node_id))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}}
|
||
const openHistoryFolder=async(snapshot:CatalogSnapshot,nodeID:string)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,nodeID))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}}
|
||
const closeHistory=()=>{setHistoryOpen(false);setHistoryActive(null);setHistoryPage(null);setHistoryError(null)}
|
||
|
||
useEffect(()=>{const controller=new AbortController();setLoading(true);void fetchCatalogSources(controller.signal).then(result=>{if(controller.signal.aborted)return;setSources(result);setSelected(current=>current&&result.find(source=>source.connection_id===current.connection_id)||result[0]||null);setParentID(current=>current||result[0]?.root_node_id||'');setError(null)}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载来源目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[revision])
|
||
useEffect(()=>{if(!selected||!parentID)return;const controller=new AbortController();setLoading(true);void fetchCatalogNodes(selected.connection_id,parentID,50,offset,controller.signal).then(result=>{if(!controller.signal.aborted){setPage(result);setError(null)}}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[selected,parentID,offset,revision])
|
||
|
||
useEffect(()=>{const controller=new AbortController();void fetchCatalogRoutingCapabilities(controller.signal).then(setRoutingCapabilities).catch(()=>setRoutingCapabilities({internal_enabled:false,external_enabled:false}));return()=>controller.abort()},[revision])
|
||
useEffect(()=>{if(!selected)return;const controller=new AbortController();void fetchJobs(controller.signal).then(items=>{if(controller.signal.aborted)return;const match=items.find(job=>job.type==='source.catalog.refresh'&&job.payload&&typeof job.payload==='object'&&(job.payload as Record<string,unknown>).connection_id===selected.connection_id);setRefreshJob(match||null)}).catch(()=>undefined);return()=>controller.abort()},[selected])
|
||
useEffect(()=>{if(!refreshJob||!['pending','running','retry'].includes(refreshJob.state))return;const timer=window.setTimeout(()=>void fetchJobs().then(items=>{const current=items.find(job=>job.id===refreshJob.id);if(!current)return;setRefreshJob(previous=>{if(previous?.state!=='completed'&¤t.state==='completed')setRevision(value=>value+1);return current})}).catch(()=>undefined),750);return()=>window.clearTimeout(timer)},[refreshJob])
|
||
useEffect(()=>{if(!selected||!page){setContextTabs([]);return}setContextTabs(page.breadcrumbs.map(item=>({key:item.node_id,label:item.name,onActivate:()=>{setParentID(item.node_id);setOffset(0)}})));return()=>setContextTabs([])},[selected,page,setContextTabs])
|
||
const selectSource=(source:CatalogSource)=>{setSelected(source);setParentID(source.root_node_id);setOffset(0);setPage(null);setRefreshJob(null);closeHistory()}
|
||
const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)}
|
||
const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise<boolean>=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,reason:`目录页策略:${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份;已记入意图 ${result.intents_recorded} 份;排队从源头取件 ${result.fetch_queued} 份;列入待入库 ${result.planned_publications} 份;标记禁止 ${result.blocked_items} 份;目录外无文件可标注 ${result.unbound_files} 份。到“解析监控 → 待入库”点开始处理即可。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}}
|
||
const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}}
|
||
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} onFiltersChanged={()=>setRevision(value=>value+1)} />
|
||
}
|