姚章浩
1283f65a95
用户反馈：加了钉盘来源后「来源与目录」里看不到。根因不是丢数据，而是列表只列"已经有快照" 的来源 —— 首次扫描要几分钟到几十分钟（钉盘空间 2900+ 个文件夹），这段时间页面上什么都没有。 - 新 Store 方法 ListCatalogScanProgress：已启用 + 还没有 catalog head + 有在跑/排队的 source.catalog.refresh（三条件同时成立）。进度直接取 worker 写在任务载荷里的同一份计数 （progress/message/scanned_directories/discovered_directories/scanned_files），不另立口径。 - /api/v1/source-catalog/sources 在既有列表后追加这些行（snapshot_status="scanning" + scan_progress/scan_message）。走窄接口按需接线：未接线时安静跳过，不会把旧列表弄坏。 - 前端：左列显示「首次扫描中 44%」，选中后是进度面板（百分比 + 进度条 + 当前文案 + 已发现文件数），并说明"扫完自动出现目录"；有来源在扫描时每 3 秒自动刷新列表， 扫完那一行自动变成「本地只读快照」。扫描中不拉目录节点、也不显示标注/格式入口。 - 解析器保持严格：snapshot_status 只接受 ready/scanning，scanning 必须带 scan_progress， 否则整包拒绝 —— 不给"假装 0%"的机会。 测试：postgres 集成测试（真库）覆盖四种不该出现的来源（停用、已有快照、任务类型不符、 历史上已完成的那次任务）；前端 parser + 渲染各一条；Go 全绿、前端 192 条 + tsc 干净。 面板实测：左列「公司级营销资料管理 · 首次扫描中 44%」，主区「首次扫描中 · 44% / 正在扫描目录：已完成 1540 个文件夹 / 已发现约 4934 份文件」。
116 lines
16 KiB
TypeScript
116 lines
16 KiB
TypeScript
import type { Job } from './jobs'
|
||
import { newOperationID } from './operationKey'
|
||
|
||
export type CatalogSourceType='dingtalk'|'pp'|'alist'
|
||
export type CatalogPolicy='internal'|'external'|'both'|'blocked'
|
||
export interface CatalogSource {connection_id:string;source_type:CatalogSourceType;name:string;root_node_id:string;folder_count:number;file_count:number;observed_at:string;snapshot_only:boolean;snapshot_status:'ready'|'scanning';last_refresh_duration_ms:number;scan_progress?:number;scan_message?:string}
|
||
export interface CatalogBreadcrumb {node_id:string;name:string}
|
||
export interface CatalogNode {node_id:string;parent_node_id?:string;node_type:'folder'|'file';name:string;direct_child_count:number;subtree_file_count:number;subtree_bytes:number;size_bytes?:number;modified_at?:string;content_sha256?:string;visibility:'internal'|'external'|'both'|'blocked'|'mixed'|'unknown';imported_count:number;parsed_count:number;active_count:number;failed_count:number;routable_count:number}
|
||
export interface CatalogSnapshot {snapshot_id:string;connection_id:string;root_node_id:string;source_name:string;node_count:number;folder_count:number;file_count:number;observed_at:string;created_at:string;is_head:boolean}
|
||
export interface CatalogSnapshotPage {items:CatalogSnapshot[];total:number;limit:number;offset:number}
|
||
export interface CatalogNodePage {current:CatalogNode;breadcrumbs:CatalogBreadcrumb[];items:CatalogNode[];total:number;limit:number;offset:number}
|
||
const sourceTypes=new Set(['dingtalk','pp','alist']);const nodeTypes=new Set(['folder','file']);const visibilities=new Set(['internal','external','both','blocked','mixed','unknown'])
|
||
const record=(value:unknown):value is Record<string,unknown>=>typeof value==='object'&&value!==null
|
||
const count=(value:unknown):value is number=>typeof value==='number'&&Number.isInteger(value)&&value>=0
|
||
function parseNode(value:unknown):CatalogNode{
|
||
if(!record(value)||typeof value.node_id!=='string'||typeof value.node_type!=='string'||!nodeTypes.has(value.node_type)||typeof value.name!=='string'||!count(value.direct_child_count)||!count(value.subtree_file_count)||!count(value.subtree_bytes)||typeof value.visibility!=='string'||!visibilities.has(value.visibility)||!count(value.imported_count)||!count(value.parsed_count)||!count(value.active_count)||!count(value.failed_count)||!count(value.routable_count)||value.content_sha256!==undefined&&typeof value.content_sha256!=='string')throw new Error('目录节点响应无效')
|
||
return value as unknown as CatalogNode
|
||
}
|
||
export async function fetchCatalogSources(signal?:AbortSignal):Promise<CatalogSource[]>{
|
||
const response=await fetch('/api/v1/source-catalog/sources',{headers:{Accept:'application/json'},signal})
|
||
if(!response.ok)throw new Error(`加载来源目录失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown
|
||
if(!record(value)||!Array.isArray(value.data))throw new Error('来源目录响应无效')
|
||
return value.data.map(raw=>{
|
||
if(!record(raw)||typeof raw.connection_id!=='string'||typeof raw.source_type!=='string'||!sourceTypes.has(raw.source_type)||typeof raw.name!=='string'||typeof raw.root_node_id!=='string'||!count(raw.folder_count)||!count(raw.file_count)||typeof raw.observed_at!=='string'||raw.snapshot_only!==true||(raw.snapshot_status!=='ready'&&raw.snapshot_status!=='scanning')||!count(raw.last_refresh_duration_ms))throw new Error('来源目录响应无效')
|
||
// scanning = 已登记但首次扫描还没落地：带进度，正好让新来源立刻出现在列表里。
|
||
const scanning=raw.snapshot_status==='scanning'
|
||
if(scanning&&!count(raw.scan_progress))throw new Error('来源目录响应无效')
|
||
return {connection_id:raw.connection_id,source_type:raw.source_type as CatalogSourceType,name:raw.name,root_node_id:raw.root_node_id,folder_count:raw.folder_count,file_count:raw.file_count,observed_at:raw.observed_at,snapshot_only:true,snapshot_status:raw.snapshot_status as 'ready'|'scanning',last_refresh_duration_ms:raw.last_refresh_duration_ms,scan_progress:scanning?raw.scan_progress as number:undefined,scan_message:typeof raw.scan_message==='string'?raw.scan_message:undefined}
|
||
})
|
||
}
|
||
export async function fetchCatalogSnapshots(connectionID:string,limit=20,offset=0,signal?:AbortSignal):Promise<CatalogSnapshotPage>{
|
||
const query=new URLSearchParams({connection_id:connectionID,limit:String(limit),offset:String(offset)})
|
||
const response=await fetch(`/api/v1/source-catalog/snapshots?${query}`,{headers:{Accept:'application/json'},signal})
|
||
if(!response.ok)throw new Error(`加载快照历史失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown
|
||
if(!record(value)||!Array.isArray(value.items)||!count(value.total)||!count(value.limit)||!count(value.offset))throw new Error('快照历史响应无效')
|
||
const items=value.items.map(raw=>{if(!record(raw)||typeof raw.snapshot_id!=='string'||typeof raw.connection_id!=='string'||typeof raw.root_node_id!=='string'||typeof raw.source_name!=='string'||!count(raw.node_count)||!count(raw.folder_count)||!count(raw.file_count)||typeof raw.observed_at!=='string'||typeof raw.created_at!=='string'||typeof raw.is_head!=='boolean')throw new Error('快照历史条目无效');return raw as unknown as CatalogSnapshot})
|
||
return {items,total:value.total,limit:value.limit,offset:value.offset}
|
||
}
|
||
export async function fetchCatalogSnapshotNodes(connectionID:string,snapshotID:string,parentID:string,limit=50,offset=0,signal?:AbortSignal):Promise<CatalogNodePage>{
|
||
const query=new URLSearchParams({connection_id:connectionID,parent_id:parentID,limit:String(limit),offset:String(offset)})
|
||
const response=await fetch(`/api/v1/source-catalog/snapshots/${encodeURIComponent(snapshotID)}/nodes?${query}`,{headers:{Accept:'application/json'},signal})
|
||
if(!response.ok)throw new Error(`加载历史快照目录失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown
|
||
if(!record(value)||!Array.isArray(value.items)||!count(value.total)||!count(value.limit)||!count(value.offset))throw new Error('历史快照目录响应无效')
|
||
return {current:parseNode(value.current),breadcrumbs:Array.isArray(value.breadcrumbs)?value.breadcrumbs.map(raw=>{if(!record(raw)||typeof raw.node_id!=='string'||typeof raw.name!=='string')throw new Error('目录路径响应无效');return raw as unknown as CatalogBreadcrumb}):[],items:value.items.map(parseNode),total:value.total,limit:value.limit,offset:value.offset}
|
||
}
|
||
export interface ConfirmChangeSetResult{change_set_id:string;confirmed_items:number;uploads_created:number;uploads_existing:number;removals_requested:number;skipped_no_artifact:number;skipped_no_target:number;no_change_ignored:number}
|
||
export async function confirmCatalogChangeSet(changeSetID:string,itemSeqs:number[],reason:string):Promise<{result:ConfirmChangeSetResult;operation:string}>{
|
||
const response=await fetch(`/api/v1/source-catalog/change-sets/${encodeURIComponent(changeSetID)}/confirm`,{method:'POST',headers:{Accept:'application/json','Content-Type':'application/json','Idempotency-Key':newOperationID()},body:JSON.stringify({item_seqs:itemSeqs,reason})})
|
||
if(!response.ok)throw new Error(response.status===409?'操作标识与另一请求冲突，请刷新后重试':`确认快照差异失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown;if(!record(value)||!record(value.data)||!count(value.data.confirmed_items)||!count(value.data.uploads_created)||!count(value.data.uploads_existing)||!count(value.data.removals_requested)||!count(value.data.skipped_no_artifact)||!count(value.data.skipped_no_target)||!count(value.data.no_change_ignored)||typeof value.operation!=='string')throw new Error('确认快照差异响应无效')
|
||
return {result:value.data as unknown as ConfirmChangeSetResult,operation:value.operation}
|
||
}
|
||
export async function fetchCatalogNodes(connectionID:string,parentID:string,limit=50,offset=0,signal?:AbortSignal):Promise<CatalogNodePage>{
|
||
const query=new URLSearchParams({connection_id:connectionID,parent_id:parentID,limit:String(limit),offset:String(offset)})
|
||
const response=await fetch(`/api/v1/source-catalog/nodes?${query}`,{headers:{Accept:'application/json'},signal})
|
||
if(!response.ok)throw new Error(response.status===404?'目录已更新，请返回来源首页':`加载目录失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown
|
||
if(!record(value)||!Array.isArray(value.breadcrumbs)||!Array.isArray(value.items)||!count(value.total)||!count(value.limit)||!count(value.offset))throw new Error('目录列表响应无效')
|
||
const breadcrumbs=value.breadcrumbs.map(raw=>{if(!record(raw)||typeof raw.node_id!=='string'||typeof raw.name!=='string')throw new Error('目录路径响应无效');return raw as unknown as CatalogBreadcrumb})
|
||
return {current:parseNode(value.current),breadcrumbs,items:value.items.map(parseNode),total:value.total,limit:value.limit,offset:value.offset}
|
||
}
|
||
|
||
export async function fetchAllCatalogFolders(connectionID:string,parentID:string,signal?:AbortSignal):Promise<CatalogNode[]>{
|
||
const folders:CatalogNode[]=[]
|
||
const seen=new Set<string>()
|
||
let offset=0
|
||
while(true){
|
||
const page=await fetchCatalogNodes(connectionID,parentID,100,offset,signal)
|
||
for(const node of page.items){
|
||
if(node.node_type==='folder'&&!seen.has(node.node_id)){seen.add(node.node_id);folders.push(node)}
|
||
}
|
||
const next=page.offset+page.limit
|
||
if(next>=page.total)return folders
|
||
if(page.limit<=0||next<=offset)throw new Error('目录分页游标无效')
|
||
offset=next
|
||
}
|
||
}
|
||
|
||
export interface CatalogRoutingCapabilities{internal_enabled:boolean;external_enabled:boolean}
|
||
export interface CatalogRouteInput{connection_id:string;node_ids:string[];policy:CatalogPolicy;reason:string}
|
||
export interface CatalogRouteResult{selected_files:number;intents_recorded:number;fetch_queued:number;planned_publications:number;blocked_items:number;unbound_files:number}
|
||
|
||
export async function fetchCatalogRoutingCapabilities(signal?:AbortSignal):Promise<CatalogRoutingCapabilities>{
|
||
const response=await fetch('/api/v1/source-catalog/routing-capabilities',{headers:{Accept:'application/json'},signal});if(!response.ok)throw new Error(`加载分流配置失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown;if(!record(value)||!record(value.data)||typeof value.data.internal_enabled!=='boolean'||typeof value.data.external_enabled!=='boolean')throw new Error('分流配置响应无效')
|
||
return value.data as unknown as CatalogRoutingCapabilities
|
||
}
|
||
export async function routeCatalogFiles(input:CatalogRouteInput):Promise<CatalogRouteResult>{
|
||
const response=await fetch('/api/v1/source-catalog/route',{method:'POST',headers:{Accept:'application/json','Content-Type':'application/json','Idempotency-Key':newOperationID()},body:JSON.stringify(input)});if(!response.ok)throw new Error(response.status===503?'对内/对外知识库尚未配置':`处理策略保存失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown;if(!record(value)||!record(value.data)||!count(value.data.selected_files)||!count(value.data.intents_recorded)||!count(value.data.fetch_queued)||!count(value.data.planned_publications)||!count(value.data.blocked_items)||!count(value.data.unbound_files))throw new Error('文件处理响应无效')
|
||
return value.data as unknown as CatalogRouteResult
|
||
}
|
||
export async function triggerCatalogRefresh(connectionID:string):Promise<Job>{
|
||
const response=await fetch(`/api/v1/source-catalog/sources/${encodeURIComponent(connectionID)}/refresh`,{method:'POST',headers:{Accept:'application/json','Idempotency-Key':newOperationID()}});if(!response.ok)throw new Error(`启动实时刷新失败（HTTP ${response.status}）`)
|
||
const value=await response.json() as unknown;if(!record(value)||!record(value.data)||typeof value.data.id!=='string'||value.data.type!=='source.catalog.refresh')throw new Error('实时刷新任务响应无效')
|
||
return value.data as unknown as Job
|
||
}
|
||
|
||
export type CatalogChangeAction='add'|'update'|'move'|'move_scope'|'delete'|'no_change'
|
||
export interface CatalogChangeItem{action:CatalogChangeAction;title:string;old_path:string;new_path:string;old_size_bytes?:number;new_size_bytes?:number;old_modified_at?:string;new_modified_at?:string;old_content_sha256?:string;new_content_sha256?:string;reason_code:string;seq:number;old_scope?:string;new_scope?:string;old_version_id?:string;new_version_id?:string}
|
||
export interface CatalogChangeSetDetail{id:string;connection_id:string;source_name:string;state:'planned'|'confirmed'|'applying'|'applied'|'cancelled'|'failed';add_count:number;update_count:number;move_count:number;move_scope_count:number;delete_count:number;no_change_count:number;created_at:string;items:CatalogChangeItem[];total:number;limit:number;offset:number}
|
||
const changeActions=new Set<CatalogChangeAction>(['add','update','move','move_scope','delete','no_change'])
|
||
export async function fetchLatestCatalogChangeSet(connectionID:string,action:CatalogChangeAction|'',limit=20,offset=0,signal?:AbortSignal):Promise<CatalogChangeSetDetail>{
|
||
const query=new URLSearchParams({connection_id:connectionID});if(action)query.set('action',action);query.set('limit',String(limit));query.set('offset',String(offset))
|
||
const response=await fetch(`/api/v1/source-catalog/change-sets/latest?${query}`,{headers:{Accept:'application/json'},signal})
|
||
if(!response.ok)throw new Error(response.status===404?'当前来源尚无快照差异':`加载快照差异失败（HTTP ${response.status}）`)
|
||
const envelope=await response.json() as unknown;if(!record(envelope)||!record(envelope.data))throw new Error('快照差异响应无效')
|
||
const raw=envelope.data
|
||
if(typeof raw.id!=='string'||typeof raw.connection_id!=='string'||typeof raw.source_name!=='string'||typeof raw.state!=='string'||!['planned','confirmed','applying','applied','cancelled','failed'].includes(raw.state)||!count(raw.add_count)||!count(raw.update_count)||!count(raw.move_count)||!count(raw.move_scope_count)||!count(raw.delete_count)||!count(raw.no_change_count)||typeof raw.created_at!=='string'||!Array.isArray(raw.items)||!count(raw.total)||!count(raw.limit)||!count(raw.offset))throw new Error('快照差异响应无效')
|
||
const items=raw.items.map(item=>{if(!record(item)||typeof item.action!=='string'||!changeActions.has(item.action as CatalogChangeAction)||typeof item.title!=='string'||item.old_path!==undefined&&typeof item.old_path!=='string'||item.new_path!==undefined&&typeof item.new_path!=='string'||item.old_content_sha256!==undefined&&typeof item.old_content_sha256!=='string'||item.new_content_sha256!==undefined&&typeof item.new_content_sha256!=='string'||item.old_scope!==undefined&&typeof item.old_scope!=='string'||item.new_scope!==undefined&&typeof item.new_scope!=='string'||item.old_version_id!==undefined&&typeof item.old_version_id!=='string'||item.new_version_id!==undefined&&typeof item.new_version_id!=='string'||typeof item.reason_code!=='string'||!count(item.seq))throw new Error('快照差异条目无效');return {action:item.action as CatalogChangeAction,title:item.title,old_path:typeof item.old_path==='string'?item.old_path:'',new_path:typeof item.new_path==='string'?item.new_path:'',...(typeof item.old_size_bytes==='number'?{old_size_bytes:item.old_size_bytes}:{}),...(typeof item.new_size_bytes==='number'?{new_size_bytes:item.new_size_bytes}:{}),...(typeof item.old_modified_at==='string'?{old_modified_at:item.old_modified_at}:{}),...(typeof item.new_modified_at==='string'?{new_modified_at:item.new_modified_at}:{}),...(typeof item.old_content_sha256==='string'&&item.old_content_sha256?{old_content_sha256:item.old_content_sha256}:{}),...(typeof item.new_content_sha256==='string'&&item.new_content_sha256?{new_content_sha256:item.new_content_sha256}:{}),...(typeof item.old_scope==='string'&&item.old_scope?{old_scope:item.old_scope}:{}),...(typeof item.new_scope==='string'&&item.new_scope?{new_scope:item.new_scope}:{}),...(typeof item.old_version_id==='string'&&item.old_version_id?{old_version_id:item.old_version_id}:{}),...(typeof item.new_version_id==='string'&&item.new_version_id?{new_version_id:item.new_version_id}:{}),reason_code:item.reason_code,seq:item.seq}})
|
||
return {id:raw.id,connection_id:raw.connection_id,source_name:raw.source_name,state:raw.state as CatalogChangeSetDetail['state'],add_count:raw.add_count,update_count:raw.update_count,move_count:raw.move_count,move_scope_count:raw.move_scope_count,delete_count:raw.delete_count,no_change_count:raw.no_change_count,created_at:raw.created_at,items,total:raw.total,limit:raw.limit,offset:raw.offset}
|
||
}
|