feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+100 -34
View File
@@ -11,6 +11,7 @@ import { onMounted, ref, nextTick, onUnmounted, watch, computed } from "vue";
import { downKnowledgeDetails, deleteGeneratedQuestion, getChunkByIdOnly, previewKnowledgeFile } from "@/api/knowledge-base/index";
import { MessagePlugin, DialogPlugin } from "tdesign-vue-next";
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL, hydrateProtectedFileImages, isValidURL } from '@/utils/security';
import { normalizeSpuriousTablePrefixes } from '@/utils/markdownTableNormalize';
import { openMermaidFullscreen } from '@/utils/mermaidViewer';
import { useI18n } from 'vue-i18n';
import { useAuthStore } from '@/stores/auth';
@@ -283,8 +284,15 @@ let page = 1;
let loadingChunks = false;
let pendingRequestedPage: number | null = null;
let pendingChunksBeforeLoad = 0;
let doc = null;
const CHUNK_PAGE_SIZE = 25;
/** Scroll container for the main doc drawer (not the first .t-drawer__body on the page). */
let docScrollEl: HTMLElement | null = null;
let mdContentWrap = ref()
// Drawer uses attach="body", so markdown nodes live outside mdContentWrap in the DOM.
const docMarkdownRoot = ref<HTMLElement | null>(null)
const getMarkdownRenderRoot = (): ParentNode | null =>
docMarkdownRoot.value ?? (mdContentWrap.value as ParentNode | null) ?? null
let url = ref('')
// 视图模式:chunks / merged / preview
// file 类型默认「预览」,URL / 手动创建 默认「全文」
@@ -365,18 +373,40 @@ const mergeChunks = (chunks: any[]): string => {
return merged;
};
const findDocDrawerScrollEl = (): HTMLElement | null =>
document.querySelector('.doc-main-drawer .t-drawer__body') as HTMLElement | null;
const unbindDrawerScroll = () => {
if (docScrollEl) {
docScrollEl.removeEventListener('scroll', handleDetailsScroll);
docScrollEl = null;
}
};
const bindDrawerScroll = () => {
unbindDrawerScroll();
docScrollEl = findDocDrawerScrollEl();
if (docScrollEl) {
docScrollEl.addEventListener('scroll', handleDetailsScroll, { passive: true });
}
};
onMounted(() => {
loadTraceDrawerWidth();
loadMainDrawerWidth();
window.addEventListener('resize', onTraceDrawerWindowResize, { passive: true });
nextTick(() => {
const drawers = document.getElementsByClassName('t-drawer__body');
if (drawers && drawers.length > 0) {
doc = drawers[0];
doc.addEventListener('scroll', handleDetailsScroll);
}
})
})
});
watch(() => props.visible, (visible) => {
if (visible) {
nextTick(() => {
bindDrawerScroll();
maybeLoadMoreChunks();
});
} else {
unbindDrawerScroll();
}
});
watch(() => props.details?.id, () => {
page = 1;
loadingChunks = false;
@@ -396,15 +426,16 @@ watch(() => props.details?.chunkLoading, (val) => {
pendingRequestedPage = null;
pendingChunksBeforeLoad = 0;
loadingChunks = false;
if (props.visible) {
nextTick(() => maybeLoadMoreChunks());
}
}
});
onUnmounted(() => {
window.removeEventListener('resize', onTraceDrawerWindowResize);
cleanupTraceDrawerResize();
cleanupMainDrawerResize();
if (doc) {
doc.removeEventListener('scroll', handleDetailsScroll);
}
unbindDrawerScroll();
if (audioBlobUrl.value) {
URL.revokeObjectURL(audioBlobUrl.value);
}
@@ -561,7 +592,10 @@ const loadAudioPreview = async () => {
};
const runMarkdownPostRenderPipeline = async () => {
await nextTick();
const renderRoot = mdContentWrap.value as ParentNode;
const renderRoot = getMarkdownRenderRoot();
if (!renderRoot) {
return;
}
await hydrateProtectedFileImages(renderRoot);
const images = renderRoot?.querySelectorAll?.('img.markdown-image') as NodeListOf<HTMLImageElement> | undefined;
if (images) {
@@ -576,26 +610,29 @@ const runMarkdownPostRenderPipeline = async () => {
await renderMermaidDiagrams();
};
watch(() => props.details.md, (newVal) => {
watch(() => props.details.md, () => {
runMarkdownPostRenderPipeline();
}, { immediate: true, deep: true })
}, { immediate: true, deep: true, flush: 'post' })
watch(() => viewMode.value, (mode) => {
if ((mode === 'chunks' || mode === 'merged') && props.visible) {
runMarkdownPostRenderPipeline();
if (mode === 'chunks') {
nextTick(() => maybeLoadMoreChunks());
}
}
});
}, { flush: 'post' });
watch(() => props.visible, (visible) => {
if (visible && (viewMode.value === 'chunks' || viewMode.value === 'merged')) {
runMarkdownPostRenderPipeline();
}
});
}, { flush: 'post' });
// 渲染 Mermaid 图表的函数
const renderMermaidDiagrams = async () => {
try {
const mermaidElements = mdContentWrap.value?.querySelectorAll('.mermaid');
const mermaidElements = getMarkdownRenderRoot()?.querySelectorAll('.mermaid');
console.log('[Mermaid] Found mermaid elements:', mermaidElements?.length);
if (mermaidElements && mermaidElements.length > 0) {
await mermaid.run({
@@ -624,12 +661,13 @@ const handleMermaidClick = (e: Event) => {
// 为 Mermaid 容器绑定点击全屏事件(绑定在 div 上,不是 SVG 上)
const bindMermaidClickEvents = () => {
if (!mdContentWrap.value) {
console.log('[Mermaid] mdContentWrap is null');
const renderRoot = getMarkdownRenderRoot();
if (!renderRoot) {
console.log('[Mermaid] markdown render root is null');
return;
}
// 绑定在 .mermaid div 上,而不是 SVG 上
const mermaidDivs = mdContentWrap.value.querySelectorAll('.mermaid');
const mermaidDivs = renderRoot.querySelectorAll('.mermaid');
console.log('[Mermaid] Found mermaid divs:', mermaidDivs.length);
mermaidDivs.forEach((div, index) => {
const divEl = div as HTMLElement;
@@ -663,6 +701,9 @@ const processMarkdown = (markdownText) => {
// 处理被 <p> 包裹的表格行,转换为正常的表格行,并在前后补空行
processedText = processedText.replace(/<p>\s*(\|[\s\S]*?\|)\s*<\/p>/gi, '\n$1\n');
// MarkItDown 常在表格前插入空行 + 分隔行,渲染会出现多余空行
processedText = normalizeSpuriousTablePrefixes(processedText);
// 保留表格单元格中的 <br>,不转成换行,避免打散表格;其他区域原样交给 marked 处理
// 先预处理数学定界符,再做安全预处理
@@ -683,7 +724,8 @@ const processMarkdown = (markdownText) => {
};
const handleClose = () => {
emit("closeDoc", false);
if (doc) doc.scrollTop = 0;
const scrollEl = docScrollEl || findDocDrawerScrollEl();
if (scrollEl) scrollEl.scrollTop = 0;
viewMode.value = 'merged';
};
@@ -973,19 +1015,41 @@ const downloadFile = () => {
MessagePlugin.error(t('file.downloadFailed'));
});
};
const requestNextChunkPage = () => {
if (loadingChunks || props.details?.chunkLoading) return;
const total = props.details?.total ?? 0;
const loaded = props.details?.md?.length ?? 0;
if (loaded >= total || total === 0) return;
const pageNum = Math.ceil(total / CHUNK_PAGE_SIZE);
if (page + 1 > pageNum) return;
page++;
loadingChunks = true;
pendingRequestedPage = page;
pendingChunksBeforeLoad = loaded;
emit('getDoc', page);
};
/** When the list is shorter than the drawer, scroll never fires — prefetch until scrollable or done. */
const maybeLoadMoreChunks = () => {
if (!props.visible || loadingChunks || props.details?.chunkLoading) return;
const el = docScrollEl || findDocDrawerScrollEl();
if (!el) return;
const loaded = props.details?.md?.length ?? 0;
const total = props.details?.total ?? 0;
if (loaded >= total) return;
const { scrollHeight, clientHeight } = el;
if (scrollHeight <= clientHeight + 8) {
requestNextChunkPage();
}
};
const handleDetailsScroll = () => {
if (doc && !loadingChunks) {
let pageNum = Math.ceil(props.details.total / 25);
const { scrollTop, scrollHeight, clientHeight } = doc;
if (scrollTop + clientHeight >= scrollHeight - 8) {
if (props.details.md.length < props.details.total && page + 1 <= pageNum) {
page++;
loadingChunks = true;
pendingRequestedPage = page;
pendingChunksBeforeLoad = props.details.md.length;
emit("getDoc", page);
}
}
if (loadingChunks || props.details?.chunkLoading) return;
const el = docScrollEl || findDocDrawerScrollEl();
if (!el) return;
const { scrollTop, scrollHeight, clientHeight } = el;
if (scrollTop + clientHeight >= scrollHeight - 8) {
requestNextChunkPage();
}
};
</script>
@@ -1052,6 +1116,7 @@ const handleDetailsScroll = () => {
</div>
</t-drawer>
<div ref="docMarkdownRoot" class="doc-markdown-root">
<!-- URL类型专属区域(保留:source 是真实链接,不与标题重复) -->
<div v-if="details.type === 'url'" class="url_box">
<span class="label">{{ $t('knowledgeBase.urlSource') }}</span>
@@ -1203,6 +1268,7 @@ const handleDetailsScroll = () => {
<DocumentPreview :knowledgeId="details.id" :fileType="details.file_type" :fileName="details.title"
:active="viewMode === 'preview'" />
</div>
</div>
</t-drawer>
</div>