fix(frontend): rework chunking debug UX in KB editor

The chunking debug panel was an inline section that crowded the
strategy form, broke its own layout with negative margins, and crashed
on result.rejected being null. The trigger button required scrolling to
the form's bottom to reach.

- Move debug into a right-side t-drawer pinned above the modal stacking
  context (attach=body, z-index 3000) so it overlays cleanly.
- Promote the trigger to a sticky header sibling next to the section
  title, with right padding to clear the modal close button.
- Replace unicode arrows / + / − with tdesign-icons (Chevron, Play,
  ErrorCircle) and rebuild the result layout: pill-shaped breadcrumb
  tags, line-clamped previews with gradient fade on collapse, ordered
  chunk list, structured error block.
- Drop custom char counter (t-textarea already has one).
- Add four sample fixtures (markdown / faq / chapter / plain) loadable
  as preset chips so users have realistic content to test against.
- Guard `result.rejected` nullability that crashed the panel.
- Normalize the recursive tier label to legacy in the UI since both
  tiers behave identically — pairs with the backend cleanup.
- Rename strategy labels in zh-CN/en-US/ko-KR/ru-RU from internal jargon
  ("auto", "heading", "heuristic", "legacy") to descriptive names
  ("Automatic", "Heading-aware", "Structure-aware", "Length-based") and
  add presetLabel + sample i18n keys.
This commit is contained in:
wizardchen
2026-05-06 20:19:57 +08:00
committed by lyingbug
parent 6efe781a50
commit 49cd392308
7 changed files with 813 additions and 318 deletions
+15 -8
View File
@@ -1924,20 +1924,20 @@ export default {
strategyPlaceholder: 'Select strategy (defaults to classic recursive splitting)',
strategies: {
auto: {
label: 'Automatic (recommended)',
tooltip: 'A document profiler analyzes structure and picks heading-aware, smart-detection or classic splitting per upload.'
label: 'Automatic',
tooltip: 'A document profiler picks between heading-aware, structure-aware and length-based splitting per upload.'
},
heading: {
label: 'Markdown-optimized',
tooltip: 'Splits primarily at Markdown headings (#, ##, ###). Best for clean Markdown with explicit section structure.'
label: 'Heading-aware',
tooltip: 'Splits at Markdown heading boundaries (#, ##, ###); each chunk carries its heading path. Best for well-structured Markdown.'
},
heuristic: {
label: 'Smart structure detection',
tooltip: 'Detects page-breaks, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps titles. Ideal for PDFs without proper Markdown.'
label: 'Structure-aware',
tooltip: 'Splits on detected structural cues: page-breaks, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps titles. Ideal for PDFs without Markdown headings.'
},
legacy: {
label: 'Classic',
tooltip: 'Classic recursive splitting on separators only — the original behavior. Use this if the new strategies misbehave on your content.'
label: 'Length-based',
tooltip: 'Ignores structure; splits recursively by character count and separators — the original behavior. Use when the structure-aware strategies misbehave on your content.'
}
},
overlapWarning: 'Overlap is large compared to chunk size — chunks will share most of their content.',
@@ -1957,6 +1957,13 @@ export default {
toggleHint: 'Run the chunker against sample text without re-uploading',
sampleLabel: 'Sample text',
samplePlaceholder: 'Paste a Markdown / plain-text snippet to see how the current configuration would chunk it…',
presetLabel: 'Load sample:',
samples: {
markdown: 'Markdown doc',
faq: 'FAQ list',
chapter: 'PDF chapters',
plain: 'Plain prose'
},
runButton: 'Run preview',
loading: 'Running chunker on sample…',
errorPrefix: 'Preview failed',
+15 -8
View File
@@ -2533,20 +2533,20 @@ export default {
strategyPlaceholder: "전략 선택 (기본: 클래식 재귀 분할)",
strategies: {
auto: {
label: "자동 (권장)",
tooltip: "문서 프로파일러가 구조를 분석하여 헤딩 인식, 스마트 감지 또는 클래식 분할을 선택합니다."
label: "자동",
tooltip: "문서 프로파일러가 콘텐츠 구조에 따라 「헤딩 기준」, 「구조 인식」, 「길이 기준」 분할 중 하나를 자동 선택합니다."
},
heading: {
label: "Markdown 최적화",
tooltip: "주로 Markdown 헤딩(#, ##, ###)에서 분할합니다. 명확한 섹션 구조의 Markdown에 적합합니다."
label: "헤딩 기준",
tooltip: "Markdown 헤딩(#, ##, ###) 경계에서 분할하며 각 청크에 헤딩 경로가 부착됩니다. 잘 구조화된 Markdown 문서에 적합합니다."
},
heuristic: {
label: "스마트 구조 감지",
tooltip: "페이지 구분자, 번호 매겨진 섹션, 다국어 챕터 마커(DE/EN/ZH), 대문자 제목을 감지합니다. Markdown 구조가 없는 PDF에 적합합니다."
label: "구조 인식",
tooltip: "페이지 구분자, 번호 매겨진 섹션, 다국어 챕터 마커(DE/EN/ZH), 대문자 제목 등 구조적 신호를 감지하여 분할합니다. Markdown 헤딩이 없는 PDF/스캔본에 적합합니다."
},
legacy: {
label: "클래식",
tooltip: "구분자만 사용한 클래식 재귀 분할 — 원래 동작. 새 전략이 콘텐츠에서 잘못 작동하는 경우 사용하세요."
label: "길이 기준",
tooltip: "구조를 무시하고 문자 수와 구분자로만 재귀 분할합니다 — 원래 동작. 위 전략들이 콘텐츠에서 잘못 작동할 때 사용하세요."
}
},
overlapWarning: "오버랩이 청크 크기에 비해 큽니다 — 청크가 대부분의 콘텐츠를 공유합니다.",
@@ -2566,6 +2566,13 @@ export default {
toggleHint: "재업로드 없이 샘플 텍스트에 대해 청커 실행",
sampleLabel: "샘플 텍스트",
samplePlaceholder: "현재 구성으로 어떻게 청크되는지 보려면 Markdown / 일반 텍스트 스니펫을 붙여넣으세요…",
presetLabel: "샘플 불러오기:",
samples: {
markdown: "Markdown 문서",
faq: "FAQ 목록",
chapter: "PDF 챕터",
plain: "일반 텍스트"
},
runButton: "미리보기 실행",
loading: "샘플에 대해 청커 실행 중…",
errorPrefix: "미리보기 실패",
+15 -8
View File
@@ -2136,20 +2136,20 @@ export default {
strategyPlaceholder: 'Выберите стратегию (по умолчанию классическое рекурсивное разбиение)',
strategies: {
auto: {
label: 'Автоматически (рекомендуется)',
tooltip: 'Профилировщик документа анализирует структуру и выбирает заголовочное, эвристическое или классическое разбиение для каждой загрузки.'
label: 'Автоматически',
tooltip: 'Профилировщик документа сам выбирает между «По заголовкам», «По структуре» и «По длине» разбиением для каждой загрузки.'
},
heading: {
label: 'Оптимизировано для Markdown',
tooltip: 'Разбивает преимущественно по заголовкам Markdown (#, ##, ###). Подходит для чистого Markdown с явной структурой разделов.'
label: 'По заголовкам',
tooltip: 'Разбивает по границам заголовков Markdown (#, ##, ###); каждый фрагмент получает свой путь заголовков. Подходит для хорошо структурированного Markdown.'
},
heuristic: {
label: 'Умное определение структуры',
tooltip: 'Определяет разрывы страниц, нумерованные разделы, многоязычные маркеры глав (DE/EN/ZH), заголовки в верхнем регистре. Идеально для PDF без Markdown.'
label: 'По структуре',
tooltip: 'Разбивает по обнаруженным структурным признакам: разрывы страниц, нумерованные разделы, многоязычные маркеры глав (DE/EN/ZH), заголовки в верхнем регистре. Идеально для PDF без заголовков Markdown.'
},
legacy: {
label: 'Классический',
tooltip: 'Классическое рекурсивное разбиение только по разделителям — оригинальное поведение. Используйте, если новые стратегии работают некорректно.'
label: 'По длине',
tooltip: 'Игнорирует структуру и разбивает рекурсивно по числу символов и разделителям — оригинальное поведение. Используйте, если стратегии с учётом структуры работают некорректно.'
}
},
overlapWarning: 'Перекрытие велико по сравнению с размером блока — блоки будут содержать большую часть одинакового контента.',
@@ -2169,6 +2169,13 @@ export default {
toggleHint: 'Запустить разбиение для примера текста без повторной загрузки',
sampleLabel: 'Пример текста',
samplePlaceholder: 'Вставьте фрагмент Markdown / простого текста, чтобы увидеть, как его разобьет текущая конфигурация…',
presetLabel: 'Загрузить пример:',
samples: {
markdown: 'Markdown-документ',
faq: 'Список FAQ',
chapter: 'Главы PDF',
plain: 'Обычный текст'
},
runButton: 'Запустить предпросмотр',
loading: 'Запуск разбиения на примере…',
errorPrefix: 'Ошибка предпросмотра',
+15 -8
View File
@@ -2492,20 +2492,20 @@ export default {
strategyPlaceholder: "选择策略(默认使用经典递归分块)",
strategies: {
auto: {
label: "自动(推荐)",
tooltip: "文档分析器自动选择标题感知、结构检测或经典分块。"
label: "自动",
tooltip: "文档分析器根据内容结构自动在「按标题切分」「结构感知」「按长度切分」之间选择。"
},
heading: {
label: "Markdown 优化",
tooltip: "主要按 Markdown 标题(#、##、###)切分。适合结构清晰的 Markdown。"
label: "按标题切分",
tooltip: "在 Markdown 标题(#、##、###)边界处切分,每块自动带上所在标题路径。适合结构清晰的 Markdown 文档。"
},
heuristic: {
label: "智能结构检测",
tooltip: "检测分页符、编号章节、多语言章节标记(DE/EN/ZH)、全大写标题。适用于无 Markdown 结构的 PDF。"
label: "结构感知",
tooltip: "识别分页符、编号章节、多语言章节标记(DE/EN/ZH)、全大写标题等结构信号进行切分。适合没有 Markdown 标题的 PDF / 扫描件。"
},
legacy: {
label: "经典",
tooltip: "仅按分隔符进行经典递归分块——原始行为。如果新策略效果不佳请使用此项。"
label: "按长度切分",
tooltip: "忽略结构,仅按字符数和分隔符递归切分——原始行为。当上述策略对你的内容效果不佳时使用。"
}
},
overlapWarning: "重叠相对于分块大小较大——分块之间会共享大部分内容。",
@@ -2525,6 +2525,13 @@ export default {
toggleHint: "无需重新上传即可对示例文本运行分块器",
sampleLabel: "示例文本",
samplePlaceholder: "粘贴 Markdown / 纯文本片段以查看当前配置的分块结果…",
presetLabel: "载入示例:",
samples: {
markdown: "Markdown 文档",
faq: "FAQ 列表",
chapter: "PDF 章节",
plain: "纯文本"
},
runButton: "运行预览",
loading: "正在对示例运行分块器…",
errorPrefix: "预览失败",
@@ -1,161 +1,220 @@
<template>
<div class="kb-chunking-debug">
<div class="debug-toggle" @click="open = !open">
<span class="toggle-arrow" :class="{ open }">▸</span>
<span class="toggle-label">{{ $t('knowledgeEditor.chunking.debug.toggle') }}</span>
<span class="toggle-hint">{{ $t('knowledgeEditor.chunking.debug.toggleHint') }}</span>
</div>
<!-- Compact trigger sits inline with the section header so the test
action stays accessible no matter how far the form has scrolled. -->
<t-button
type="button"
theme="primary"
variant="outline"
size="small"
@click="open = true"
>
<template #icon><play-circle-icon /></template>
{{ $t('knowledgeEditor.chunking.debug.toggle') }}
</t-button>
<div v-if="open" class="debug-panel">
<div class="debug-input">
<label>{{ $t('knowledgeEditor.chunking.debug.sampleLabel') }}</label>
<t-textarea
v-model="sample"
:placeholder="$t('knowledgeEditor.chunking.debug.samplePlaceholder')"
:autosize="{ minRows: 6, maxRows: 14 }"
:maxlength="MAX_CHARS"
/>
<div class="input-meta">
<span :class="{ warn: sample.length > MAX_CHARS * 0.8 }">
{{ sample.length }} / {{ MAX_CHARS }} {{ $t('knowledgeEditor.chunking.characters') }}
</span>
<!-- type="button" prevents any accidental parent-form submit. -->
<t-button
type="button"
theme="primary"
:loading="loading"
:disabled="!sample || sample.length === 0"
@click.prevent.stop="runPreview"
>
{{ $t('knowledgeEditor.chunking.debug.runButton') }}
</t-button>
</div>
</div>
<!-- Loading state — explicit so the user sees something is happening
even if the result block hasn't appeared yet. -->
<div v-if="loading" class="debug-loading">
<t-loading size="small" />
<span>{{ $t('knowledgeEditor.chunking.debug.loading') }}</span>
</div>
<!-- Error block: prominent so it can't be missed when an API call fails. -->
<div v-if="error && !loading" class="debug-error">
<strong>{{ $t('knowledgeEditor.chunking.debug.errorPrefix') }}:</strong> {{ error }}
</div>
<div v-if="result && !loading" class="debug-result">
<!-- Tier summary -->
<div class="result-header">
<div class="tier-row">
<span class="result-label">{{ $t('knowledgeEditor.chunking.debug.selectedTier') }}:</span>
<t-tag
:theme="tierTheme(result.selected_tier)"
variant="light-outline"
size="medium"
>
{{ tierDisplay(result.selected_tier) }}
</t-tag>
<span v-if="fallbackWarning" class="fallback-warning">
{{ $t('knowledgeEditor.chunking.debug.fallbackWarning') }}
</span>
</div>
<div v-if="result.rejected.length > 0" class="tier-row">
<span class="result-label">{{ $t('knowledgeEditor.chunking.debug.rejected') }}:</span>
<span class="rejection-list">
<t-tag
v-for="r in result.rejected"
:key="r.tier"
theme="default"
variant="light"
<t-drawer
v-model:visible="open"
:header="$t('knowledgeEditor.chunking.debug.toggle')"
:footer="false"
size="720px"
placement="right"
:close-on-overlay-click="true"
:destroy-on-close="false"
attach="body"
:z-index="3000"
>
<div class="drawer-body">
<!-- Input section -->
<section class="drawer-section">
<div class="section-title-row">
<div class="section-title">{{ $t('knowledgeEditor.chunking.debug.sampleLabel') }}</div>
<div class="sample-presets">
<span class="presets-label">{{ $t('knowledgeEditor.chunking.debug.presetLabel') }}</span>
<t-button
v-for="p in samples"
:key="p.id"
type="button"
variant="text"
size="small"
class="preset-chip"
@click="loadSample(p.id)"
>
{{ tierDisplay(r.tier) }}: {{ r.reason }}
{{ $t(`knowledgeEditor.chunking.debug.${p.labelKey}`) }}
</t-button>
</div>
</div>
<t-textarea
v-model="sample"
:placeholder="$t('knowledgeEditor.chunking.debug.samplePlaceholder')"
:autosize="{ minRows: 6, maxRows: 12 }"
:maxlength="MAX_CHARS"
/>
<div class="input-actions">
<!-- type="button" prevents any accidental parent-form submit. -->
<t-button
type="button"
theme="primary"
:loading="loading"
:disabled="!sample || sample.length === 0"
@click.prevent.stop="runPreview"
>
<template #icon><play-circle-icon /></template>
{{ $t('knowledgeEditor.chunking.debug.runButton') }}
</t-button>
</div>
</section>
<!-- Loading state — explicit so the user sees something is happening
even if the result block hasn't appeared yet. -->
<div v-if="loading" class="debug-loading">
<t-loading size="small" />
<span>{{ $t('knowledgeEditor.chunking.debug.loading') }}</span>
</div>
<!-- Error block: prominent so it can't be missed when an API call fails. -->
<div v-else-if="error" class="debug-error">
<error-circle-icon class="error-icon" />
<div>
<strong>{{ $t('knowledgeEditor.chunking.debug.errorPrefix') }}</strong>
<span>{{ error }}</span>
</div>
</div>
<section v-else-if="result" class="drawer-section debug-result">
<!-- Tier summary -->
<div class="result-header">
<div class="tier-row">
<span class="result-label">{{ $t('knowledgeEditor.chunking.debug.selectedTier') }}:</span>
<t-tag
:theme="tierTheme(result.selected_tier)"
variant="light-outline"
size="medium"
>
{{ tierDisplay(result.selected_tier) }}
</t-tag>
<span v-if="fallbackWarning" class="fallback-warning">
{{ $t('knowledgeEditor.chunking.debug.fallbackWarning') }}
</span>
</div>
<div v-if="(result.rejected || []).length > 0" class="tier-row">
<span class="result-label">{{ $t('knowledgeEditor.chunking.debug.rejected') }}:</span>
<span class="rejection-list">
<t-tag
v-for="r in (result.rejected || [])"
:key="r.tier"
theme="default"
variant="light"
size="small"
>
{{ tierDisplay(r.tier) }}: {{ r.reason }}
</t-tag>
</span>
</div>
</div>
<!-- Profile stats -->
<div class="profile-grid">
<div class="profile-cell">
<div class="cell-value">{{ result.profile.total_lines }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.lines') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.total_chars }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.chars') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.md_heading_total }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.headings') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.form_feed_count }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.pageBreaks') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">
{{
result.profile.german_chapter_count +
result.profile.english_chapter_count +
result.profile.chinese_chapter_count
}}
</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.chapterMarkers') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ (result.profile.detected_langs || []).join(', ') || '—' }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.languages') }}</div>
</div>
</div>
<!-- Chunk stats line -->
<div class="chunk-stats">
<span class="stats-count">
<strong>{{ result.stats.count }}</strong>
{{ $t('knowledgeEditor.chunking.debug.stats.chunks') }}
</span>
<span class="stats-sep">·</span>
<span>Ø {{ result.stats.avg_chars }}</span>
<span class="stats-sep">·</span>
<span>σ {{ result.stats.stddev_chars }}</span>
<span class="stats-sep">·</span>
<span>min {{ result.stats.min_chars }}</span>
<span class="stats-sep">·</span>
<span>max {{ result.stats.max_chars }}</span>
<span v-if="result.stats.truncated_to" class="truncation-hint">
{{ $t('knowledgeEditor.chunking.debug.stats.truncated', { total: result.stats.truncated_to }) }}
</span>
</div>
</div>
<!-- Profile stats -->
<div class="profile-grid">
<div class="profile-cell">
<div class="cell-value">{{ result.profile.total_lines }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.lines') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.total_chars }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.chars') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.md_heading_total }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.headings') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ result.profile.form_feed_count }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.pageBreaks') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">
{{
result.profile.german_chapter_count +
result.profile.english_chapter_count +
result.profile.chinese_chapter_count
}}
</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.chapterMarkers') }}</div>
</div>
<div class="profile-cell">
<div class="cell-value">{{ (result.profile.detected_langs || []).join(', ') || '—' }}</div>
<div class="cell-label">{{ $t('knowledgeEditor.chunking.debug.profile.languages') }}</div>
</div>
</div>
<!-- Chunk stats line -->
<div class="chunk-stats">
<strong>{{ result.stats.count }}</strong>
{{ $t('knowledgeEditor.chunking.debug.stats.chunks') }} —
Ø {{ result.stats.avg_chars }} {{ $t('knowledgeEditor.chunking.characters') }},
σ {{ result.stats.stddev_chars }},
min {{ result.stats.min_chars }}, max {{ result.stats.max_chars }}
<span v-if="result.stats.truncated_to" class="truncation-hint">
({{ $t('knowledgeEditor.chunking.debug.stats.truncated', { total: result.stats.truncated_to }) }})
</span>
</div>
<!-- Chunks list -->
<div class="chunks-list">
<div
v-for="c in result.chunks"
:key="c.seq"
class="chunk-card"
:class="{ expanded: expandedChunks.has(c.seq) }"
>
<div class="chunk-meta" @click="toggleChunk(c.seq)">
<span class="chunk-seq">#{{ c.seq }}</span>
<span class="chunk-size">{{ c.size_chars }} {{ $t('knowledgeEditor.chunking.characters') }} / ~{{ c.size_tokens_approx }} tok</span>
<span class="chunk-pos">[{{ c.start }}–{{ c.end }}]</span>
<span class="chunk-toggle">{{ expandedChunks.has(c.seq) ? '−' : '+' }}</span>
</div>
<div v-if="c.context_header" class="chunk-header">
<strong>{{ $t('knowledgeEditor.chunking.debug.contextHeader') }}:</strong>
<pre>{{ c.context_header }}</pre>
</div>
<div class="chunk-content" :class="{ truncated: !expandedChunks.has(c.seq) }">
<pre>{{ expandedChunks.has(c.seq) ? c.content : c.content.slice(0, 200) }}</pre>
</div>
</div>
</div>
<!-- Chunks list — no inner scroll; the drawer body handles scrolling
so expanded cards always show their full content. -->
<ol class="chunks-list">
<li
v-for="c in result.chunks"
:key="c.seq"
class="chunk-card"
:class="{ expanded: expandedChunks.has(c.seq) }"
>
<button
type="button"
class="chunk-meta"
:aria-expanded="expandedChunks.has(c.seq)"
@click="toggleChunk(c.seq)"
>
<span class="chunk-seq">#{{ c.seq }}</span>
<span class="chunk-size">
{{ c.size_chars }} {{ $t('knowledgeEditor.chunking.characters') }}
<span class="chunk-tokens">· ~{{ c.size_tokens_approx }} tok</span>
</span>
<span class="chunk-pos">{{ c.start }}–{{ c.end }}</span>
<span v-if="c.context_header" class="chunk-context-pill" :title="c.context_header">
{{ c.context_header }}
</span>
<chevron-down-icon class="chunk-toggle" :class="{ open: expandedChunks.has(c.seq) }" />
</button>
<div class="chunk-body" :class="{ collapsed: !expandedChunks.has(c.seq) }">
<pre class="chunk-text">{{ c.content }}</pre>
</div>
</li>
</ol>
</section>
</div>
</div>
</t-drawer>
</div>
</template>
<script setup lang="ts">
import { ref, computed } from 'vue'
import { ref, computed, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { MessagePlugin } from 'tdesign-vue-next'
import {
ChevronDownIcon,
PlayCircleIcon,
ErrorCircleIcon,
} from 'tdesign-icons-vue-next'
import { previewChunking } from '@/api/chunker'
import type { PreviewChunkingResponse, StrategyTier } from '@/types/chunker'
import { CHUNKING_SAMPLES, DEFAULT_SAMPLE_ID } from './chunkingSamples'
interface Props {
config: {
@@ -181,9 +240,31 @@ const error = ref('')
const result = ref<PreviewChunkingResponse | null>(null)
const expandedChunks = ref(new Set<number>())
const samples = CHUNKING_SAMPLES
// Auto-load the default preset the first time the user opens the drawer with
// an empty textarea. We don't overwrite their input on subsequent opens, and
// we don't pre-load on component mount (zero cost when the drawer is unused).
watch(open, (isOpen) => {
if (isOpen && sample.value.trim() === '') {
loadSample(DEFAULT_SAMPLE_ID)
}
})
const loadSample = (id: string) => {
const preset = samples.find((s) => s.id === id)
if (!preset) return
sample.value = preset.text
// Clear any previous run so the user isn't looking at stale results
// attributed to the old text.
result.value = null
error.value = ''
expandedChunks.value = new Set()
}
const fallbackWarning = computed(() => {
if (!result.value) return false
return result.value.selected_tier === 'legacy' && result.value.rejected.length > 0
return result.value.selected_tier === 'legacy' && (result.value.rejected || []).length > 0
})
const runPreview = async () => {
@@ -245,12 +326,19 @@ const toggleChunk = (seq: number) => {
expandedChunks.value = next
}
// `recursive` and `legacy` use the same SplitText path under the hood
// (see internal/infrastructure/chunker/strategy.go); their distinction is
// only a debugging hint about how the tier was reached. Surface them under
// the user-facing legacy label to avoid implying two different splitters.
const normalizeTier = (tier: StrategyTier): StrategyTier =>
tier === 'recursive' ? 'legacy' : tier
const tierDisplay = (tier: StrategyTier) => {
return t(`knowledgeEditor.chunking.strategies.${tier}.label`)
return t(`knowledgeEditor.chunking.strategies.${normalizeTier(tier)}.label`)
}
const tierTheme = (tier: StrategyTier) => {
switch (tier) {
switch (normalizeTier(tier)) {
case 'heading':
case 'heuristic':
return 'success'
@@ -265,108 +353,107 @@ const tierTheme = (tier: StrategyTier) => {
<style lang="less" scoped>
.kb-chunking-debug {
margin-top: 24px;
border-top: 1px solid var(--td-component-stroke);
padding-top: 16px;
flex-shrink: 0;
}
.debug-toggle {
.drawer-body {
display: flex;
flex-direction: column;
gap: 20px;
}
.drawer-section {
display: flex;
flex-direction: column;
gap: 8px;
}
.section-title-row {
display: flex;
align-items: center;
gap: 10px;
cursor: pointer;
user-select: none;
padding: 6px 0;
&:hover {
.toggle-label {
color: var(--td-text-color-primary);
}
}
justify-content: space-between;
gap: 12px;
flex-wrap: wrap;
}
.toggle-arrow {
display: inline-block;
font-size: 12px;
transition: transform 0.15s ease;
color: var(--td-text-color-secondary);
&.open {
transform: rotate(90deg);
}
}
.toggle-label {
font-size: 14px;
.section-title {
font-size: 13px;
font-weight: 500;
color: var(--td-text-color-secondary);
color: var(--td-text-color-primary);
}
.toggle-hint {
font-size: 12px;
color: var(--td-text-color-placeholder);
}
.sample-presets {
display: flex;
align-items: center;
gap: 4px;
flex-wrap: wrap;
.debug-panel {
margin-top: 12px;
padding: 16px;
background: var(--td-bg-color-container);
border: 1px solid var(--td-component-stroke);
border-radius: 6px;
}
.debug-input {
label {
display: block;
font-size: 13px;
font-weight: 500;
margin-bottom: 6px;
color: var(--td-text-color-primary);
.presets-label {
font-size: 12px;
color: var(--td-text-color-placeholder);
margin-right: 4px;
}
.input-meta {
margin-top: 8px;
display: flex;
align-items: center;
justify-content: space-between;
font-size: 12px;
.preset-chip {
--td-comp-paddinglr-s: 8px;
color: var(--td-text-color-secondary);
font-size: 12px;
.warn {
color: var(--td-warning-color);
&:hover {
color: var(--td-brand-color);
}
}
}
.input-actions {
display: flex;
justify-content: flex-end;
margin-top: 4px;
}
// debug-result manages its own internal vertical rhythm via margin-bottom
// on each block; drop the section-level gap so spacing isn't doubled.
.debug-result {
gap: 0;
}
.debug-loading {
margin-top: 16px;
display: flex;
align-items: center;
gap: 10px;
padding: 12px 14px;
background: var(--td-bg-color-container-hover);
border-radius: 4px;
border-radius: 6px;
font-size: 13px;
color: var(--td-text-color-secondary);
}
.debug-error {
margin-top: 12px;
display: flex;
align-items: flex-start;
gap: 10px;
padding: 12px 14px;
background: var(--td-error-color-light);
border-left: 3px solid var(--td-error-color);
color: var(--td-error-color);
border-radius: 6px;
font-size: 13px;
border-radius: 0 4px 4px 0;
font-weight: 500;
color: var(--td-error-color);
.error-icon {
flex-shrink: 0;
margin-top: 2px;
font-size: 16px;
}
strong {
margin-right: 4px;
display: block;
margin-bottom: 2px;
}
}
.debug-result {
margin-top: 16px;
span {
color: var(--td-error-color);
font-weight: 400;
word-break: break-word;
}
}
.result-header {
@@ -405,45 +492,62 @@ const tierTheme = (tier: StrategyTier) => {
.profile-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 12px;
grid-template-columns: repeat(auto-fit, minmax(110px, 1fr));
gap: 1px;
margin-bottom: 16px;
padding: 12px;
background: var(--td-bg-color-container-hover);
border-radius: 4px;
background: var(--td-component-stroke);
border: 1px solid var(--td-component-stroke);
border-radius: 6px;
overflow: hidden;
}
.profile-cell {
padding: 12px 8px;
text-align: center;
background: var(--td-bg-color-container);
}
.cell-value {
font-size: 18px;
font-weight: 600;
font-variant-numeric: tabular-nums;
color: var(--td-text-color-primary);
line-height: 1.2;
}
.cell-label {
margin-top: 4px;
font-size: 11px;
color: var(--td-text-color-secondary);
margin-top: 2px;
}
.chunk-stats {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: 8px;
margin-bottom: 12px;
padding: 10px 14px;
background: var(--td-bg-color-container-hover);
border-radius: 6px;
font-size: 13px;
color: var(--td-text-color-secondary);
margin-bottom: 12px;
padding: 8px 12px;
background: var(--td-bg-color-container-hover);
border-radius: 4px;
font-variant-numeric: tabular-nums;
strong {
.stats-count strong {
margin-right: 4px;
color: var(--td-text-color-primary);
font-size: 14px;
font-weight: 600;
}
.stats-sep {
color: var(--td-text-color-placeholder);
}
}
.truncation-hint {
margin-left: auto;
color: var(--td-warning-color);
font-size: 12px;
}
@@ -452,93 +556,137 @@ const tierTheme = (tier: StrategyTier) => {
display: flex;
flex-direction: column;
gap: 8px;
max-height: 500px;
overflow-y: auto;
margin: 0;
padding: 0;
list-style: none;
}
.chunk-card {
border: 1px solid var(--td-component-stroke);
border-radius: 4px;
border-radius: 6px;
background: var(--td-bg-color-container);
overflow: hidden;
transition: border-color 0.15s ease, box-shadow 0.15s ease;
&.expanded {
border-color: var(--td-brand-color-light-active);
box-shadow: 0 0 0 1px var(--td-brand-color-light) inset;
}
}
.chunk-meta {
display: flex;
align-items: center;
gap: 12px;
padding: 8px 12px;
width: 100%;
margin: 0;
padding: 10px 14px;
background: var(--td-bg-color-container-hover);
border: none;
cursor: pointer;
font-size: 12px;
color: var(--td-text-color-secondary);
text-align: left;
&:hover {
background: var(--td-bg-color-component-hover);
}
&:focus-visible {
outline: 2px solid var(--td-brand-color-focus);
outline-offset: -2px;
}
}
.chunk-seq {
font-weight: 600;
flex-shrink: 0;
color: var(--td-text-color-primary);
font-weight: 600;
font-variant-numeric: tabular-nums;
}
.chunk-size {
flex-shrink: 0;
color: var(--td-text-color-primary);
font-variant-numeric: tabular-nums;
.chunk-tokens {
color: var(--td-text-color-secondary);
font-weight: 400;
}
}
.chunk-pos {
flex-shrink: 0;
color: var(--td-text-color-placeholder);
font-family: monospace;
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
font-variant-numeric: tabular-nums;
}
.chunk-context-pill {
flex: 0 1 auto;
min-width: 0;
max-width: 240px;
padding: 2px 8px;
background: var(--td-brand-color-light);
color: var(--td-brand-color);
border-radius: 10px;
font-size: 11px;
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.chunk-toggle {
margin-left: auto;
font-weight: 700;
font-size: 14px;
flex-shrink: 0;
font-size: 16px;
color: var(--td-text-color-secondary);
transition: transform 0.15s ease;
&.open {
transform: rotate(180deg);
}
}
.chunk-header {
padding: 6px 12px;
.chunk-body {
border-top: 1px solid var(--td-component-stroke);
background: var(--td-bg-color-container);
font-size: 12px;
strong {
display: inline-block;
margin-right: 8px;
color: var(--td-text-color-secondary);
}
pre {
display: inline;
margin: 0;
font-size: 12px;
color: var(--td-brand-color);
white-space: pre-wrap;
word-break: break-word;
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
}
}
.chunk-content {
padding: 8px 12px;
.chunk-text {
margin: 0;
padding: 12px 14px;
font-size: 12.5px;
line-height: 1.6;
white-space: pre-wrap;
word-break: break-word;
color: var(--td-text-color-primary);
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
}
pre {
margin: 0;
font-size: 12px;
line-height: 1.5;
white-space: pre-wrap;
word-break: break-word;
color: var(--td-text-color-primary);
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
}
// Collapsed preview: clamp the visible lines so each card stays compact, but
// stays a real text block (not a JS slice) so wrapping/spacing match the
// expanded view exactly.
.chunk-body.collapsed .chunk-text {
display: -webkit-box;
-webkit-line-clamp: 3;
-webkit-box-orient: vertical;
overflow: hidden;
position: relative;
&.truncated pre::after {
content: '…';
color: var(--td-text-color-placeholder);
margin-left: 4px;
&::after {
content: '';
position: absolute;
inset: auto 0 0 0;
height: 28px;
background: linear-gradient(
to bottom,
transparent,
var(--td-bg-color-container)
);
pointer-events: none;
}
}
</style>
@@ -1,13 +1,18 @@
<template>
<div class="kb-chunking-settings">
<div class="section-header">
<h2>{{ $t('knowledgeEditor.chunking.title') }}</h2>
<p class="section-description">{{ $t('knowledgeEditor.chunking.description') }}</p>
<div class="section-header-text">
<h2>{{ $t('knowledgeEditor.chunking.title') }}</h2>
<p class="section-description">{{ $t('knowledgeEditor.chunking.description') }}</p>
</div>
<!-- Trigger lives next to the section title so users can sample
anytime without scrolling to the bottom of a long form. -->
<KBChunkingDebug :config="debugConfig" />
</div>
<div class="settings-group">
<!-- Strategy -->
<div class="setting-row strategy-row">
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.strategyLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.strategyDescription') }}</p>
@@ -156,10 +161,10 @@
</div>
<!-- Advanced section toggle -->
<div class="advanced-toggle" @click="advancedOpen = !advancedOpen">
<span class="toggle-arrow" :class="{ 'open': advancedOpen }">▸</span>
<button type="button" class="advanced-toggle" @click="advancedOpen = !advancedOpen">
<chevron-right-icon class="toggle-arrow" :class="{ open: advancedOpen }" />
<span>{{ $t('knowledgeEditor.chunking.advancedLabel') }}</span>
</div>
</button>
<div v-if="advancedOpen" class="advanced-section">
<!-- Token Limit -->
@@ -201,8 +206,6 @@
</div>
</div>
<!-- Debug panel: try a sample text against the current config without re-uploading. -->
<KBChunkingDebug :config="debugConfig" />
</div>
</div>
</template>
@@ -210,6 +213,7 @@
<script setup lang="ts">
import { ref, watch, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { ChevronRightIcon } from 'tdesign-icons-vue-next'
import KBChunkingDebug from './KBChunkingDebug.vue'
interface ParserEngineRule {
@@ -380,7 +384,29 @@ const emitUpdate = () => {
}
.section-header {
margin-bottom: 32px;
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
// Stick to the top of the scrollable section so the inline test trigger
// stays in reach no matter how far the form has scrolled. Negative top
// and matching negative margins compensate for content-wrapper's
// padding (24px 32px) so the sticky band visually spans the full width
// when stuck.
position: sticky;
top: -24px;
z-index: 5;
background: var(--td-bg-color-container);
// Reserve clearance on the right so the inline trigger never sits under
// the modal's absolutely-positioned close button.
padding: 24px 80px 16px 32px;
margin: -24px -32px 24px -32px;
border-bottom: 1px solid var(--td-component-stroke);
.section-header-text {
flex: 1;
min-width: 0;
}
h2 {
font-size: 20px;
@@ -419,16 +445,8 @@ const emitUpdate = () => {
}
}
.strategy-row {
// Strategy is the most prominent setting — slight emphasis.
background: linear-gradient(to right, var(--td-bg-color-container-hover), transparent);
padding-left: 12px;
margin-left: -12px;
border-radius: 6px;
}
.strategy-info-panel {
margin: -8px 0 12px 12px;
margin: 0 0 16px 0;
padding: 10px 14px;
background: var(--td-bg-color-container-hover);
border-left: 3px solid var(--td-brand-color);
@@ -439,6 +457,7 @@ const emitUpdate = () => {
p {
margin: 0;
word-break: break-word;
}
strong {
@@ -499,10 +518,13 @@ const emitUpdate = () => {
}
.advanced-toggle {
display: flex;
display: inline-flex;
align-items: center;
gap: 8px;
gap: 6px;
padding: 16px 0 8px 0;
margin: 0;
background: transparent;
border: none;
cursor: pointer;
font-size: 14px;
font-weight: 500;
@@ -512,12 +534,17 @@ const emitUpdate = () => {
&:hover {
color: var(--td-text-color-primary);
}
&:focus-visible {
outline: 2px solid var(--td-brand-color-focus);
outline-offset: 2px;
border-radius: 4px;
}
}
.toggle-arrow {
display: inline-block;
font-size: 16px;
transition: transform 0.15s ease;
font-size: 12px;
&.open {
transform: rotate(90deg);
@@ -525,7 +552,8 @@ const emitUpdate = () => {
}
.advanced-section {
padding-left: 12px;
border-left: 2px solid var(--td-component-stroke);
// Visually grouped via the toggle above; avoid a left rule that hugs the
// panel edge and looks detached from the rest of the form.
margin-top: 4px;
}
</style>
@@ -0,0 +1,291 @@
// Curated sample texts for the chunking debug drawer. Each preset is sized
// to ≈2000–4000 characters and shaped to exercise a distinct tier of the
// chunker, so users can quickly see how their config behaves on realistic
// content without preparing their own sample.
export interface ChunkingSample {
id: string
// i18n key under knowledgeEditor.chunking.debug.samples.<id>
labelKey: string
text: string
}
const MARKDOWN_SAMPLE = `# WeKnora 知识框架
WeKnora 是一个基于 LLM 的开源企业知识框架,集 RAG 问答、ReAct 智能体、Wiki 知识图谱于一体。本文介绍其设计动机、架构与典型用法。
## 设计动机
企业内部知识散落在 Confluence、飞书、Notion、Git 仓库等多种系统,传统全文检索难以理解语义,单一 LLM 又缺少可信赖的上下文来源。WeKnora 的目标是:
- **多源接入**:把分散内容统一抽取、清洗、向量化
- **多策略检索**:稠密、稀疏、知识图谱多路召回 + RRF 融合
- **可解释推理**:ReAct Agent 让推理链条可追踪、可干预
- **可观测**:原生集成 Langfuse,每一步推理、每一次工具调用都有 trace
## 核心特性
### 多模型支持
兼容 20+ 主流 LLM:OpenAI、Anthropic、DeepSeek、Qwen、智谱、混元、Gemini,以及本地 Ollama。模型层抽象 \`Provider\` 接口,新增厂商只需实现 \`Chat()\` 方法并注册即可。
### 多向量库支持
通过 \`RETRIEVE_DRIVER\` 环境变量切换:
| 驱动 | 适合场景 |
|------|---------|
| pgvector | 已有 PostgreSQL,规模 < 1000 万 chunk |
| Elasticsearch | 需要混合检索 + 关键词高亮 |
| Milvus | 大规模、低延迟 |
| Weaviate | 需要混合 BM25 + GraphQL 查询 |
| Qdrant | 默认推荐,部署简单、性能均衡 |
### 数据源连接器
- **飞书**:自动同步知识库与文档树,支持增量更新
- **Notion**:基于 API 抽取 Page / Database
- **语雀**:按 Group 拉取,保留多级目录
每个连接器实现统一的 \`DataSource\` 接口,新增数据源约 200 行代码。
## 快速开始
### 环境要求
- Docker 20.10+ 与 Docker Compose v2
- 至少 8GB 内存、20GB 磁盘空间
- 如需本地运行 LLM,建议 NVIDIA GPU 24GB 以上显存
### 启动命令
\`\`\`bash
git clone https://github.com/Tencent/WeKnora && cd WeKnora
cp .env.example .env # 修改你的模型与数据库配置
make dev-start # 启动 postgres / redis / qdrant
make dev-app # 启动后端,热重载
make dev-frontend # 启动前端,自动刷新
\`\`\`
打开浏览器访问 http://localhost:5173 即可。
## 架构概览
WeKnora 后端采用清晰的分层架构:
\`\`\`
┌────────────────────────┐
│ HTTP 层 (Gin) │ 请求路由 / 鉴权 / 限流
├────────────────────────┤
│ 业务层 (service) │ 核心算法 / 编排
├────────────────────────┤
│ 数据层 (repository) │ 数据库访问 / 缓存
├────────────────────────┤
│ 领域模型 (types) │ KnowledgeBase / Chunk / Agent ...
└────────────────────────┘
\`\`\`
依赖注入容器(go/dig)在 \`cmd/server/main.go\` 装配所有依赖,handler 通过接口拿到 service,service 通过接口拿到 repository,所有可替换实现都在启动时绑定。
### 检索流水线
文档上传 → docreader 服务解析 → 分块(含父子分块) → 嵌入 → 写入向量库与图数据库 → 用户查询时多路召回 → RRF 融合 → 重排 → 返回。
### Agent 编排
Agent 引擎执行经典 ReAct 循环:
1. **Reason**:LLM 根据系统提示和当前观察输出推理
2. **Act**:解析工具调用 JSON,分发给内置工具或 MCP 工具
3. **Observe**:把工具返回值塞回上下文
4. 重复 1-3 直到产出最终答案或达到最大迭代
每一步都通过 SSE 流式返回前端,并写入 Langfuse trace 便于排查。
## 进一步阅读
- API 文档:\`docs/api/README.md\`
- 配置项清单:\`config/config.yaml\` 与 \`.env.example\`
- 故障排查:\`docs/QA.md\`
- 路线图:\`docs/ROADMAP.md\``
const FAQ_SAMPLE = `# WeKnora 部署与使用 FAQ
本文档汇总社区与内部用户最常问到的问题,按"安装 / 配置 / 检索 / 模型 / 性能"分类。
## 安装与启动
### Q1: Docker 镜像在哪里下载?
官方镜像通过 daocloud 加速分发:
\`\`\`
docker pull docker.m.daocloud.io/wechatopenai/weknora-app:v0.5.0
docker pull docker.m.daocloud.io/wechatopenai/weknora-docreader:v0.5.0
docker pull docker.m.daocloud.io/wechatopenai/weknora-ui:v0.5.0
\`\`\`
### Q2: 启动后访问 5173 显示旧界面?
浏览器缓存了旧版前端资源。Ctrl+Shift+R 强制刷新即可,必要时清空站点存储。
### Q3: 注册新用户报 500 Internal Server Error
通常是后端容器未正常启动。先检查 \`make dev-logs | grep app\`,常见为数据库连接失败或迁移未执行。
### Q4: 上传文档报 column "xxx" does not exist
数据库迁移未完成。运行 \`make migrate-up\`,确认所有迁移成功后重启 app 容器。
### Q5: 文档上传后状态一直是 processing
检查 docreader 容器是否运行(\`docker compose ps\`),并查看其日志中是否有解析报错。常见原因:依赖的 Python 包缺失、PDF 加密、扫描件 OCR 超时。
## 配置
### Q6: 如何切换向量库?
修改 \`.env\` 中的 \`RETRIEVE_DRIVER\`,可选值:\`qdrant\` (默认) / \`pgvector\` / \`elasticsearch\` / \`milvus\` / \`weaviate\`。切换后需要重新嵌入存量数据。
### Q7: 如何修改默认 chunk 大小?
全局默认在 \`config/config.yaml\` 的 \`knowledge.chunking\` 段;单个知识库可在 UI 的"分块设置"页覆盖。修改后仅对新上传文档生效。
### Q8: 如何启用多模态?
在知识库的"多模态配置"中开启,并选择一个 VLM 模型(如 GPT-4o、Qwen2.5-VL)。开启后,文档中的图片会被 VLM 描述并参与检索。
### Q9: 怎么把日志改成 JSON 格式?
设置 \`LOG_FORMAT=json LOG_LEVEL=info\`,便于接入 ELK / Loki 等日志栈。
## 检索与召回
### Q10: 检索结果为空但确认文档已索引
按这个顺序排查:
1. 确认知识库状态为 \`completed\`,而不是 \`indexing\` 或 \`failed\`
2. 检查向量库连接(默认 qdrant:6334),\`docker compose ps\` 看 qdrant 是否健康
3. 在管理页"重建索引",确认嵌入模型工作正常
4. 查看后端日志中 retriever 是否报 dimension mismatch
### Q11: 检索结果命中但答非所问
通常是 chunk 边界切割了关键句。尝试:
- 启用父子分块,让向量召回小块、上下文用大块
- 调高 chunk overlap 到 chunk size 的 15%-20%
- 切换到"按标题切分"或"结构感知"策略
### Q12: 排序看起来不合理
重排(rerank)模型可能未启用。在"模型设置"中配置 BGE-Reranker 或 Cohere Rerank,并在知识库配置中打开重排。
## 模型与 Token
### Q13: 调用 OpenAI 超时
检查代理网络与 \`OPENAI_BASE_URL\` 配置;本地无外网时切换到 Ollama 或国内大模型。
### Q14: token 用量在哪里看?
开启 Langfuse 后,每个会话、每次推理、每个工具调用的 token 消耗都有完整 trace,可按时间、租户、模型聚合。
### Q15: 如何接入私有 LLM?
实现 \`internal/models/chat/provider/Provider\` 接口,注册到 provider 工厂,配置文件中加一条记录即可,约 100-200 行代码。
## 性能
### Q16: 大文档上传慢怎么办?
docreader 是 CPU 密集型,建议横向扩容:在 docker-compose 中调高 docreader 服务的 \`deploy.replicas\`,或独立部署到更强的机器。
### Q17: 嵌入吞吐瓶颈
切换到本地 Ollama + GPU 加速,或用商用 API 的批量端点(如 OpenAI 的 batch API)。设置 \`EMBED_BATCH_SIZE=64\` 显著提升吞吐。
### Q18: 数据库慢查询
启用 \`pg_stat_statements\`,定位慢 SQL;常见瓶颈是 chunks 表的元数据 JSONB 查询,可针对热点字段加索引。`
const CHAPTER_SAMPLE = `第 1 章 引言
1.1 文档目的
本文档描述某分布式知识检索平台的整体架构、组件划分与部署方案,面向运维工程师、SRE 与系统架构师。读者应熟悉 Docker、Kubernetes、PostgreSQL 等基础组件。
1.2 名词约定
- 网关层(Gateway):负责入口流量调度与协议转换
- 业务层(Service):承载领域逻辑与编排
- 存储层(Storage):封装持久化与缓存细节
- 推理层(Inference):与外部 LLM / 嵌入模型交互的隔离层
1.3 文档版本
本版本 v1.4,对应平台代码 v0.5.x 系列。文档随代码同步更新,主要变更见附录 A。
第 2 章 系统架构
2.1 总体设计
平台采用经典的微服务架构,但在边界划分上保持克制——只在确有独立伸缩需求的位置切分服务。当前共 4 个长驻服务:网关、业务后端、文档解析、向量索引。
2.2 模块划分
2.2.1 用户与权限
用户、组织、共享空间、角色与权限规则集中在 user-service 中。RBAC 模型支持继承与覆盖,外部 OIDC 接入通过适配层完成。
2.2.2 内容与索引
内容侧负责文档生命周期:上传、解析、分块、嵌入、入库、检索、引用追踪。Chunk 是最细粒度的检索单元,每个 chunk 携带源文档元数据与位置信息。
2.2.3 推理与编排
Agent 编排器负责 ReAct 循环,工具调用通过 MCP 协议或内置注册表分发。所有 LLM 调用统一经过模型代理层,方便切换、限流与计费。
2.3 数据流
文档上传后依次经过:解析 → 清洗 → 分块 → 嵌入 → 写库。查询路径:rewrite → 多路召回 → fusion → rerank → context 拼接 → LLM 生成。
第 3 章 部署指南
3.1 环境要求
物理资源:8 vCPU / 16 GB 内存 / 100 GB SSD 起步;生产环境建议 16 vCPU / 32 GB / 500 GB。GPU 仅在本地推理时必需。
3.2 容器编排
3.2.1 单机 Docker Compose
适合 PoC 与中小团队(< 100 人,< 100 万 chunk)。一条命令拉起全部服务:
docker compose up -d
3.2.2 Kubernetes Helm
适合规模化部署。Helm chart 在 helm/ 目录,包含 statefulset、配置 secret、ingress 模版。可与现有 PG / Redis 集群对接。
3.3 配置最佳实践
- 把外部依赖(数据库、对象存储、向量库)放在配置中心,避免硬编码
- 模型 API key 用 Secret 管理,按环境隔离
- 日志走 stdout/stderr 由编排平台采集,避免本地落盘
第 4 章 常见运维场景
4.1 升级流程
蓝绿或滚动均可。升级前必看 CHANGELOG 中是否有 schema 变更,迁移先于业务镜像替换。
4.2 备份与恢复
PG 全量每日备份 + WAL 增量;MinIO/对象存储依赖云端版本管理。Qdrant 用 snapshot 接口定期落盘到对象存储。
4.3 故障排查
按"网关 → 业务 → 存储 → 模型"四个维度逐层排查。每层都有健康检查接口与 Langfuse trace 入口,组合起来可在 5 分钟内定位 80% 问题。`
const PLAIN_SAMPLE = `知识库的检索质量受多个因素影响,最直接的是切分策略与嵌入模型的匹配度。切分过粗会导致单段语义混杂、相关度被稀释;切分过细则丢失上下文,单独检索某一段无法回答跨段问题。一般建议切分大小落在嵌入模型推荐窗口的 50%–80%,既保证语义完整又留出余量。常见嵌入模型如 BGE、Cohere 的 embed-v3 推荐窗口为 512 tokens,对应字符大致在 300–800 之间,因为中文一个字约 0.5–1.2 个 token,英文一个词约 1.3 个 token。
除了切分大小,重叠(overlap)也会显著影响召回完整性。重叠为 0 时,跨段问题往往只能召回半句话;重叠 10%–20% 通常就足够覆盖大多数边界情况;重叠超过 30% 会让相邻分块大量同质化,反而增加索引成本而不提升召回。一个简单的判断方法:对你的目标查询,平均答案长度的 1/3 作为 overlap 起点,再根据 A/B 测试微调。
分隔符的选择应贴合文档真实结构。纯文本可保留默认双换行作为强分隔;Markdown 文档建议保留标题层级,先按标题切,再在大标题内部按段落切;代码与表格混合的内容则需要更精细的策略,例如 Tree-sitter 抽取代码结构、识别表格边界,或者直接用脚本把这两类内容预先分离再走文本切分。如果文档里大量混合中英文,分隔符里务必同时包含中文标点(。!?)和英文标点(. ! ?),否则只对一种语言生效,效果会很差。
嵌入模型的选择往往被低估。开源里 BGE-large 系列在中文 benchmark 上稳定领先,BGE-M3 同时支持稠密、稀疏、ColBERT 多向量;商用 OpenAI text-embedding-3-large 通用性强、成本不算高,Voyage-3-large 在英文上略胜一筹但中文一般。对长文档场景,注意嵌入模型的最大输入长度——OpenAI 是 8191 tokens,BGE 默认 512,超长输入会被截断或分段嵌入再平均,效果都不好。
最后,别忽视后处理。重排(rerank)几乎在所有场景都能提升 5%–15% 的端到端效果,代价是每次查询多 100–300ms 延迟。常见 reranker 包括 BGE-Reranker、Cohere Rerank,前者开源、后者付费但效果略好。如果你的检索路径包含关键词召回,强烈建议用 reranker 做第二层过滤,不然 BM25 的纯字面命中会污染上下文。`
export const CHUNKING_SAMPLES: ChunkingSample[] = [
{ id: "markdown", labelKey: "samples.markdown", text: MARKDOWN_SAMPLE },
{ id: "faq", labelKey: "samples.faq", text: FAQ_SAMPLE },
{ id: "chapter", labelKey: "samples.chapter", text: CHAPTER_SAMPLE },
{ id: "plain", labelKey: "samples.plain", text: PLAIN_SAMPLE },
];
export const DEFAULT_SAMPLE_ID = "markdown";