From 46d9c396c6f221259c63fb66a61120b80460e7d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 2 May 2026 20:40:07 +0000 Subject: [PATCH] feat(frontend): expose chunking strategy in KB editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces the adaptive 3-tier chunking controls in the existing "Chunking" section of the KB editor modal. Users can now pick a strategy from a dropdown without crafting JSON for the API: - Automatic (recommended) → backend profiler picks the tier - Markdown-optimized → forces heading-aware splitting - Smart structure detection → forces heuristic boundary detection - Classic → original recursive splitter (legacy) The fifth backend value `recursive` is intentionally hidden from the UI — it's functionally near `legacy` and would dilute the choice. Advanced options (collapsible, disabled when strategy=legacy): - TokenLimit (0–8192 approx tokens; 0 = char-based budget) - Languages multi-select (DE/EN/ZH; empty = auto-detect) Behaviour: - New KBs default to strategy="auto" - Existing KBs without the field render with empty strategy and keep classic behavior on save unless the user actively picks one - Overlap warning when overlap >= chunkSize/2 Wired through: - KBChunkingSettings.vue: strategy select + advanced collapsible + ChunkingConfig interface extended - KnowledgeBaseEditorModal.vue: form-state defaults, kb-load mapping, buildSubmitData (chunking_config) + updateKBConfig (documentSplitting) pass strategy/token_limit/languages - api/initialization/index.ts: types extended - i18n: en-US / zh-CN / ko-KR / ru-RU all carry strategy + advanced keys (6 keys each) https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k --- frontend/src/api/initialization/index.ts | 15 ++ frontend/src/i18n/locales/en-US.ts | 35 ++- frontend/src/i18n/locales/ko-KR.ts | 33 +++ frontend/src/i18n/locales/ru-RU.ts | 35 ++- frontend/src/i18n/locales/zh-CN.ts | 33 +++ .../knowledge/KnowledgeBaseEditorModal.vue | 30 ++- .../knowledge/settings/KBChunkingSettings.vue | 208 +++++++++++++++++- 7 files changed, 382 insertions(+), 7 deletions(-) diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index e0c168be..5c5fec4c 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -50,6 +50,14 @@ export interface InitializationConfig { chunkSize: number; chunkOverlap: number; separators: string[]; + // Adaptive chunking strategy. Empty / "legacy" = classic recursive splitter. + // "auto" lets the backend profiler pick a tier; "heading" / "heuristic" + // pin the tier explicitly. See backend chunker package for details. + strategy?: string; + // Cap chunk size in approx tokens. 0 = char-based budget only. + tokenLimit?: number; + // Language hints for heuristic patterns ("de", "en", "zh"). Empty = auto-detect. + languages?: string[]; }; // Frontend-only hint for storage selection UI storageType?: 'cos' | 'minio'; @@ -94,6 +102,13 @@ export interface KBModelConfigRequest { enableParentChild?: boolean parentChunkSize?: number childChunkSize?: number + // Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy"). + // Empty string keeps the existing backend default (legacy / no change). + strategy?: string + // Approximate token budget per chunk; 0 = char-based. + tokenLimit?: number + // Language hints for heuristic patterns. Empty = auto-detect. + languages?: string[] } multimodal: { enabled: boolean diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 7c954498..87272316 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1918,7 +1918,40 @@ export default { parentChunkSizeLabel: 'Parent Chunk Size', parentChunkSizeDescription: 'Size of parent chunks that provide context (256-4096)', childChunkSizeLabel: 'Child Chunk Size', - childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)' + childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)', + strategyLabel: 'Chunking Strategy', + strategyDescription: 'Choose how documents are split into chunks. The Automatic mode profiles each document and picks the best strategy.', + strategyPlaceholder: 'Select strategy (defaults to classic recursive splitting)', + strategies: { + auto: { + label: 'Automatic (recommended)', + tooltip: 'A document profiler analyzes structure and picks heading-aware, smart-detection or classic splitting per upload.' + }, + heading: { + label: 'Markdown-optimized', + tooltip: 'Splits primarily at Markdown headings (#, ##, ###). Best for clean Markdown with explicit section structure.' + }, + heuristic: { + label: 'Smart structure detection', + tooltip: 'Detects page-breaks, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps titles. Ideal for PDFs without proper Markdown.' + }, + legacy: { + label: 'Classic', + tooltip: 'Classic recursive splitting on separators only — the original behavior. Use this if the new strategies misbehave on your content.' + } + }, + overlapWarning: 'Overlap is large compared to chunk size — chunks will share most of their content.', + advancedLabel: 'Advanced options', + tokenLimitLabel: 'Token limit per chunk', + tokenLimitDescription: 'Cap chunk size in approximate tokens. When set, chunk size is automatically reduced to stay below this token budget. 0 = off (use character size only).', + languagesLabel: 'Language hints', + languagesDescription: 'Hint which languages structure-detection patterns should look for. Leave empty for auto-detection.', + languagesPlaceholder: 'Auto-detect', + languageOptions: { + de: 'German', + en: 'English', + zh: 'Chinese' + } }, multimodal: { title: 'Image Processing Configuration', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 283ffa5c..4a111acf 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2528,6 +2528,39 @@ export default { parentChunkSizeDescription: "컨텍스트를 제공하는 부모 청크의 문자 수 (256-4096)", childChunkSizeLabel: "자식 청크 크기", childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)", + strategyLabel: "청킹 전략", + strategyDescription: "문서를 청크로 분할하는 방법을 선택합니다. 자동 모드는 문서를 프로파일링하여 최적의 전략을 선택합니다.", + strategyPlaceholder: "전략 선택 (기본: 클래식 재귀 분할)", + strategies: { + auto: { + label: "자동 (권장)", + tooltip: "문서 프로파일러가 구조를 분석하여 헤딩 인식, 스마트 감지 또는 클래식 분할을 선택합니다." + }, + heading: { + label: "Markdown 최적화", + tooltip: "주로 Markdown 헤딩(#, ##, ###)에서 분할합니다. 명확한 섹션 구조의 Markdown에 적합합니다." + }, + heuristic: { + label: "스마트 구조 감지", + tooltip: "페이지 구분자, 번호 매겨진 섹션, 다국어 챕터 마커(DE/EN/ZH), 대문자 제목을 감지합니다. Markdown 구조가 없는 PDF에 적합합니다." + }, + legacy: { + label: "클래식", + tooltip: "구분자만 사용한 클래식 재귀 분할 — 원래 동작. 새 전략이 콘텐츠에서 잘못 작동하는 경우 사용하세요." + } + }, + overlapWarning: "오버랩이 청크 크기에 비해 큽니다 — 청크가 대부분의 콘텐츠를 공유합니다.", + advancedLabel: "고급 옵션", + tokenLimitLabel: "청크당 토큰 제한", + tokenLimitDescription: "근사 토큰 수로 청크 크기를 제한합니다. 설정되면 토큰 예산을 유지하기 위해 청크 크기가 자동으로 축소됩니다. 0 = 끄기 (문자 크기만 사용).", + languagesLabel: "언어 힌트", + languagesDescription: "구조 감지 패턴이 찾아야 할 언어를 힌트로 제공합니다. 자동 감지를 위해 비워두세요.", + languagesPlaceholder: "자동 감지", + languageOptions: { + de: "독일어", + en: "영어", + zh: "중국어" + } }, multimodal: { title: "이미지 처리 설정", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 6e9f6ccf..29c7612a 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2130,7 +2130,40 @@ export default { parentChunkSizeLabel: 'Размер родительского блока', parentChunkSizeDescription: 'Размер родительских блоков для контекста (256-4096)', childChunkSizeLabel: 'Размер дочернего блока', - childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)' + childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)', + strategyLabel: 'Стратегия разбиения', + strategyDescription: 'Выберите способ разбиения документов на блоки. Автоматический режим анализирует каждый документ и выбирает оптимальную стратегию.', + strategyPlaceholder: 'Выберите стратегию (по умолчанию классическое рекурсивное разбиение)', + strategies: { + auto: { + label: 'Автоматически (рекомендуется)', + tooltip: 'Профилировщик документа анализирует структуру и выбирает заголовочное, эвристическое или классическое разбиение для каждой загрузки.' + }, + heading: { + label: 'Оптимизировано для Markdown', + tooltip: 'Разбивает преимущественно по заголовкам Markdown (#, ##, ###). Подходит для чистого Markdown с явной структурой разделов.' + }, + heuristic: { + label: 'Умное определение структуры', + tooltip: 'Определяет разрывы страниц, нумерованные разделы, многоязычные маркеры глав (DE/EN/ZH), заголовки в верхнем регистре. Идеально для PDF без Markdown.' + }, + legacy: { + label: 'Классический', + tooltip: 'Классическое рекурсивное разбиение только по разделителям — оригинальное поведение. Используйте, если новые стратегии работают некорректно.' + } + }, + overlapWarning: 'Перекрытие велико по сравнению с размером блока — блоки будут содержать большую часть одинакового контента.', + advancedLabel: 'Расширенные параметры', + tokenLimitLabel: 'Лимит токенов на блок', + tokenLimitDescription: 'Ограничьте размер блока приблизительным количеством токенов. При установке размер блока автоматически уменьшается, чтобы оставаться в пределах лимита. 0 = выключено (только символьный размер).', + languagesLabel: 'Языковые подсказки', + languagesDescription: 'Подсказка о том, какие языки должны искать паттерны определения структуры. Оставьте пустым для автоопределения.', + languagesPlaceholder: 'Автоопределение', + languageOptions: { + de: 'Немецкий', + en: 'Английский', + zh: 'Китайский' + } }, multimodal: { title: 'Обработка изображений', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 94edd23e..93f1e911 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2487,6 +2487,39 @@ export default { parentChunkSizeDescription: "提供上下文的父块字符数(256-4096)", childChunkSizeLabel: "子块大小", childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)", + strategyLabel: "分块策略", + strategyDescription: "选择文档的分块方式。自动模式会分析每个文档的结构并选择最佳策略。", + strategyPlaceholder: "选择策略(默认使用经典递归分块)", + strategies: { + auto: { + label: "自动(推荐)", + tooltip: "文档分析器自动选择标题感知、结构检测或经典分块。" + }, + heading: { + label: "Markdown 优化", + tooltip: "主要按 Markdown 标题(#、##、###)切分。适合结构清晰的 Markdown。" + }, + heuristic: { + label: "智能结构检测", + tooltip: "检测分页符、编号章节、多语言章节标记(DE/EN/ZH)、全大写标题。适用于无 Markdown 结构的 PDF。" + }, + legacy: { + label: "经典", + tooltip: "仅按分隔符进行经典递归分块——原始行为。如果新策略效果不佳请使用此项。" + } + }, + overlapWarning: "重叠相对于分块大小较大——分块之间会共享大部分内容。", + advancedLabel: "高级选项", + tokenLimitLabel: "每块 Token 上限", + tokenLimitDescription: "按近似 Token 数限制分块大小。设置后会自动缩小分块以保持在 Token 预算内。0 = 关闭(仅按字符数)。", + languagesLabel: "语言提示", + languagesDescription: "提示结构检测模式应识别哪些语言。留空则自动检测。", + languagesPlaceholder: "自动检测", + languageOptions: { + de: "德语", + en: "英语", + zh: "中文" + } }, multimodal: { title: "图像处理配置", diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 63f09fad..2f868190 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -490,7 +490,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => { parserEngineRules: undefined as any, enableParentChild: true, parentChunkSize: 4096, - childChunkSize: 384 + childChunkSize: 384, + // New KBs default to the adaptive auto-strategy. User can change in the UI. + strategy: 'auto' as string, + tokenLimit: 0, + languages: [] as string[] }, storageProvider: '' as string, multimodalConfig: { @@ -587,7 +591,12 @@ const loadKBData = async () => { parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined, enableParentChild: kb.chunking_config?.enable_parent_child || false, parentChunkSize: kb.chunking_config?.parent_chunk_size || 4096, - childChunkSize: kb.chunking_config?.child_chunk_size || 384 + childChunkSize: kb.chunking_config?.child_chunk_size || 384, + // Existing KBs without strategy field render as empty (= legacy behavior). + // The user has to actively pick a value to opt in to the new tiers. + strategy: kb.chunking_config?.strategy || '', + tokenLimit: kb.chunking_config?.token_limit || 0, + languages: kb.chunking_config?.languages || [] }, storageProvider: (kb.storage_provider_config?.provider || kb.storage_config?.provider || 'local') as string, multimodalConfig: { @@ -840,6 +849,17 @@ const buildSubmitData = () => { enable_parent_child: formData.value.chunkingConfig.enableParentChild, parent_chunk_size: formData.value.chunkingConfig.parentChunkSize, child_chunk_size: formData.value.chunkingConfig.childChunkSize, + // Adaptive chunking fields. Empty/zero values are omitted on the + // backend side via omitempty; we send them through unconditionally. + ...(formData.value.chunkingConfig.strategy + ? { strategy: formData.value.chunkingConfig.strategy } + : {}), + ...(formData.value.chunkingConfig.tokenLimit + ? { token_limit: formData.value.chunkingConfig.tokenLimit } + : {}), + ...(formData.value.chunkingConfig.languages?.length + ? { languages: formData.value.chunkingConfig.languages } + : {}), ...(formData.value.chunkingConfig.parserEngineRules?.length ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } : {}) @@ -1023,7 +1043,11 @@ const doSubmit = async () => { parserEngineRules: data.chunking_config.parser_engine_rules || undefined, enableParentChild: data.chunking_config.enable_parent_child || false, parentChunkSize: data.chunking_config.parent_chunk_size || 4096, - childChunkSize: data.chunking_config.child_chunk_size || 384 + childChunkSize: data.chunking_config.child_chunk_size || 384, + // Adaptive chunking — empty / 0 / [] means "leave server default". + strategy: data.chunking_config.strategy || undefined, + tokenLimit: data.chunking_config.token_limit || undefined, + languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined }, multimodal: { enabled: !!data.vlm_config?.enabled diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index 93773e8d..8b1e2558 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -6,6 +6,32 @@
+ +
+
+ +

{{ $t('knowledgeEditor.chunking.strategyDescription') }}

+
+
+ +
+
+ + +
+

+ {{ currentStrategyInfo.label }}: + {{ currentStrategyInfo.tooltip }} +

+
+
@@ -33,6 +59,7 @@

{{ $t('knowledgeEditor.chunking.overlapDescription') }}

+

{{ $t('knowledgeEditor.chunking.overlapWarning') }}

@@ -127,6 +154,52 @@
+ + +
+ ▸ + {{ $t('knowledgeEditor.chunking.advancedLabel') }} +
+ +
+ +
+
+ +

{{ $t('knowledgeEditor.chunking.tokenLimitDescription') }}

+
+
+ +
+
+ + +
+
+ +

{{ $t('knowledgeEditor.chunking.languagesDescription') }}

+
+
+ +
+
+
@@ -148,6 +221,12 @@ interface ChunkingConfig { enableParentChild: boolean parentChunkSize: number childChunkSize: number + // New: adaptive chunking strategy. Empty string = legacy / not set. + strategy?: string + // New: cap chunk size in approx tokens. 0 = char-based budget only. + tokenLimit?: number + // New: language hints for heuristic patterns (de/en/zh). + languages?: string[] } interface Props { @@ -160,13 +239,60 @@ const emit = defineEmits<{ 'update:config': [value: ChunkingConfig] }>() +const { t } = useI18n() + const localChunkSize = ref(props.config.chunkSize) const localChunkOverlap = ref(props.config.chunkOverlap) const localSeparators = ref([...props.config.separators]) const localEnableParentChild = ref(props.config.enableParentChild ?? false) const localParentChunkSize = ref(props.config.parentChunkSize || 4096) const localChildChunkSize = ref(props.config.childChunkSize || 384) -const { t } = useI18n() +const localStrategy = ref(props.config.strategy ?? '') +const localTokenLimit = ref(props.config.tokenLimit ?? 0) +const localLanguages = ref([...(props.config.languages ?? [])]) +const advancedOpen = ref(false) + +const strategyOptions = computed(() => [ + { + label: t('knowledgeEditor.chunking.strategies.auto.label'), + value: 'auto', + tooltip: t('knowledgeEditor.chunking.strategies.auto.tooltip') + }, + { + label: t('knowledgeEditor.chunking.strategies.heading.label'), + value: 'heading', + tooltip: t('knowledgeEditor.chunking.strategies.heading.tooltip') + }, + { + label: t('knowledgeEditor.chunking.strategies.heuristic.label'), + value: 'heuristic', + tooltip: t('knowledgeEditor.chunking.strategies.heuristic.tooltip') + }, + { + label: t('knowledgeEditor.chunking.strategies.legacy.label'), + value: 'legacy', + tooltip: t('knowledgeEditor.chunking.strategies.legacy.tooltip') + } +]) + +const currentStrategyInfo = computed(() => { + if (!localStrategy.value) { + return null + } + return strategyOptions.value.find(o => o.value === localStrategy.value) ?? null +}) + +const advancedDisabled = computed(() => localStrategy.value === 'legacy') + +const overlapTooHigh = computed( + () => localChunkOverlap.value > 0 && localChunkOverlap.value >= localChunkSize.value / 2 +) + +const languageOptions = computed(() => [ + { label: t('knowledgeEditor.chunking.languageOptions.de'), value: 'de' }, + { label: t('knowledgeEditor.chunking.languageOptions.en'), value: 'en' }, + { label: t('knowledgeEditor.chunking.languageOptions.zh'), value: 'zh' } +]) const separatorOptions = computed(() => [ { label: t('knowledgeEditor.chunking.separators.doubleNewline'), value: '\n\n' }, @@ -186,6 +312,9 @@ watch(() => props.config, (newConfig) => { localEnableParentChild.value = newConfig.enableParentChild ?? false localParentChunkSize.value = newConfig.parentChunkSize || 4096 localChildChunkSize.value = newConfig.childChunkSize || 384 + localStrategy.value = newConfig.strategy ?? '' + localTokenLimit.value = newConfig.tokenLimit ?? 0 + localLanguages.value = [...(newConfig.languages ?? [])] }, { deep: true }) const handleChunkSizeChange = () => { emitUpdate() } @@ -194,6 +323,9 @@ const handleSeparatorsChange = () => { emitUpdate() } const handleParentChildChange = () => { emitUpdate() } const handleParentChunkSizeChange = () => { emitUpdate() } const handleChildChunkSizeChange = () => { emitUpdate() } +const handleStrategyChange = () => { emitUpdate() } +const handleTokenLimitChange = () => { emitUpdate() } +const handleLanguagesChange = () => { emitUpdate() } const emitUpdate = () => { emit('update:config', { @@ -203,7 +335,10 @@ const emitUpdate = () => { parserEngineRules: props.config.parserEngineRules, enableParentChild: localEnableParentChild.value, parentChunkSize: localParentChunkSize.value, - childChunkSize: localChildChunkSize.value + childChunkSize: localChildChunkSize.value, + strategy: localStrategy.value, + tokenLimit: localTokenLimit.value, + languages: localLanguages.value }) } @@ -247,6 +382,37 @@ const emitUpdate = () => { &:last-child { border-bottom: none; } + + &.disabled { + opacity: 0.5; + } +} + +.strategy-row { + // Strategy is the most prominent setting — slight emphasis. + background: linear-gradient(to right, var(--td-bg-color-container-hover), transparent); + padding-left: 12px; + margin-left: -12px; + border-radius: 6px; +} + +.strategy-info-panel { + margin: -8px 0 12px 12px; + padding: 10px 14px; + background: var(--td-bg-color-container-hover); + border-left: 3px solid var(--td-brand-color); + border-radius: 0 4px 4px 0; + font-size: 13px; + color: var(--td-text-color-secondary); + line-height: 1.5; + + p { + margin: 0; + } + + strong { + color: var(--td-text-color-primary); + } } .setting-info { @@ -268,6 +434,13 @@ const emitUpdate = () => { margin: 0; line-height: 1.5; } + + .warn { + font-size: 12px; + color: var(--td-warning-color); + margin: 4px 0 0 0; + line-height: 1.4; + } } .setting-control { @@ -293,4 +466,35 @@ const emitUpdate = () => { min-width: 80px; text-align: right; } + +.advanced-toggle { + display: flex; + align-items: center; + gap: 8px; + padding: 16px 0 8px 0; + cursor: pointer; + font-size: 14px; + font-weight: 500; + color: var(--td-text-color-secondary); + user-select: none; + + &:hover { + color: var(--td-text-color-primary); + } +} + +.toggle-arrow { + display: inline-block; + transition: transform 0.15s ease; + font-size: 12px; + + &.open { + transform: rotate(90deg); + } +} + +.advanced-section { + padding-left: 12px; + border-left: 2px solid var(--td-component-stroke); +}