diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index e0c168be..5c5fec4c 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -50,6 +50,14 @@ export interface InitializationConfig { chunkSize: number; chunkOverlap: number; separators: string[]; + // Adaptive chunking strategy. Empty / "legacy" = classic recursive splitter. + // "auto" lets the backend profiler pick a tier; "heading" / "heuristic" + // pin the tier explicitly. See backend chunker package for details. + strategy?: string; + // Cap chunk size in approx tokens. 0 = char-based budget only. + tokenLimit?: number; + // Language hints for heuristic patterns ("de", "en", "zh"). Empty = auto-detect. + languages?: string[]; }; // Frontend-only hint for storage selection UI storageType?: 'cos' | 'minio'; @@ -94,6 +102,13 @@ export interface KBModelConfigRequest { enableParentChild?: boolean parentChunkSize?: number childChunkSize?: number + // Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy"). + // Empty string keeps the existing backend default (legacy / no change). + strategy?: string + // Approximate token budget per chunk; 0 = char-based. + tokenLimit?: number + // Language hints for heuristic patterns. Empty = auto-detect. + languages?: string[] } multimodal: { enabled: boolean diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 7c954498..87272316 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1918,7 +1918,40 @@ export default { parentChunkSizeLabel: 'Parent Chunk Size', parentChunkSizeDescription: 'Size of parent chunks that provide context (256-4096)', childChunkSizeLabel: 'Child Chunk Size', - childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)' + childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)', + strategyLabel: 'Chunking Strategy', + strategyDescription: 'Choose how documents are split into chunks. The Automatic mode profiles each document and picks the best strategy.', + strategyPlaceholder: 'Select strategy (defaults to classic recursive splitting)', + strategies: { + auto: { + label: 'Automatic (recommended)', + tooltip: 'A document profiler analyzes structure and picks heading-aware, smart-detection or classic splitting per upload.' + }, + heading: { + label: 'Markdown-optimized', + tooltip: 'Splits primarily at Markdown headings (#, ##, ###). Best for clean Markdown with explicit section structure.' + }, + heuristic: { + label: 'Smart structure detection', + tooltip: 'Detects page-breaks, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps titles. Ideal for PDFs without proper Markdown.' + }, + legacy: { + label: 'Classic', + tooltip: 'Classic recursive splitting on separators only — the original behavior. Use this if the new strategies misbehave on your content.' + } + }, + overlapWarning: 'Overlap is large compared to chunk size — chunks will share most of their content.', + advancedLabel: 'Advanced options', + tokenLimitLabel: 'Token limit per chunk', + tokenLimitDescription: 'Cap chunk size in approximate tokens. When set, chunk size is automatically reduced to stay below this token budget. 0 = off (use character size only).', + languagesLabel: 'Language hints', + languagesDescription: 'Hint which languages structure-detection patterns should look for. Leave empty for auto-detection.', + languagesPlaceholder: 'Auto-detect', + languageOptions: { + de: 'German', + en: 'English', + zh: 'Chinese' + } }, multimodal: { title: 'Image Processing Configuration', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 283ffa5c..4a111acf 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2528,6 +2528,39 @@ export default { parentChunkSizeDescription: "컨텍스트를 제공하는 부모 청크의 문자 수 (256-4096)", childChunkSizeLabel: "자식 청크 크기", childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)", + strategyLabel: "청킹 전략", + strategyDescription: "문서를 청크로 분할하는 방법을 선택합니다. 자동 모드는 문서를 프로파일링하여 최적의 전략을 선택합니다.", + strategyPlaceholder: "전략 선택 (기본: 클래식 재귀 분할)", + strategies: { + auto: { + label: "자동 (권장)", + tooltip: "문서 프로파일러가 구조를 분석하여 헤딩 인식, 스마트 감지 또는 클래식 분할을 선택합니다." + }, + heading: { + label: "Markdown 최적화", + tooltip: "주로 Markdown 헤딩(#, ##, ###)에서 분할합니다. 명확한 섹션 구조의 Markdown에 적합합니다." + }, + heuristic: { + label: "스마트 구조 감지", + tooltip: "페이지 구분자, 번호 매겨진 섹션, 다국어 챕터 마커(DE/EN/ZH), 대문자 제목을 감지합니다. Markdown 구조가 없는 PDF에 적합합니다." + }, + legacy: { + label: "클래식", + tooltip: "구분자만 사용한 클래식 재귀 분할 — 원래 동작. 새 전략이 콘텐츠에서 잘못 작동하는 경우 사용하세요." + } + }, + overlapWarning: "오버랩이 청크 크기에 비해 큽니다 — 청크가 대부분의 콘텐츠를 공유합니다.", + advancedLabel: "고급 옵션", + tokenLimitLabel: "청크당 토큰 제한", + tokenLimitDescription: "근사 토큰 수로 청크 크기를 제한합니다. 설정되면 토큰 예산을 유지하기 위해 청크 크기가 자동으로 축소됩니다. 0 = 끄기 (문자 크기만 사용).", + languagesLabel: "언어 힌트", + languagesDescription: "구조 감지 패턴이 찾아야 할 언어를 힌트로 제공합니다. 자동 감지를 위해 비워두세요.", + languagesPlaceholder: "자동 감지", + languageOptions: { + de: "독일어", + en: "영어", + zh: "중국어" + } }, multimodal: { title: "이미지 처리 설정", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 6e9f6ccf..29c7612a 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2130,7 +2130,40 @@ export default { parentChunkSizeLabel: 'Размер родительского блока', parentChunkSizeDescription: 'Размер родительских блоков для контекста (256-4096)', childChunkSizeLabel: 'Размер дочернего блока', - childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)' + childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)', + strategyLabel: 'Стратегия разбиения', + strategyDescription: 'Выберите способ разбиения документов на блоки. Автоматический режим анализирует каждый документ и выбирает оптимальную стратегию.', + strategyPlaceholder: 'Выберите стратегию (по умолчанию классическое рекурсивное разбиение)', + strategies: { + auto: { + label: 'Автоматически (рекомендуется)', + tooltip: 'Профилировщик документа анализирует структуру и выбирает заголовочное, эвристическое или классическое разбиение для каждой загрузки.' + }, + heading: { + label: 'Оптимизировано для Markdown', + tooltip: 'Разбивает преимущественно по заголовкам Markdown (#, ##, ###). Подходит для чистого Markdown с явной структурой разделов.' + }, + heuristic: { + label: 'Умное определение структуры', + tooltip: 'Определяет разрывы страниц, нумерованные разделы, многоязычные маркеры глав (DE/EN/ZH), заголовки в верхнем регистре. Идеально для PDF без Markdown.' + }, + legacy: { + label: 'Классический', + tooltip: 'Классическое рекурсивное разбиение только по разделителям — оригинальное поведение. Используйте, если новые стратегии работают некорректно.' + } + }, + overlapWarning: 'Перекрытие велико по сравнению с размером блока — блоки будут содержать большую часть одинакового контента.', + advancedLabel: 'Расширенные параметры', + tokenLimitLabel: 'Лимит токенов на блок', + tokenLimitDescription: 'Ограничьте размер блока приблизительным количеством токенов. При установке размер блока автоматически уменьшается, чтобы оставаться в пределах лимита. 0 = выключено (только символьный размер).', + languagesLabel: 'Языковые подсказки', + languagesDescription: 'Подсказка о том, какие языки должны искать паттерны определения структуры. Оставьте пустым для автоопределения.', + languagesPlaceholder: 'Автоопределение', + languageOptions: { + de: 'Немецкий', + en: 'Английский', + zh: 'Китайский' + } }, multimodal: { title: 'Обработка изображений', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 94edd23e..93f1e911 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2487,6 +2487,39 @@ export default { parentChunkSizeDescription: "提供上下文的父块字符数(256-4096)", childChunkSizeLabel: "子块大小", childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)", + strategyLabel: "分块策略", + strategyDescription: "选择文档的分块方式。自动模式会分析每个文档的结构并选择最佳策略。", + strategyPlaceholder: "选择策略(默认使用经典递归分块)", + strategies: { + auto: { + label: "自动(推荐)", + tooltip: "文档分析器自动选择标题感知、结构检测或经典分块。" + }, + heading: { + label: "Markdown 优化", + tooltip: "主要按 Markdown 标题(#、##、###)切分。适合结构清晰的 Markdown。" + }, + heuristic: { + label: "智能结构检测", + tooltip: "检测分页符、编号章节、多语言章节标记(DE/EN/ZH)、全大写标题。适用于无 Markdown 结构的 PDF。" + }, + legacy: { + label: "经典", + tooltip: "仅按分隔符进行经典递归分块——原始行为。如果新策略效果不佳请使用此项。" + } + }, + overlapWarning: "重叠相对于分块大小较大——分块之间会共享大部分内容。", + advancedLabel: "高级选项", + tokenLimitLabel: "每块 Token 上限", + tokenLimitDescription: "按近似 Token 数限制分块大小。设置后会自动缩小分块以保持在 Token 预算内。0 = 关闭(仅按字符数)。", + languagesLabel: "语言提示", + languagesDescription: "提示结构检测模式应识别哪些语言。留空则自动检测。", + languagesPlaceholder: "自动检测", + languageOptions: { + de: "德语", + en: "英语", + zh: "中文" + } }, multimodal: { title: "图像处理配置", diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 63f09fad..2f868190 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -490,7 +490,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => { parserEngineRules: undefined as any, enableParentChild: true, parentChunkSize: 4096, - childChunkSize: 384 + childChunkSize: 384, + // New KBs default to the adaptive auto-strategy. User can change in the UI. + strategy: 'auto' as string, + tokenLimit: 0, + languages: [] as string[] }, storageProvider: '' as string, multimodalConfig: { @@ -587,7 +591,12 @@ const loadKBData = async () => { parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined, enableParentChild: kb.chunking_config?.enable_parent_child || false, parentChunkSize: kb.chunking_config?.parent_chunk_size || 4096, - childChunkSize: kb.chunking_config?.child_chunk_size || 384 + childChunkSize: kb.chunking_config?.child_chunk_size || 384, + // Existing KBs without strategy field render as empty (= legacy behavior). + // The user has to actively pick a value to opt in to the new tiers. + strategy: kb.chunking_config?.strategy || '', + tokenLimit: kb.chunking_config?.token_limit || 0, + languages: kb.chunking_config?.languages || [] }, storageProvider: (kb.storage_provider_config?.provider || kb.storage_config?.provider || 'local') as string, multimodalConfig: { @@ -840,6 +849,17 @@ const buildSubmitData = () => { enable_parent_child: formData.value.chunkingConfig.enableParentChild, parent_chunk_size: formData.value.chunkingConfig.parentChunkSize, child_chunk_size: formData.value.chunkingConfig.childChunkSize, + // Adaptive chunking fields. Empty/zero values are omitted on the + // backend side via omitempty; we send them through unconditionally. + ...(formData.value.chunkingConfig.strategy + ? { strategy: formData.value.chunkingConfig.strategy } + : {}), + ...(formData.value.chunkingConfig.tokenLimit + ? { token_limit: formData.value.chunkingConfig.tokenLimit } + : {}), + ...(formData.value.chunkingConfig.languages?.length + ? { languages: formData.value.chunkingConfig.languages } + : {}), ...(formData.value.chunkingConfig.parserEngineRules?.length ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } : {}) @@ -1023,7 +1043,11 @@ const doSubmit = async () => { parserEngineRules: data.chunking_config.parser_engine_rules || undefined, enableParentChild: data.chunking_config.enable_parent_child || false, parentChunkSize: data.chunking_config.parent_chunk_size || 4096, - childChunkSize: data.chunking_config.child_chunk_size || 384 + childChunkSize: data.chunking_config.child_chunk_size || 384, + // Adaptive chunking — empty / 0 / [] means "leave server default". + strategy: data.chunking_config.strategy || undefined, + tokenLimit: data.chunking_config.token_limit || undefined, + languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined }, multimodal: { enabled: !!data.vlm_config?.enabled diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index 93773e8d..8b1e2558 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -6,6 +6,32 @@
{{ $t('knowledgeEditor.chunking.strategyDescription') }}
++ {{ currentStrategyInfo.label }}: + {{ currentStrategyInfo.tooltip }} +
+{{ $t('knowledgeEditor.chunking.overlapDescription') }}
+{{ $t('knowledgeEditor.chunking.overlapWarning') }}
{{ $t('knowledgeEditor.chunking.tokenLimitDescription') }}
+{{ $t('knowledgeEditor.chunking.languagesDescription') }}
+