feat(frontend): expose chunking strategy in KB editor

Surfaces the adaptive 3-tier chunking controls in the existing
"Chunking" section of the KB editor modal. Users can now pick a
strategy from a dropdown without crafting JSON for the API:

- Automatic (recommended)  → backend profiler picks the tier
- Markdown-optimized       → forces heading-aware splitting
- Smart structure detection → forces heuristic boundary detection
- Classic                  → original recursive splitter (legacy)

The fifth backend value `recursive` is intentionally hidden from the
UI — it's functionally near `legacy` and would dilute the choice.

Advanced options (collapsible, disabled when strategy=legacy):
- TokenLimit (0–8192 approx tokens; 0 = char-based budget)
- Languages multi-select (DE/EN/ZH; empty = auto-detect)

Behaviour:
- New KBs default to strategy="auto"
- Existing KBs without the field render with empty strategy and keep
  classic behavior on save unless the user actively picks one
- Overlap warning when overlap >= chunkSize/2

Wired through:
- KBChunkingSettings.vue: strategy select + advanced collapsible +
  ChunkingConfig interface extended
- KnowledgeBaseEditorModal.vue: form-state defaults, kb-load mapping,
  buildSubmitData (chunking_config) + updateKBConfig (documentSplitting)
  pass strategy/token_limit/languages
- api/initialization/index.ts: types extended
- i18n: en-US / zh-CN / ko-KR / ru-RU all carry strategy + advanced
  keys (6 keys each)

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 20:40:07 +00:00
committed by lyingbug
parent 2feab191bb
commit 46d9c396c6
7 changed files with 382 additions and 7 deletions
+15
View File
@@ -50,6 +50,14 @@ export interface InitializationConfig {
chunkSize: number;
chunkOverlap: number;
separators: string[];
// Adaptive chunking strategy. Empty / "legacy" = classic recursive splitter.
// "auto" lets the backend profiler pick a tier; "heading" / "heuristic"
// pin the tier explicitly. See backend chunker package for details.
strategy?: string;
// Cap chunk size in approx tokens. 0 = char-based budget only.
tokenLimit?: number;
// Language hints for heuristic patterns ("de", "en", "zh"). Empty = auto-detect.
languages?: string[];
};
// Frontend-only hint for storage selection UI
storageType?: 'cos' | 'minio';
@@ -94,6 +102,13 @@ export interface KBModelConfigRequest {
enableParentChild?: boolean
parentChunkSize?: number
childChunkSize?: number
// Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy").
// Empty string keeps the existing backend default (legacy / no change).
strategy?: string
// Approximate token budget per chunk; 0 = char-based.
tokenLimit?: number
// Language hints for heuristic patterns. Empty = auto-detect.
languages?: string[]
}
multimodal: {
enabled: boolean
+34 -1
View File
@@ -1918,7 +1918,40 @@ export default {
parentChunkSizeLabel: 'Parent Chunk Size',
parentChunkSizeDescription: 'Size of parent chunks that provide context (256-4096)',
childChunkSizeLabel: 'Child Chunk Size',
childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)'
childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)',
strategyLabel: 'Chunking Strategy',
strategyDescription: 'Choose how documents are split into chunks. The Automatic mode profiles each document and picks the best strategy.',
strategyPlaceholder: 'Select strategy (defaults to classic recursive splitting)',
strategies: {
auto: {
label: 'Automatic (recommended)',
tooltip: 'A document profiler analyzes structure and picks heading-aware, smart-detection or classic splitting per upload.'
},
heading: {
label: 'Markdown-optimized',
tooltip: 'Splits primarily at Markdown headings (#, ##, ###). Best for clean Markdown with explicit section structure.'
},
heuristic: {
label: 'Smart structure detection',
tooltip: 'Detects page-breaks, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps titles. Ideal for PDFs without proper Markdown.'
},
legacy: {
label: 'Classic',
tooltip: 'Classic recursive splitting on separators only — the original behavior. Use this if the new strategies misbehave on your content.'
}
},
overlapWarning: 'Overlap is large compared to chunk size — chunks will share most of their content.',
advancedLabel: 'Advanced options',
tokenLimitLabel: 'Token limit per chunk',
tokenLimitDescription: 'Cap chunk size in approximate tokens. When set, chunk size is automatically reduced to stay below this token budget. 0 = off (use character size only).',
languagesLabel: 'Language hints',
languagesDescription: 'Hint which languages structure-detection patterns should look for. Leave empty for auto-detection.',
languagesPlaceholder: 'Auto-detect',
languageOptions: {
de: 'German',
en: 'English',
zh: 'Chinese'
}
},
multimodal: {
title: 'Image Processing Configuration',
+33
View File
@@ -2528,6 +2528,39 @@ export default {
parentChunkSizeDescription: "컨텍스트를 제공하는 부모 청크의 문자 수 (256-4096)",
childChunkSizeLabel: "자식 청크 크기",
childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)",
strategyLabel: "청킹 전략",
strategyDescription: "문서를 청크로 분할하는 방법을 선택합니다. 자동 모드는 문서를 프로파일링하여 최적의 전략을 선택합니다.",
strategyPlaceholder: "전략 선택 (기본: 클래식 재귀 분할)",
strategies: {
auto: {
label: "자동 (권장)",
tooltip: "문서 프로파일러가 구조를 분석하여 헤딩 인식, 스마트 감지 또는 클래식 분할을 선택합니다."
},
heading: {
label: "Markdown 최적화",
tooltip: "주로 Markdown 헤딩(#, ##, ###)에서 분할합니다. 명확한 섹션 구조의 Markdown에 적합합니다."
},
heuristic: {
label: "스마트 구조 감지",
tooltip: "페이지 구분자, 번호 매겨진 섹션, 다국어 챕터 마커(DE/EN/ZH), 대문자 제목을 감지합니다. Markdown 구조가 없는 PDF에 적합합니다."
},
legacy: {
label: "클래식",
tooltip: "구분자만 사용한 클래식 재귀 분할 — 원래 동작. 새 전략이 콘텐츠에서 잘못 작동하는 경우 사용하세요."
}
},
overlapWarning: "오버랩이 청크 크기에 비해 큽니다 — 청크가 대부분의 콘텐츠를 공유합니다.",
advancedLabel: "고급 옵션",
tokenLimitLabel: "청크당 토큰 제한",
tokenLimitDescription: "근사 토큰 수로 청크 크기를 제한합니다. 설정되면 토큰 예산을 유지하기 위해 청크 크기가 자동으로 축소됩니다. 0 = 끄기 (문자 크기만 사용).",
languagesLabel: "언어 힌트",
languagesDescription: "구조 감지 패턴이 찾아야 할 언어를 힌트로 제공합니다. 자동 감지를 위해 비워두세요.",
languagesPlaceholder: "자동 감지",
languageOptions: {
de: "독일어",
en: "영어",
zh: "중국어"
}
},
multimodal: {
title: "이미지 처리 설정",
+34 -1
View File
@@ -2130,7 +2130,40 @@ export default {
parentChunkSizeLabel: 'Размер родительского блока',
parentChunkSizeDescription: 'Размер родительских блоков для контекста (256-4096)',
childChunkSizeLabel: 'Размер дочернего блока',
childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)'
childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)',
strategyLabel: 'Стратегия разбиения',
strategyDescription: 'Выберите способ разбиения документов на блоки. Автоматический режим анализирует каждый документ и выбирает оптимальную стратегию.',
strategyPlaceholder: 'Выберите стратегию (по умолчанию классическое рекурсивное разбиение)',
strategies: {
auto: {
label: 'Автоматически (рекомендуется)',
tooltip: 'Профилировщик документа анализирует структуру и выбирает заголовочное, эвристическое или классическое разбиение для каждой загрузки.'
},
heading: {
label: 'Оптимизировано для Markdown',
tooltip: 'Разбивает преимущественно по заголовкам Markdown (#, ##, ###). Подходит для чистого Markdown с явной структурой разделов.'
},
heuristic: {
label: 'Умное определение структуры',
tooltip: 'Определяет разрывы страниц, нумерованные разделы, многоязычные маркеры глав (DE/EN/ZH), заголовки в верхнем регистре. Идеально для PDF без Markdown.'
},
legacy: {
label: 'Классический',
tooltip: 'Классическое рекурсивное разбиение только по разделителям — оригинальное поведение. Используйте, если новые стратегии работают некорректно.'
}
},
overlapWarning: 'Перекрытие велико по сравнению с размером блока — блоки будут содержать большую часть одинакового контента.',
advancedLabel: 'Расширенные параметры',
tokenLimitLabel: 'Лимит токенов на блок',
tokenLimitDescription: 'Ограничьте размер блока приблизительным количеством токенов. При установке размер блока автоматически уменьшается, чтобы оставаться в пределах лимита. 0 = выключено (только символьный размер).',
languagesLabel: 'Языковые подсказки',
languagesDescription: 'Подсказка о том, какие языки должны искать паттерны определения структуры. Оставьте пустым для автоопределения.',
languagesPlaceholder: 'Автоопределение',
languageOptions: {
de: 'Немецкий',
en: 'Английский',
zh: 'Китайский'
}
},
multimodal: {
title: 'Обработка изображений',
+33
View File
@@ -2487,6 +2487,39 @@ export default {
parentChunkSizeDescription: "提供上下文的父块字符数(256-4096)",
childChunkSizeLabel: "子块大小",
childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)",
strategyLabel: "分块策略",
strategyDescription: "选择文档的分块方式。自动模式会分析每个文档的结构并选择最佳策略。",
strategyPlaceholder: "选择策略(默认使用经典递归分块)",
strategies: {
auto: {
label: "自动(推荐)",
tooltip: "文档分析器自动选择标题感知、结构检测或经典分块。"
},
heading: {
label: "Markdown 优化",
tooltip: "主要按 Markdown 标题(#、##、###)切分。适合结构清晰的 Markdown。"
},
heuristic: {
label: "智能结构检测",
tooltip: "检测分页符、编号章节、多语言章节标记(DE/EN/ZH)、全大写标题。适用于无 Markdown 结构的 PDF。"
},
legacy: {
label: "经典",
tooltip: "仅按分隔符进行经典递归分块——原始行为。如果新策略效果不佳请使用此项。"
}
},
overlapWarning: "重叠相对于分块大小较大——分块之间会共享大部分内容。",
advancedLabel: "高级选项",
tokenLimitLabel: "每块 Token 上限",
tokenLimitDescription: "按近似 Token 数限制分块大小。设置后会自动缩小分块以保持在 Token 预算内。0 = 关闭(仅按字符数)。",
languagesLabel: "语言提示",
languagesDescription: "提示结构检测模式应识别哪些语言。留空则自动检测。",
languagesPlaceholder: "自动检测",
languageOptions: {
de: "德语",
en: "英语",
zh: "中文"
}
},
multimodal: {
title: "图像处理配置",
@@ -490,7 +490,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
parserEngineRules: undefined as any,
enableParentChild: true,
parentChunkSize: 4096,
childChunkSize: 384
childChunkSize: 384,
// New KBs default to the adaptive auto-strategy. User can change in the UI.
strategy: 'auto' as string,
tokenLimit: 0,
languages: [] as string[]
},
storageProvider: '' as string,
multimodalConfig: {
@@ -587,7 +591,12 @@ const loadKBData = async () => {
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined,
enableParentChild: kb.chunking_config?.enable_parent_child || false,
parentChunkSize: kb.chunking_config?.parent_chunk_size || 4096,
childChunkSize: kb.chunking_config?.child_chunk_size || 384
childChunkSize: kb.chunking_config?.child_chunk_size || 384,
// Existing KBs without strategy field render as empty (= legacy behavior).
// The user has to actively pick a value to opt in to the new tiers.
strategy: kb.chunking_config?.strategy || '',
tokenLimit: kb.chunking_config?.token_limit || 0,
languages: kb.chunking_config?.languages || []
},
storageProvider: (kb.storage_provider_config?.provider || kb.storage_config?.provider || 'local') as string,
multimodalConfig: {
@@ -840,6 +849,17 @@ const buildSubmitData = () => {
enable_parent_child: formData.value.chunkingConfig.enableParentChild,
parent_chunk_size: formData.value.chunkingConfig.parentChunkSize,
child_chunk_size: formData.value.chunkingConfig.childChunkSize,
// Adaptive chunking fields. Empty/zero values are omitted on the
// backend side via omitempty; we send them through unconditionally.
...(formData.value.chunkingConfig.strategy
? { strategy: formData.value.chunkingConfig.strategy }
: {}),
...(formData.value.chunkingConfig.tokenLimit
? { token_limit: formData.value.chunkingConfig.tokenLimit }
: {}),
...(formData.value.chunkingConfig.languages?.length
? { languages: formData.value.chunkingConfig.languages }
: {}),
...(formData.value.chunkingConfig.parserEngineRules?.length
? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules }
: {})
@@ -1023,7 +1043,11 @@ const doSubmit = async () => {
parserEngineRules: data.chunking_config.parser_engine_rules || undefined,
enableParentChild: data.chunking_config.enable_parent_child || false,
parentChunkSize: data.chunking_config.parent_chunk_size || 4096,
childChunkSize: data.chunking_config.child_chunk_size || 384
childChunkSize: data.chunking_config.child_chunk_size || 384,
// Adaptive chunking — empty / 0 / [] means "leave server default".
strategy: data.chunking_config.strategy || undefined,
tokenLimit: data.chunking_config.token_limit || undefined,
languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined
},
multimodal: {
enabled: !!data.vlm_config?.enabled
@@ -6,6 +6,32 @@
</div>
<div class="settings-group">
<!-- Strategy -->
<div class="setting-row strategy-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.strategyLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.strategyDescription') }}</p>
</div>
<div class="setting-control">
<t-select
v-model="localStrategy"
:options="strategyOptions"
:placeholder="$t('knowledgeEditor.chunking.strategyPlaceholder')"
:clearable="true"
@change="handleStrategyChange"
style="width: 280px;"
/>
</div>
</div>
<!-- Strategy explanation panel -->
<div v-if="currentStrategyInfo" class="strategy-info-panel">
<p>
<strong>{{ currentStrategyInfo.label }}:</strong>
{{ currentStrategyInfo.tooltip }}
</p>
</div>
<!-- Chunk Size -->
<div class="setting-row">
<div class="setting-info">
@@ -33,6 +59,7 @@
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.overlapLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.overlapDescription') }}</p>
<p v-if="overlapTooHigh" class="warn">{{ $t('knowledgeEditor.chunking.overlapWarning') }}</p>
</div>
<div class="setting-control">
<div class="slider-container">
@@ -127,6 +154,52 @@
</div>
</div>
</div>
<!-- Advanced section toggle -->
<div class="advanced-toggle" @click="advancedOpen = !advancedOpen">
<span class="toggle-arrow" :class="{ 'open': advancedOpen }">▸</span>
<span>{{ $t('knowledgeEditor.chunking.advancedLabel') }}</span>
</div>
<div v-if="advancedOpen" class="advanced-section">
<!-- Token Limit -->
<div class="setting-row" :class="{ disabled: advancedDisabled }">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.tokenLimitLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.tokenLimitDescription') }}</p>
</div>
<div class="setting-control">
<t-input-number
v-model="localTokenLimit"
:min="0"
:max="8192"
:step="64"
:disabled="advancedDisabled"
@change="handleTokenLimitChange"
style="width: 200px;"
/>
</div>
</div>
<!-- Languages -->
<div class="setting-row" :class="{ disabled: advancedDisabled }">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.languagesLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.languagesDescription') }}</p>
</div>
<div class="setting-control">
<t-select
v-model="localLanguages"
:options="languageOptions"
multiple
:disabled="advancedDisabled"
:placeholder="$t('knowledgeEditor.chunking.languagesPlaceholder')"
@change="handleLanguagesChange"
style="width: 280px;"
/>
</div>
</div>
</div>
</div>
</div>
</template>
@@ -148,6 +221,12 @@ interface ChunkingConfig {
enableParentChild: boolean
parentChunkSize: number
childChunkSize: number
// New: adaptive chunking strategy. Empty string = legacy / not set.
strategy?: string
// New: cap chunk size in approx tokens. 0 = char-based budget only.
tokenLimit?: number
// New: language hints for heuristic patterns (de/en/zh).
languages?: string[]
}
interface Props {
@@ -160,13 +239,60 @@ const emit = defineEmits<{
'update:config': [value: ChunkingConfig]
}>()
const { t } = useI18n()
const localChunkSize = ref(props.config.chunkSize)
const localChunkOverlap = ref(props.config.chunkOverlap)
const localSeparators = ref([...props.config.separators])
const localEnableParentChild = ref(props.config.enableParentChild ?? false)
const localParentChunkSize = ref(props.config.parentChunkSize || 4096)
const localChildChunkSize = ref(props.config.childChunkSize || 384)
const { t } = useI18n()
const localStrategy = ref(props.config.strategy ?? '')
const localTokenLimit = ref(props.config.tokenLimit ?? 0)
const localLanguages = ref<string[]>([...(props.config.languages ?? [])])
const advancedOpen = ref(false)
const strategyOptions = computed(() => [
{
label: t('knowledgeEditor.chunking.strategies.auto.label'),
value: 'auto',
tooltip: t('knowledgeEditor.chunking.strategies.auto.tooltip')
},
{
label: t('knowledgeEditor.chunking.strategies.heading.label'),
value: 'heading',
tooltip: t('knowledgeEditor.chunking.strategies.heading.tooltip')
},
{
label: t('knowledgeEditor.chunking.strategies.heuristic.label'),
value: 'heuristic',
tooltip: t('knowledgeEditor.chunking.strategies.heuristic.tooltip')
},
{
label: t('knowledgeEditor.chunking.strategies.legacy.label'),
value: 'legacy',
tooltip: t('knowledgeEditor.chunking.strategies.legacy.tooltip')
}
])
const currentStrategyInfo = computed(() => {
if (!localStrategy.value) {
return null
}
return strategyOptions.value.find(o => o.value === localStrategy.value) ?? null
})
const advancedDisabled = computed(() => localStrategy.value === 'legacy')
const overlapTooHigh = computed(
() => localChunkOverlap.value > 0 && localChunkOverlap.value >= localChunkSize.value / 2
)
const languageOptions = computed(() => [
{ label: t('knowledgeEditor.chunking.languageOptions.de'), value: 'de' },
{ label: t('knowledgeEditor.chunking.languageOptions.en'), value: 'en' },
{ label: t('knowledgeEditor.chunking.languageOptions.zh'), value: 'zh' }
])
const separatorOptions = computed(() => [
{ label: t('knowledgeEditor.chunking.separators.doubleNewline'), value: '\n\n' },
@@ -186,6 +312,9 @@ watch(() => props.config, (newConfig) => {
localEnableParentChild.value = newConfig.enableParentChild ?? false
localParentChunkSize.value = newConfig.parentChunkSize || 4096
localChildChunkSize.value = newConfig.childChunkSize || 384
localStrategy.value = newConfig.strategy ?? ''
localTokenLimit.value = newConfig.tokenLimit ?? 0
localLanguages.value = [...(newConfig.languages ?? [])]
}, { deep: true })
const handleChunkSizeChange = () => { emitUpdate() }
@@ -194,6 +323,9 @@ const handleSeparatorsChange = () => { emitUpdate() }
const handleParentChildChange = () => { emitUpdate() }
const handleParentChunkSizeChange = () => { emitUpdate() }
const handleChildChunkSizeChange = () => { emitUpdate() }
const handleStrategyChange = () => { emitUpdate() }
const handleTokenLimitChange = () => { emitUpdate() }
const handleLanguagesChange = () => { emitUpdate() }
const emitUpdate = () => {
emit('update:config', {
@@ -203,7 +335,10 @@ const emitUpdate = () => {
parserEngineRules: props.config.parserEngineRules,
enableParentChild: localEnableParentChild.value,
parentChunkSize: localParentChunkSize.value,
childChunkSize: localChildChunkSize.value
childChunkSize: localChildChunkSize.value,
strategy: localStrategy.value,
tokenLimit: localTokenLimit.value,
languages: localLanguages.value
})
}
</script>
@@ -247,6 +382,37 @@ const emitUpdate = () => {
&:last-child {
border-bottom: none;
}
&.disabled {
opacity: 0.5;
}
}
.strategy-row {
// Strategy is the most prominent setting — slight emphasis.
background: linear-gradient(to right, var(--td-bg-color-container-hover), transparent);
padding-left: 12px;
margin-left: -12px;
border-radius: 6px;
}
.strategy-info-panel {
margin: -8px 0 12px 12px;
padding: 10px 14px;
background: var(--td-bg-color-container-hover);
border-left: 3px solid var(--td-brand-color);
border-radius: 0 4px 4px 0;
font-size: 13px;
color: var(--td-text-color-secondary);
line-height: 1.5;
p {
margin: 0;
}
strong {
color: var(--td-text-color-primary);
}
}
.setting-info {
@@ -268,6 +434,13 @@ const emitUpdate = () => {
margin: 0;
line-height: 1.5;
}
.warn {
font-size: 12px;
color: var(--td-warning-color);
margin: 4px 0 0 0;
line-height: 1.4;
}
}
.setting-control {
@@ -293,4 +466,35 @@ const emitUpdate = () => {
min-width: 80px;
text-align: right;
}
.advanced-toggle {
display: flex;
align-items: center;
gap: 8px;
padding: 16px 0 8px 0;
cursor: pointer;
font-size: 14px;
font-weight: 500;
color: var(--td-text-color-secondary);
user-select: none;
&:hover {
color: var(--td-text-color-primary);
}
}
.toggle-arrow {
display: inline-block;
transition: transform 0.15s ease;
font-size: 12px;
&.open {
transform: rotate(90deg);
}
}
.advanced-section {
padding-left: 12px;
border-left: 2px solid var(--td-component-stroke);
}
</style>