feat: update chunk size configurations for knowledge base processing
- Increased default sizes for parent and child chunks in the knowledge base editor and settings, enhancing data processing capabilities. - Adjusted slider settings in the UI to reflect new minimum and maximum values for chunk sizes, improving user control over configurations. - Updated backend logic to ensure consistent handling of chunk sizes across the application. These changes optimize the chunking process, allowing for more efficient data handling and improved performance in knowledge base operations.
This commit is contained in:
@@ -298,8 +298,8 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
||||
separators: ['\n\n', '\n', '。', '!', '?', ';', ';'],
|
||||
parserEngineRules: undefined as any,
|
||||
enableParentChild: false,
|
||||
parentChunkSize: 1024,
|
||||
childChunkSize: 256
|
||||
parentChunkSize: 4096,
|
||||
childChunkSize: 384
|
||||
},
|
||||
storageProvider: 'local' as string,
|
||||
multimodalConfig: {
|
||||
@@ -378,8 +378,8 @@ const loadKBData = async () => {
|
||||
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
|
||||
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined,
|
||||
enableParentChild: kb.chunking_config?.enable_parent_child || false,
|
||||
parentChunkSize: kb.chunking_config?.parent_chunk_size || 1024,
|
||||
childChunkSize: kb.chunking_config?.child_chunk_size || 256
|
||||
parentChunkSize: kb.chunking_config?.parent_chunk_size || 4096,
|
||||
childChunkSize: kb.chunking_config?.child_chunk_size || 384
|
||||
},
|
||||
storageProvider: (kb.storage_config?.provider || 'local') as string,
|
||||
multimodalConfig: {
|
||||
@@ -614,8 +614,8 @@ const handleSubmit = async () => {
|
||||
separators: data.chunking_config.separators,
|
||||
parserEngineRules: data.chunking_config.parser_engine_rules || undefined,
|
||||
enableParentChild: data.chunking_config.enable_parent_child || false,
|
||||
parentChunkSize: data.chunking_config.parent_chunk_size || 1024,
|
||||
childChunkSize: data.chunking_config.child_chunk_size || 256
|
||||
parentChunkSize: data.chunking_config.parent_chunk_size || 4096,
|
||||
childChunkSize: data.chunking_config.child_chunk_size || 384
|
||||
},
|
||||
multimodal: {
|
||||
enabled: !!data.vlm_config?.enabled
|
||||
|
||||
@@ -94,10 +94,10 @@
|
||||
<div class="slider-container">
|
||||
<t-slider
|
||||
v-model="localParentChunkSize"
|
||||
:min="256"
|
||||
:max="4096"
|
||||
:min="512"
|
||||
:max="8192"
|
||||
:step="64"
|
||||
:marks="{ 256: '256', 1024: '1024', 2048: '2048', 4096: '4096' }"
|
||||
:marks="{ 512: '512', 2048: '2048', 4096: '4096', 8192: '8192' }"
|
||||
@change="handleParentChunkSizeChange"
|
||||
style="width: 200px;"
|
||||
/>
|
||||
@@ -117,9 +117,9 @@
|
||||
<t-slider
|
||||
v-model="localChildChunkSize"
|
||||
:min="64"
|
||||
:max="1024"
|
||||
:max="2048"
|
||||
:step="32"
|
||||
:marks="{ 64: '64', 256: '256', 512: '512', 1024: '1024' }"
|
||||
:marks="{ 64: '64', 384: '384', 1024: '1024', 2048: '2048' }"
|
||||
@change="handleChildChunkSizeChange"
|
||||
style="width: 200px;"
|
||||
/>
|
||||
@@ -164,8 +164,8 @@ const localChunkSize = ref(props.config.chunkSize)
|
||||
const localChunkOverlap = ref(props.config.chunkOverlap)
|
||||
const localSeparators = ref([...props.config.separators])
|
||||
const localEnableParentChild = ref(props.config.enableParentChild ?? false)
|
||||
const localParentChunkSize = ref(props.config.parentChunkSize || 1024)
|
||||
const localChildChunkSize = ref(props.config.childChunkSize || 256)
|
||||
const localParentChunkSize = ref(props.config.parentChunkSize || 4096)
|
||||
const localChildChunkSize = ref(props.config.childChunkSize || 384)
|
||||
const { t } = useI18n()
|
||||
|
||||
const separatorOptions = computed(() => [
|
||||
@@ -184,8 +184,8 @@ watch(() => props.config, (newConfig) => {
|
||||
localChunkOverlap.value = newConfig.chunkOverlap
|
||||
localSeparators.value = [...newConfig.separators]
|
||||
localEnableParentChild.value = newConfig.enableParentChild ?? false
|
||||
localParentChunkSize.value = newConfig.parentChunkSize || 1024
|
||||
localChildChunkSize.value = newConfig.childChunkSize || 256
|
||||
localParentChunkSize.value = newConfig.parentChunkSize || 4096
|
||||
localChildChunkSize.value = newConfig.childChunkSize || 384
|
||||
}, { deep: true })
|
||||
|
||||
const handleChunkSizeChange = () => { emitUpdate() }
|
||||
|
||||
@@ -1326,11 +1326,11 @@ type ProcessChunksOptions struct {
|
||||
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
|
||||
parentSize := cc.ParentChunkSize
|
||||
if parentSize <= 0 {
|
||||
parentSize = 1024
|
||||
parentSize = 4096
|
||||
}
|
||||
childSize := cc.ChildChunkSize
|
||||
if childSize <= 0 {
|
||||
childSize = 256
|
||||
childSize = 384
|
||||
}
|
||||
parent = chunker.SplitterConfig{
|
||||
ChunkSize: parentSize,
|
||||
|
||||
@@ -125,10 +125,10 @@ type ChunkingConfig struct {
|
||||
// When enabled, large parent chunks provide context while small child chunks
|
||||
// are used for vector matching. Retrieval matches on child but returns parent content.
|
||||
EnableParentChild bool `yaml:"enable_parent_child,omitempty" json:"enable_parent_child,omitempty"`
|
||||
// ParentChunkSize is the size of parent chunks (default: 1024).
|
||||
// ParentChunkSize is the size of parent chunks (default: 4096).
|
||||
// Only used when EnableParentChild is true.
|
||||
ParentChunkSize int `yaml:"parent_chunk_size,omitempty" json:"parent_chunk_size,omitempty"`
|
||||
// ChildChunkSize is the size of child chunks used for embedding (default: 256).
|
||||
// ChildChunkSize is the size of child chunks used for embedding (default: 384).
|
||||
// Only used when EnableParentChild is true.
|
||||
ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user