feat: implement parent-child chunking strategy for enhanced context management

- Added support for parent-child chunking in the knowledge base, allowing for a two-level chunking strategy that improves context retrieval.
- Introduced new configuration options for enabling parent-child chunking, including parent and child chunk sizes.
- Updated relevant components and services to handle parent chunk data, ensuring proper linking and retrieval of child chunks.
- Enhanced user interface elements to reflect the new chunking options and provide better user guidance.

These changes significantly improve the knowledge extraction process by providing richer context through hierarchical chunking.
This commit is contained in:
wizardchen
2026-03-04 20:35:10 +08:00
committed by lyingbug
parent 8399527022
commit b333499078
22 changed files with 1100 additions and 403 deletions
+3
View File
@@ -83,6 +83,9 @@ export interface KBModelConfigRequest {
chunkOverlap: number chunkOverlap: number
separators: string[] separators: string[]
parserEngineRules?: { file_types: string[]; engine: string }[] parserEngineRules?: { file_types: string[]; engine: string }[]
enableParentChild?: boolean
parentChunkSize?: number
childChunkSize?: number
} }
multimodal: { multimodal: {
enabled: boolean enabled: boolean
+132 -17
View File
@@ -4,8 +4,8 @@ import { marked } from "marked";
import hljs from "highlight.js"; import hljs from "highlight.js";
import "highlight.js/styles/github.css"; import "highlight.js/styles/github.css";
import mermaid from "mermaid"; import mermaid from "mermaid";
import { onMounted, ref, nextTick, onUnmounted, onUpdated, watch } from "vue"; import { onMounted, ref, nextTick, onUnmounted, watch } from "vue";
import { downKnowledgeDetails, deleteGeneratedQuestion } from "@/api/knowledge-base/index"; import { downKnowledgeDetails, deleteGeneratedQuestion, getChunkByIdOnly } from "@/api/knowledge-base/index";
import { MessagePlugin, DialogPlugin } from "tdesign-vue-next"; import { MessagePlugin, DialogPlugin } from "tdesign-vue-next";
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL, hydrateProtectedFileImages } from '@/utils/security'; import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL, hydrateProtectedFileImages } from '@/utils/security';
import { openMermaidFullscreen } from '@/utils/mermaidViewer'; import { openMermaidFullscreen } from '@/utils/mermaidViewer';
@@ -267,24 +267,39 @@ const isMarkdownFile = (fileType?: string): boolean => {
const markdownTypes = ['md', 'markdown']; const markdownTypes = ['md', 'markdown'];
return markdownTypes.includes(fileType.toLowerCase()); return markdownTypes.includes(fileType.toLowerCase());
}; };
const runMarkdownPostRenderPipeline = async () => {
await nextTick();
const renderRoot = mdContentWrap.value as ParentNode;
await hydrateProtectedFileImages(renderRoot);
const images = renderRoot?.querySelectorAll?.('img.markdown-image') as NodeListOf<HTMLImageElement> | undefined;
if (images) {
images.forEach(async item => {
const isValid = await checkImage(item.src);
if (!isValid) {
item.remove();
}
})
}
// 渲染 Mermaid 图表
await renderMermaidDiagrams();
};
watch(() => props.details.md, (newVal) => { watch(() => props.details.md, (newVal) => {
nextTick(async () => { runMarkdownPostRenderPipeline();
const renderRoot = (doc as ParentNode) || mdContentWrap.value;
await hydrateProtectedFileImages(renderRoot);
const images = renderRoot?.querySelectorAll?.('img.markdown-image') as NodeListOf<HTMLImageElement> | undefined;
if (images) {
images.forEach(async item => {
const isValid = await checkImage(item.src);
if (!isValid) {
item.remove();
}
})
}
// 渲染 Mermaid 图表
await renderMermaidDiagrams();
})
}, { immediate: true, deep: true }) }, { immediate: true, deep: true })
watch(() => viewMode.value, (mode) => {
if ((mode === 'chunks' || mode === 'merged') && props.visible) {
runMarkdownPostRenderPipeline();
}
});
watch(() => props.visible, (visible) => {
if (visible && (viewMode.value === 'chunks' || viewMode.value === 'merged')) {
runMarkdownPostRenderPipeline();
}
});
// 渲染 Mermaid 图表的函数 // 渲染 Mermaid 图表的函数
const renderMermaidDiagrams = async () => { const renderMermaidDiagrams = async () => {
try { try {
@@ -557,6 +572,50 @@ const isDeleting = (chunkIndex: number, questionId: string) => {
return deletingQuestion.value?.chunkIndex === chunkIndex && deletingQuestion.value?.questionId === questionId; return deletingQuestion.value?.chunkIndex === chunkIndex && deletingQuestion.value?.questionId === questionId;
}; };
// 父 Chunk 上下文展开状态
const parentContextExpanded = ref<Set<number>>(new Set());
const parentContextCache = ref<Map<string, string>>(new Map());
const parentContextLoading = ref<Set<number>>(new Set());
const hasParentChunk = (item: any) => !!item?.parent_chunk_id;
const isParentExpanded = (index: number) => parentContextExpanded.value.has(index);
const toggleParentContext = async (item: any, index: number) => {
if (parentContextExpanded.value.has(index)) {
parentContextExpanded.value.delete(index);
parentContextExpanded.value = new Set(parentContextExpanded.value);
return;
}
const parentId = item.parent_chunk_id;
if (!parentContextCache.value.has(parentId)) {
parentContextLoading.value.add(index);
parentContextLoading.value = new Set(parentContextLoading.value);
try {
const result: any = await getChunkByIdOnly(parentId);
if (result.success && result.data) {
parentContextCache.value.set(parentId, result.data.content || '');
parentContextCache.value = new Map(parentContextCache.value);
}
} catch (err) {
MessagePlugin.error(t('knowledgeBase.parentContextLoadFailed') || '加载父上下文失败');
return;
} finally {
parentContextLoading.value.delete(index);
parentContextLoading.value = new Set(parentContextLoading.value);
}
}
parentContextExpanded.value.add(index);
parentContextExpanded.value = new Set(parentContextExpanded.value);
await runMarkdownPostRenderPipeline();
};
const getParentContent = (item: any) => {
return parentContextCache.value.get(item.parent_chunk_id) || '';
};
const downloadFile = () => { const downloadFile = () => {
downKnowledgeDetails(props.details.id) downKnowledgeDetails(props.details.id)
.then((result) => { .then((result) => {
@@ -703,6 +762,14 @@ const handleDetailsScroll = () => {
<div class="chunk-header"> <div class="chunk-header">
<span class="chunk-index">{{ $t('knowledgeBase.segment') || '片段' }} {{ index + 1 }}</span> <span class="chunk-index">{{ $t('knowledgeBase.segment') || '片段' }} {{ index + 1 }}</span>
<div class="chunk-header-right"> <div class="chunk-header-right">
<t-tag
v-if="hasParentChunk(item)"
size="small"
theme="primary"
variant="light"
>
{{ $t('knowledgeBase.childChunk') || '子块' }}
</t-tag>
<t-tag <t-tag
v-if="getGeneratedQuestions(item).length > 0" v-if="getGeneratedQuestions(item).length > 0"
size="small" size="small"
@@ -716,6 +783,18 @@ const handleDetailsScroll = () => {
</div> </div>
<div class="md-content" v-html="processMarkdown(item.content)"></div> <div class="md-content" v-html="processMarkdown(item.content)"></div>
<!-- 父 Chunk 上下文展开 -->
<div v-if="hasParentChunk(item)" class="parent-context-section">
<div class="parent-context-toggle" @click="toggleParentContext(item, index)">
<t-icon v-if="!parentContextLoading.has(index)" :name="isParentExpanded(index) ? 'chevron-down' : 'chevron-right'" size="14px" />
<t-loading v-else size="small" style="width: 14px; height: 14px;" />
<span>{{ $t('knowledgeBase.viewParentContext') || '查看父块上下文' }}</span>
</div>
<div v-show="isParentExpanded(index)" class="parent-context-content">
<div class="md-content" v-html="processMarkdown(getParentContent(item))"></div>
</div>
</div>
<!-- 生成的问题展示 --> <!-- 生成的问题展示 -->
<div v-if="getGeneratedQuestions(item).length > 0" class="questions-section"> <div v-if="getGeneratedQuestions(item).length > 0" class="questions-section">
<div class="questions-toggle" @click="toggleQuestions(index)"> <div class="questions-toggle" @click="toggleQuestions(index)">
@@ -1058,6 +1137,42 @@ const handleDetailsScroll = () => {
} }
} }
// 父 Chunk 上下文样式
.parent-context-section {
margin-top: 10px;
padding-top: 8px;
border-top: 1px dashed #d0d5dd;
}
.parent-context-toggle {
display: flex;
align-items: center;
gap: 6px;
cursor: pointer;
color: #07c05f;
font-size: 12px;
font-weight: 500;
padding: 4px 0;
transition: color 0.2s ease;
&:hover {
color: #07c05f;
}
}
.parent-context-content {
margin-top: 8px;
padding: 10px 12px;
background: #f0f5ff;
border-radius: 4px;
border-left: 3px solid #07c05f;
.md-content {
color: #4e5969;
font-size: 13px;
}
}
// 生成的问题样式 // 生成的问题样式
.questions-section { .questions-section {
margin-top: 12px; margin-top: 12px;
+10 -1
View File
@@ -142,6 +142,9 @@ export default {
loadOriginalFailed: 'Failed to load original file content', loadOriginalFailed: 'Failed to load original file content',
questions: 'Questions', questions: 'Questions',
generatedQuestions: 'Generated Questions', generatedQuestions: 'Generated Questions',
childChunk: 'Child Chunk',
viewParentContext: 'View Parent Context',
parentContextLoadFailed: 'Failed to load parent context',
confirmDeleteQuestion: 'Are you sure you want to delete this question? The corresponding vector index will also be removed.', confirmDeleteQuestion: 'Are you sure you want to delete this question? The corresponding vector index will also be removed.',
legacyQuestionCannotDelete: 'Legacy format questions cannot be deleted. Please regenerate questions.', legacyQuestionCannotDelete: 'Legacy format questions cannot be deleted. Please regenerate questions.',
notInitialized: 'Knowledge base is not initialized. Please configure models in settings before uploading files', notInitialized: 'Knowledge base is not initialized. Please configure models in settings before uploading files',
@@ -1415,7 +1418,13 @@ export default {
semicolonCn: 'Chinese semicolon (;)', semicolonCn: 'Chinese semicolon (;)',
semicolonEn: 'Semicolon (;)', semicolonEn: 'Semicolon (;)',
space: 'Space ( )' space: 'Space ( )'
} },
parentChildLabel: 'Parent-Child Chunking',
parentChildDescription: 'Enable two-level parent-child chunking strategy. Large parent chunks provide context while small child chunks are used for vector matching.',
parentChunkSizeLabel: 'Parent Chunk Size',
parentChunkSizeDescription: 'Size of parent chunks that provide context (256-4096)',
childChunkSizeLabel: 'Child Chunk Size',
childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)'
}, },
advanced: { advanced: {
title: 'Advanced Settings', title: 'Advanced Settings',
+9
View File
@@ -143,6 +143,9 @@ export default {
loadOriginalFailed: "원본 파일 내용 로드 실패", loadOriginalFailed: "원본 파일 내용 로드 실패",
questions: "질문", questions: "질문",
generatedQuestions: "생성된 질문", generatedQuestions: "생성된 질문",
childChunk: "자식 청크",
viewParentContext: "부모 컨텍스트 보기",
parentContextLoadFailed: "부모 컨텍스트 로드 실패",
confirmDeleteQuestion: confirmDeleteQuestion:
"이 질문을 삭제하시겠습니까? 삭제 시 해당 벡터 인덱스도 함께 제거됩니다.", "이 질문을 삭제하시겠습니까? 삭제 시 해당 벡터 인덱스도 함께 제거됩니다.",
legacyQuestionCannotDelete: "이전 형식의 질문은 삭제할 수 없습니다. 질문을 다시 생성하세요", legacyQuestionCannotDelete: "이전 형식의 질문은 삭제할 수 없습니다. 질문을 다시 생성하세요",
@@ -1860,6 +1863,12 @@ export default {
semicolonEn: "영어 세미콜론 (;)", semicolonEn: "영어 세미콜론 (;)",
space: "공백 ( )", space: "공백 ( )",
}, },
parentChildLabel: "부모-자식 청킹",
parentChildDescription: "2단계 부모-자식 청킹 전략을 활성화합니다. 큰 부모 청크는 컨텍스트를 제공하고, 작은 자식 청크는 벡터 매칭에 사용됩니다.",
parentChunkSizeLabel: "부모 청크 크기",
parentChunkSizeDescription: "컨텍스트를 제공하는 부모 청크의 문자 수 (256-4096)",
childChunkSizeLabel: "자식 청크 크기",
childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)",
}, },
advanced: { advanced: {
title: "고급 설정", title: "고급 설정",
+10 -1
View File
@@ -118,6 +118,9 @@ export default {
loadOriginalFailed: 'Не удалось загрузить содержимое исходного файла', loadOriginalFailed: 'Не удалось загрузить содержимое исходного файла',
questions: 'Вопросы', questions: 'Вопросы',
generatedQuestions: 'Сгенерированные вопросы', generatedQuestions: 'Сгенерированные вопросы',
childChunk: 'Дочерний блок',
viewParentContext: 'Просмотр родительского контекста',
parentContextLoadFailed: 'Не удалось загрузить родительский контекст',
confirmDeleteQuestion: 'Вы уверены, что хотите удалить этот вопрос? Соответствующий векторный индекс также будет удален.', confirmDeleteQuestion: 'Вы уверены, что хотите удалить этот вопрос? Соответствующий векторный индекс также будет удален.',
legacyQuestionCannotDelete: 'Вопросы в устаревшем формате нельзя удалить. Пожалуйста, сгенерируйте вопросы заново.', legacyQuestionCannotDelete: 'Вопросы в устаревшем формате нельзя удалить. Пожалуйста, сгенерируйте вопросы заново.',
notInitialized: 'База знаний не инициализирована. Пожалуйста, настройте модели в разделе настроек перед загрузкой файлов', notInitialized: 'База знаний не инициализирована. Пожалуйста, настройте модели в разделе настроек перед загрузкой файлов',
@@ -1253,7 +1256,13 @@ export default {
semicolonCn: 'Китайская точка с запятой (;)', semicolonCn: 'Китайская точка с запятой (;)',
semicolonEn: 'Точка с запятой (;)', semicolonEn: 'Точка с запятой (;)',
space: 'Пробел ( )' space: 'Пробел ( )'
} },
parentChildLabel: 'Родительско-дочернее разбиение',
parentChildDescription: 'Включить двухуровневую стратегию разбиения. Большие родительские блоки обеспечивают контекст, а маленькие дочерние блоки используются для векторного поиска.',
parentChunkSizeLabel: 'Размер родительского блока',
parentChunkSizeDescription: 'Размер родительских блоков для контекста (256-4096)',
childChunkSizeLabel: 'Размер дочернего блока',
childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)'
}, },
advanced: { advanced: {
title: 'Расширенные настройки', title: 'Расширенные настройки',
+9
View File
@@ -141,6 +141,9 @@ export default {
loadOriginalFailed: "加载原文件内容失败", loadOriginalFailed: "加载原文件内容失败",
questions: "问题", questions: "问题",
generatedQuestions: "生成的问题", generatedQuestions: "生成的问题",
childChunk: "子块",
viewParentContext: "查看父块上下文",
parentContextLoadFailed: "加载父上下文失败",
confirmDeleteQuestion: "确定要删除这个问题吗?删除后将同时移除对应的向量索引。", confirmDeleteQuestion: "确定要删除这个问题吗?删除后将同时移除对应的向量索引。",
legacyQuestionCannotDelete: "旧格式问题无法删除,请重新生成问题", legacyQuestionCannotDelete: "旧格式问题无法删除,请重新生成问题",
docActionUnsupported: "当前知识库类型不支持该操作", docActionUnsupported: "当前知识库类型不支持该操作",
@@ -1820,6 +1823,12 @@ export default {
semicolonEn: "英文分号 (;)", semicolonEn: "英文分号 (;)",
space: "空格 ( )", space: "空格 ( )",
}, },
parentChildLabel: "父子分块",
parentChildDescription: "启用两级父子分块策略。大的父块提供上下文,小的子块用于向量匹配检索。",
parentChunkSizeLabel: "父块大小",
parentChunkSizeDescription: "提供上下文的父块字符数(256-4096)",
childChunkSizeLabel: "子块大小",
childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)",
}, },
advanced: { advanced: {
title: "高级设置", title: "高级设置",
@@ -296,7 +296,10 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
chunkSize: 512, chunkSize: 512,
chunkOverlap: 100, chunkOverlap: 100,
separators: ['\n\n', '\n', '。', '!', '?', ';', ';'], separators: ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: undefined as any parserEngineRules: undefined as any,
enableParentChild: false,
parentChunkSize: 1024,
childChunkSize: 256
}, },
storageProvider: 'local' as string, storageProvider: 'local' as string,
multimodalConfig: { multimodalConfig: {
@@ -373,7 +376,10 @@ const loadKBData = async () => {
chunkSize: kb.chunking_config?.chunk_size || 512, chunkSize: kb.chunking_config?.chunk_size || 512,
chunkOverlap: kb.chunking_config?.chunk_overlap || 100, chunkOverlap: kb.chunking_config?.chunk_overlap || 100,
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'], separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined,
enableParentChild: kb.chunking_config?.enable_parent_child || false,
parentChunkSize: kb.chunking_config?.parent_chunk_size || 1024,
childChunkSize: kb.chunking_config?.child_chunk_size || 256
}, },
storageProvider: (kb.storage_config?.provider || 'local') as string, storageProvider: (kb.storage_config?.provider || 'local') as string,
multimodalConfig: { multimodalConfig: {
@@ -503,6 +509,9 @@ const buildSubmitData = () => {
chunk_overlap: formData.value.chunkingConfig.chunkOverlap, chunk_overlap: formData.value.chunkingConfig.chunkOverlap,
separators: formData.value.chunkingConfig.separators, separators: formData.value.chunkingConfig.separators,
enable_multimodal: formData.value.multimodalConfig.enabled, enable_multimodal: formData.value.multimodalConfig.enabled,
enable_parent_child: formData.value.chunkingConfig.enableParentChild,
parent_chunk_size: formData.value.chunkingConfig.parentChunkSize,
child_chunk_size: formData.value.chunkingConfig.childChunkSize,
...(formData.value.chunkingConfig.parserEngineRules?.length ...(formData.value.chunkingConfig.parserEngineRules?.length
? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules }
: {}) : {})
@@ -603,7 +612,10 @@ const handleSubmit = async () => {
chunkSize: data.chunking_config.chunk_size, chunkSize: data.chunking_config.chunk_size,
chunkOverlap: data.chunking_config.chunk_overlap, chunkOverlap: data.chunking_config.chunk_overlap,
separators: data.chunking_config.separators, separators: data.chunking_config.separators,
parserEngineRules: data.chunking_config.parser_engine_rules || undefined parserEngineRules: data.chunking_config.parser_engine_rules || undefined,
enableParentChild: data.chunking_config.enable_parent_child || false,
parentChunkSize: data.chunking_config.parent_chunk_size || 1024,
childChunkSize: data.chunking_config.child_chunk_size || 256
}, },
multimodal: { multimodal: {
enabled: !!data.vlm_config?.enabled enabled: !!data.vlm_config?.enabled
@@ -69,6 +69,64 @@
/> />
</div> </div>
</div> </div>
<!-- Parent-Child Chunking -->
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.parentChildLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.parentChildDescription') }}</p>
</div>
<div class="setting-control">
<t-switch
v-model="localEnableParentChild"
@change="handleParentChildChange"
/>
</div>
</div>
<!-- Parent Chunk Size -->
<div v-if="localEnableParentChild" class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.parentChunkSizeLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.parentChunkSizeDescription') }}</p>
</div>
<div class="setting-control">
<div class="slider-container">
<t-slider
v-model="localParentChunkSize"
:min="256"
:max="4096"
:step="64"
:marks="{ 256: '256', 1024: '1024', 2048: '2048', 4096: '4096' }"
@change="handleParentChunkSizeChange"
style="width: 200px;"
/>
<span class="value-display">{{ localParentChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
</div>
</div>
<!-- Child Chunk Size -->
<div v-if="localEnableParentChild" class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.childChunkSizeLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.childChunkSizeDescription') }}</p>
</div>
<div class="setting-control">
<div class="slider-container">
<t-slider
v-model="localChildChunkSize"
:min="64"
:max="1024"
:step="32"
:marks="{ 64: '64', 256: '256', 512: '512', 1024: '1024' }"
@change="handleChildChunkSizeChange"
style="width: 200px;"
/>
<span class="value-display">{{ localChildChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
</div>
</div>
</div> </div>
</div> </div>
</template> </template>
@@ -87,6 +145,9 @@ interface ChunkingConfig {
chunkOverlap: number chunkOverlap: number
separators: string[] separators: string[]
parserEngineRules?: ParserEngineRule[] parserEngineRules?: ParserEngineRule[]
enableParentChild: boolean
parentChunkSize: number
childChunkSize: number
} }
interface Props { interface Props {
@@ -102,6 +163,9 @@ const emit = defineEmits<{
const localChunkSize = ref(props.config.chunkSize) const localChunkSize = ref(props.config.chunkSize)
const localChunkOverlap = ref(props.config.chunkOverlap) const localChunkOverlap = ref(props.config.chunkOverlap)
const localSeparators = ref([...props.config.separators]) const localSeparators = ref([...props.config.separators])
const localEnableParentChild = ref(props.config.enableParentChild ?? false)
const localParentChunkSize = ref(props.config.parentChunkSize || 1024)
const localChildChunkSize = ref(props.config.childChunkSize || 256)
const { t } = useI18n() const { t } = useI18n()
const separatorOptions = computed(() => [ const separatorOptions = computed(() => [
@@ -119,18 +183,27 @@ watch(() => props.config, (newConfig) => {
localChunkSize.value = newConfig.chunkSize localChunkSize.value = newConfig.chunkSize
localChunkOverlap.value = newConfig.chunkOverlap localChunkOverlap.value = newConfig.chunkOverlap
localSeparators.value = [...newConfig.separators] localSeparators.value = [...newConfig.separators]
localEnableParentChild.value = newConfig.enableParentChild ?? false
localParentChunkSize.value = newConfig.parentChunkSize || 1024
localChildChunkSize.value = newConfig.childChunkSize || 256
}, { deep: true }) }, { deep: true })
const handleChunkSizeChange = () => { emitUpdate() } const handleChunkSizeChange = () => { emitUpdate() }
const handleChunkOverlapChange = () => { emitUpdate() } const handleChunkOverlapChange = () => { emitUpdate() }
const handleSeparatorsChange = () => { emitUpdate() } const handleSeparatorsChange = () => { emitUpdate() }
const handleParentChildChange = () => { emitUpdate() }
const handleParentChunkSizeChange = () => { emitUpdate() }
const handleChildChunkSizeChange = () => { emitUpdate() }
const emitUpdate = () => { const emitUpdate = () => {
emit('update:config', { emit('update:config', {
chunkSize: localChunkSize.value, chunkSize: localChunkSize.value,
chunkOverlap: localChunkOverlap.value, chunkOverlap: localChunkOverlap.value,
separators: localSeparators.value, separators: localSeparators.value,
parserEngineRules: props.config.parserEngineRules parserEngineRules: props.config.parserEngineRules,
enableParentChild: localEnableParentChild.value,
parentChunkSize: localParentChunkSize.value,
childChunkSize: localChildChunkSize.value
}) })
} }
</script> </script>
@@ -3,22 +3,26 @@ package chatpipline
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"sort" "sort"
"strings" "strings"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/types/interfaces"
) )
// PluginMerge handles merging of search result chunks // PluginMerge handles merging of search result chunks
type PluginMerge struct { type PluginMerge struct {
chunkRepo interfaces.ChunkRepository chunkRepo interfaces.ChunkRepository
chunkService interfaces.ChunkService // for parent chunk resolution
} }
// NewPluginMerge creates and registers a new PluginMerge instance // NewPluginMerge creates and registers a new PluginMerge instance
func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository) *PluginMerge { func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository, chunkService interfaces.ChunkService) *PluginMerge {
res := &PluginMerge{ res := &PluginMerge{
chunkRepo: chunkRepo, chunkRepo: chunkRepo,
chunkService: chunkService,
} }
eventManager.Register(res) eventManager.Register(res)
return res return res
@@ -45,6 +49,32 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
"reason": "empty_rerank_result", "reason": "empty_rerank_result",
}) })
searchResult = chatManage.SearchResult searchResult = chatManage.SearchResult
// Sort by score descending so dedup keeps highest-scored entries
sort.Slice(searchResult, func(i, j int) bool {
return searchResult[i].Score > searchResult[j].Score
})
}
// Deduplicate after rerank so higher-scored duplicates are preferred
beforeDedup := len(searchResult)
searchResult = removeDuplicateResults(searchResult)
pipelineInfo(ctx, "Merge", "dedup_summary", map[string]interface{}{
"before": beforeDedup,
"after": len(searchResult),
})
// Inject relevant results from chat history with similarity filtering.
// History references were produced for a previous query, so we only keep
// those that are textually relevant to the current query to avoid injecting
// stale or off-topic context.
historyResults := filterHistoryResults(ctx, chatManage, searchResult)
if len(historyResults) > 0 {
pipelineInfo(ctx, "Merge", "history_inject", map[string]interface{}{
"session_id": chatManage.SessionID,
"history_hits": len(historyResults),
})
searchResult = append(searchResult, historyResults...)
searchResult = removeDuplicateResults(searchResult)
} }
pipelineInfo(ctx, "Merge", "candidate_ready", map[string]interface{}{ pipelineInfo(ctx, "Merge", "candidate_ready", map[string]interface{}{
@@ -60,6 +90,9 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
return next() return next()
} }
// Resolve parent chunks: replace child content with fuller parent content
searchResult = p.resolveParentChunks(ctx, chatManage, searchResult)
// Group chunks by their knowledge source ID // Group chunks by their knowledge source ID
knowledgeGroup := make(map[string]map[string][]*types.SearchResult) knowledgeGroup := make(map[string]map[string][]*types.SearchResult)
for _, chunk := range searchResult { for _, chunk := range searchResult {
@@ -145,6 +178,161 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
return next() return next()
} }
// resolveParentChunks replaces child chunk content with parent chunk content
// for results that have ParentChunkID set. This provides fuller context
// for small child chunks used in parent-child chunking strategy.
func (p *PluginMerge) resolveParentChunks(
ctx context.Context,
chatManage *types.ChatManage,
results []*types.SearchResult,
) []*types.SearchResult {
if len(results) == 0 || p.chunkRepo == nil {
return results
}
tenantID, _ := types.TenantIDFromContext(ctx)
if tenantID == 0 && chatManage != nil {
tenantID = chatManage.TenantID
}
if tenantID == 0 {
pipelineWarn(ctx, "Merge", "parent_resolve_skip", map[string]interface{}{
"reason": "missing_tenant",
})
return results
}
// Collect unique parent chunk IDs
parentIDs := make(map[string]struct{})
for _, r := range results {
if r.ParentChunkID != "" {
parentIDs[r.ParentChunkID] = struct{}{}
}
}
if len(parentIDs) == 0 {
return results
}
// Batch fetch parent chunks
ids := make([]string, 0, len(parentIDs))
for id := range parentIDs {
ids = append(ids, id)
}
parentChunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, ids)
if err != nil {
pipelineWarn(ctx, "Merge", "parent_resolve_failed", map[string]interface{}{
"error": err.Error(),
})
return results
}
parentMap := make(map[string]*types.Chunk, len(parentChunks))
for _, c := range parentChunks {
parentMap[c.ID] = c
}
// Replace child content with parent content
for _, r := range results {
if r.ParentChunkID == "" {
continue
}
parent, ok := parentMap[r.ParentChunkID]
if !ok || parent.Content == "" {
continue
}
pipelineInfo(ctx, "Merge", "parent_resolve", map[string]interface{}{
"child_id": r.ID,
"parent_id": r.ParentChunkID,
"child_len": runeLen(r.Content),
"parent_len": runeLen(parent.Content),
})
r.Content = parent.Content
r.StartAt = parent.StartAt
r.EndAt = parent.EndAt
// Track the original child as a sub-chunk
if !containsID(r.SubChunkID, r.ID) {
r.SubChunkID = append(r.SubChunkID, r.ID)
}
}
return results
}
// filterHistoryResults retrieves history references and filters them by
// textual similarity to the current query. Only references that are above
// a Jaccard similarity threshold are kept, and their scores are discounted
// to reflect that they were not directly retrieved for the current query.
// Results already present in currentResults (by chunk ID) are excluded.
func filterHistoryResults(
ctx context.Context,
chatManage *types.ChatManage,
currentResults []*types.SearchResult,
) []*types.SearchResult {
const (
// minSimilarity is the minimum Jaccard similarity between the current
// query and a history chunk's content for it to be injected.
minSimilarity = 0.15
// historyScoreDiscount reduces the original score of history results
// to rank them below freshly-retrieved results of similar relevance.
historyScoreDiscount = 0.6
// maxHistoryResults caps the number of history results injected to
// avoid overwhelming the context with stale references.
maxHistoryResults = 3
)
raw := getSearchResultFromHistory(chatManage)
if len(raw) == 0 {
return nil
}
// Build a set of chunk IDs already in current results for fast dedup
existingIDs := make(map[string]struct{}, len(currentResults))
for _, r := range currentResults {
existingIDs[r.ID] = struct{}{}
}
// Use RewriteQuery if available (it's the cleaned-up retrieval query),
// otherwise fall back to the original query.
query := chatManage.RewriteQuery
if query == "" {
query = chatManage.Query
}
queryTokens := searchutil.TokenizeSimple(query)
var filtered []*types.SearchResult
for _, r := range raw {
if _, exists := existingIDs[r.ID]; exists {
continue
}
contentTokens := searchutil.TokenizeSimple(r.Content)
sim := searchutil.Jaccard(queryTokens, contentTokens)
if sim < minSimilarity {
pipelineInfo(ctx, "Merge", "history_filter_drop", map[string]interface{}{
"chunk_id": r.ID,
"similarity": sim,
})
continue
}
r.MatchType = types.MatchTypeHistory
r.Score = r.Score * historyScoreDiscount
r.Metadata = ensureMetadata(r.Metadata)
r.Metadata["history_similarity"] = strings.TrimRight(strings.TrimRight(
fmt.Sprintf("%.4f", sim), "0"), ".")
filtered = append(filtered, r)
pipelineInfo(ctx, "Merge", "history_filter_keep", map[string]interface{}{
"chunk_id": r.ID,
"similarity": sim,
"new_score": r.Score,
})
if len(filtered) >= maxHistoryResults {
break
}
}
return filtered
}
// mergeImageInfo 合并两个chunk的ImageInfo // mergeImageInfo 合并两个chunk的ImageInfo
func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error { func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error {
// 如果source没有ImageInfo,不需要合并 // 如果source没有ImageInfo,不需要合并
@@ -0,0 +1,253 @@
package chatpipline
import (
"context"
"regexp"
"strings"
"sync"
"unicode"
"github.com/Tencent/WeKnora/internal/types"
)
// runQueryExpansion performs query expansion when initial recall is low.
// It generates query variants and runs concurrent retrieval across search targets.
func (p *PluginSearch) runQueryExpansion(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{
"current": len(chatManage.SearchResult),
"threshold": chatManage.EmbeddingTopK,
})
expansions := p.expandQueries(ctx, chatManage)
if len(expansions) == 0 {
return nil
}
pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{
"variants": len(expansions),
})
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
expKwTh := chatManage.KeywordThreshold * 0.8
// Concurrent expansion retrieval across queries and search targets
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
var muExp sync.Mutex
var wgExp sync.WaitGroup
jobs := len(expansions) * len(chatManage.SearchTargets)
capSem := 16
if jobs < capSem {
capSem = jobs
}
if capSem <= 0 {
capSem = 1
}
sem := make(chan struct{}, capSem)
pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{
"jobs": jobs,
"cap": capSem,
})
for _, q := range expansions {
for _, target := range chatManage.SearchTargets {
wgExp.Add(1)
go func(q string, t *types.SearchTarget) {
defer wgExp.Done()
sem <- struct{}{}
defer func() { <-sem }()
paramsExp := types.SearchParams{
QueryText: q,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: expKwTh,
MatchCount: expTopK,
DisableVectorMatch: false,
DisableKeywordsMatch: false,
SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage
}
// Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge {
paramsExp.KnowledgeIDs = t.KnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp)
if err != nil {
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"error": err.Error(),
})
return
}
if len(res) > 0 {
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"query": q,
"hits": len(res),
})
muExp.Lock()
expResults = append(expResults, res...)
muExp.Unlock()
}
}(q, target)
}
}
wgExp.Wait()
if len(expResults) > 0 {
pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{
"added": len(expResults),
})
}
return expResults
}
// expandQueries generates query variants locally without LLM to improve keyword recall.
// Uses simple techniques: word reordering, stopword removal, key phrase extraction.
func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string {
query := strings.TrimSpace(chatManage.RewriteQuery)
if query == "" {
return nil
}
expansions := make([]string, 0, 5)
seen := make(map[string]struct{})
seen[strings.ToLower(query)] = struct{}{}
if q := strings.ToLower(chatManage.Query); q != "" {
seen[q] = struct{}{}
}
addIfNew := func(s string) {
s = strings.TrimSpace(s)
if s == "" || len(s) < 3 {
return
}
key := strings.ToLower(s)
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
expansions = append(expansions, s)
}
// 1. Remove common stopwords and create keyword-only variant
keywords := extractKeywords(query)
if len(keywords) >= 2 {
addIfNew(strings.Join(keywords, " "))
}
// 2. Extract quoted phrases or key segments
phrases := extractPhrases(query)
for _, phrase := range phrases {
addIfNew(phrase)
}
// 3. Split by common delimiters and use longest segment
segments := splitByDelimiters(query)
for _, seg := range segments {
if len(seg) > 5 {
addIfNew(seg)
}
}
// 4. Remove question words (什么/如何/怎么/为什么/哪个 etc.)
cleaned := removeQuestionWords(query)
if cleaned != query {
addIfNew(cleaned)
}
// Limit to 5 expansions
if len(expansions) > 5 {
expansions = expansions[:5]
}
pipelineInfo(ctx, "Search", "local_expansion_result", map[string]interface{}{
"variants": len(expansions),
})
return expansions
}
// Common Chinese and English stopwords
var stopwords = map[string]struct{}{
"的": {}, "是": {}, "在": {}, "了": {}, "和": {}, "与": {}, "或": {},
"a": {}, "an": {}, "the": {}, "is": {}, "are": {}, "was": {}, "were": {},
"be": {}, "been": {}, "being": {}, "have": {}, "has": {}, "had": {},
"do": {}, "does": {}, "did": {}, "will": {}, "would": {}, "could": {},
"should": {}, "may": {}, "might": {}, "must": {}, "can": {},
"to": {}, "of": {}, "in": {}, "for": {}, "on": {}, "with": {}, "at": {},
"by": {}, "from": {}, "as": {}, "into": {}, "through": {}, "about": {},
"what": {}, "how": {}, "why": {}, "when": {}, "where": {}, "which": {},
"who": {}, "whom": {}, "whose": {},
}
// Question words in Chinese
var questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`)
func extractKeywords(text string) []string {
words := tokenize(text)
keywords := make([]string, 0, len(words))
for _, w := range words {
lower := strings.ToLower(w)
if _, isStop := stopwords[lower]; !isStop && len(w) > 1 {
keywords = append(keywords, w)
}
}
return keywords
}
func extractPhrases(text string) []string {
// Extract quoted content
var phrases []string
re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)
matches := re.FindAllStringSubmatch(text, -1)
for _, m := range matches {
if len(m) > 1 && len(m[1]) > 2 {
phrases = append(phrases, m[1])
}
}
return phrases
}
func splitByDelimiters(text string) []string {
// Split by common delimiters
re := regexp.MustCompile(`[,,;;、。!?!?\s]+`)
parts := re.Split(text, -1)
var result []string
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
result = append(result, p)
}
}
return result
}
func removeQuestionWords(text string) string {
return strings.TrimSpace(questionWords.ReplaceAllString(text, ""))
}
func tokenize(text string) []string {
var tokens []string
var current strings.Builder
for _, r := range text {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
current.WriteRune(r)
} else if unicode.Is(unicode.Han, r) {
// Flush current token
if current.Len() > 0 {
tokens = append(tokens, current.String())
current.Reset()
}
// Chinese character as single token
tokens = append(tokens, string(r))
} else {
// Delimiter
if current.Len() > 0 {
tokens = append(tokens, current.String())
current.Reset()
}
}
}
if current.Len() > 0 {
tokens = append(tokens, current.String())
}
return tokens
}
@@ -264,6 +264,20 @@ func (p *PluginRerank) rerank(ctx context.Context,
rankFilter = append(rankFilter, result) rankFilter = append(rankFilter, result)
} }
} }
// Fallback: if threshold filtering removed all results, keep top-N as safety net
// This prevents returning empty results when all scores are below threshold
if len(rankFilter) == 0 && len(rerankResp) > 0 {
fallbackN := min(3, len(rerankResp))
rankFilter = rerankResp[:fallbackN]
pipelineInfo(ctx, "Rerank", "fallback_topn", map[string]interface{}{
"reason": "all_below_threshold",
"threshold": chatManage.RerankThreshold,
"fallback_n": fallbackN,
"top_score": rerankResp[0].RelevanceScore,
})
}
return rankFilter return rankFilter
} }
@@ -3,10 +3,8 @@ package chatpipline
import ( import (
"context" "context"
"fmt" "fmt"
"regexp"
"strings" "strings"
"sync" "sync"
"unicode"
"github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/logger"
@@ -126,107 +124,13 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
} }
// If recall is low, attempt query expansion with keyword-focused search // If recall is low, attempt query expansion with keyword-focused search
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK/2) { if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{ expResults := p.runQueryExpansion(ctx, chatManage)
"current": len(chatManage.SearchResult), if len(expResults) > 0 {
"threshold": chatManage.EmbeddingTopK / 2, chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
})
expansions := p.expandQueries(ctx, chatManage)
if len(expansions) > 0 {
pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{
"variants": len(expansions),
})
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
expKwTh := chatManage.KeywordThreshold * 0.8
// Concurrent expansion retrieval across queries and search targets
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
var muExp sync.Mutex
var wgExp sync.WaitGroup
jobs := len(expansions) * len(chatManage.SearchTargets)
capSem := 16
if jobs < capSem {
capSem = jobs
}
if capSem <= 0 {
capSem = 1
}
sem := make(chan struct{}, capSem)
pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{
"jobs": jobs,
"cap": capSem,
})
for _, q := range expansions {
for _, target := range chatManage.SearchTargets {
wgExp.Add(1)
go func(q string, t *types.SearchTarget) {
defer wgExp.Done()
sem <- struct{}{}
defer func() { <-sem }()
paramsExp := types.SearchParams{
QueryText: q,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: expKwTh,
MatchCount: expTopK,
DisableVectorMatch: true,
DisableKeywordsMatch: false,
}
// Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge {
paramsExp.KnowledgeIDs = t.KnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp)
if err != nil {
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"error": err.Error(),
})
return
}
if len(res) > 0 {
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"query": q,
"hits": len(res),
})
muExp.Lock()
expResults = append(expResults, res...)
muExp.Unlock()
}
}(q, target)
}
}
wgExp.Wait()
if len(expResults) > 0 {
// Scores already normalized in HybridSearch
pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{
"added": len(expResults),
})
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
}
} }
} }
// Add relevant results from chat history
historyResult := p.getSearchResultFromHistory(chatManage)
if historyResult != nil {
pipelineInfo(ctx, "Search", "history_hits", map[string]interface{}{
"session_id": chatManage.SessionID,
"history_hits": len(historyResult),
})
chatManage.SearchResult = append(chatManage.SearchResult, historyResult...)
}
// Remove duplicate results
before := len(chatManage.SearchResult)
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
pipelineInfo(ctx, "Search", "dedup_summary", map[string]interface{}{
"before": before,
"after": len(chatManage.SearchResult),
})
// Log final scores after all processing // Log final scores after all processing
for i, r := range chatManage.SearchResult { for i, r := range chatManage.SearchResult {
pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{ pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{
@@ -253,7 +157,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
} }
// getSearchResultFromHistory retrieves relevant knowledge references from chat history // getSearchResultFromHistory retrieves relevant knowledge references from chat history
func (p *PluginSearch) getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult { func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
if len(chatManage.History) == 0 { if len(chatManage.History) == 0 {
return nil return nil
} }
@@ -275,21 +179,11 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult
contentSig := make(map[string]string) // sig -> first chunk ID contentSig := make(map[string]string) // sig -> first chunk ID
var uniqueResults []*types.SearchResult var uniqueResults []*types.SearchResult
for _, r := range results { for _, r := range results {
keys := []string{r.ID} // Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
if r.ParentChunkID != "" { // as duplicates, because different child chunks of the same parent carry
keys = append(keys, "parent:"+r.ParentChunkID) // different content segments that may all be relevant.
} if seen[r.ID] {
dup := false logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
dupKey := ""
for _, k := range keys {
if seen[k] {
dup = true
dupKey = k
break
}
}
if dup {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to key: %s", r.ID, dupKey)
continue continue
} }
sig := buildContentSignature(r.Content) sig := buildContentSignature(r.Content)
@@ -300,9 +194,7 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult
} }
contentSig[sig] = r.ID contentSig[sig] = r.ID
} }
for _, k := range keys { seen[r.ID] = true
seen[k] = true
}
uniqueResults = append(uniqueResults, r) uniqueResults = append(uniqueResults, r)
} }
return uniqueResults return uniqueResults
@@ -370,10 +262,11 @@ func (p *PluginSearch) searchByTargets(
// Build params for rewrite query // Build params for rewrite query
params := types.SearchParams{ params := types.SearchParams{
QueryText: strings.TrimSpace(chatManage.RewriteQuery), QueryText: strings.TrimSpace(chatManage.RewriteQuery),
VectorThreshold: chatManage.VectorThreshold, VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold, KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK, MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage
} }
// Apply knowledge ID filter if this is a partial KB search // Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge { if t.Type == types.SearchTargetTypeKnowledge {
@@ -546,156 +439,3 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types
}) })
return res return res
} }
// expandQueries generates query variants locally without LLM to improve keyword recall
// Uses simple techniques: word reordering, stopword removal, key phrase extraction
func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string {
query := strings.TrimSpace(chatManage.RewriteQuery)
if query == "" {
return nil
}
expansions := make([]string, 0, 5)
seen := make(map[string]struct{})
seen[strings.ToLower(query)] = struct{}{}
if q := strings.ToLower(chatManage.Query); q != "" {
seen[q] = struct{}{}
}
addIfNew := func(s string) {
s = strings.TrimSpace(s)
if s == "" || len(s) < 3 {
return
}
key := strings.ToLower(s)
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
expansions = append(expansions, s)
}
// 1. Remove common stopwords and create keyword-only variant
keywords := extractKeywords(query)
if len(keywords) >= 2 {
addIfNew(strings.Join(keywords, " "))
}
// 2. Extract quoted phrases or key segments
phrases := extractPhrases(query)
for _, phrase := range phrases {
addIfNew(phrase)
}
// 3. Split by common delimiters and use longest segment
segments := splitByDelimiters(query)
for _, seg := range segments {
if len(seg) > 5 {
addIfNew(seg)
}
}
// 4. Remove question words (什么/如何/怎么/为什么/哪个 etc.)
cleaned := removeQuestionWords(query)
if cleaned != query {
addIfNew(cleaned)
}
// Limit to 5 expansions
if len(expansions) > 5 {
expansions = expansions[:5]
}
pipelineInfo(ctx, "Search", "local_expansion_result", map[string]interface{}{
"variants": len(expansions),
})
return expansions
}
// Common Chinese and English stopwords
var stopwords = map[string]struct{}{
"的": {}, "是": {}, "在": {}, "了": {}, "和": {}, "与": {}, "或": {},
"a": {}, "an": {}, "the": {}, "is": {}, "are": {}, "was": {}, "were": {},
"be": {}, "been": {}, "being": {}, "have": {}, "has": {}, "had": {},
"do": {}, "does": {}, "did": {}, "will": {}, "would": {}, "could": {},
"should": {}, "may": {}, "might": {}, "must": {}, "can": {},
"to": {}, "of": {}, "in": {}, "for": {}, "on": {}, "with": {}, "at": {},
"by": {}, "from": {}, "as": {}, "into": {}, "through": {}, "about": {},
"what": {}, "how": {}, "why": {}, "when": {}, "where": {}, "which": {},
"who": {}, "whom": {}, "whose": {},
}
// Question words in Chinese
var questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`)
func extractKeywords(text string) []string {
words := tokenize(text)
keywords := make([]string, 0, len(words))
for _, w := range words {
lower := strings.ToLower(w)
if _, isStop := stopwords[lower]; !isStop && len(w) > 1 {
keywords = append(keywords, w)
}
}
return keywords
}
func extractPhrases(text string) []string {
// Extract quoted content
var phrases []string
re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)
matches := re.FindAllStringSubmatch(text, -1)
for _, m := range matches {
if len(m) > 1 && len(m[1]) > 2 {
phrases = append(phrases, m[1])
}
}
return phrases
}
func splitByDelimiters(text string) []string {
// Split by common delimiters
re := regexp.MustCompile(`[,,;;、。!?!?\s]+`)
parts := re.Split(text, -1)
var result []string
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
result = append(result, p)
}
}
return result
}
func removeQuestionWords(text string) string {
return strings.TrimSpace(questionWords.ReplaceAllString(text, ""))
}
func tokenize(text string) []string {
var tokens []string
var current strings.Builder
for _, r := range text {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
current.WriteRune(r)
} else if unicode.Is(unicode.Han, r) {
// Flush current token
if current.Len() > 0 {
tokens = append(tokens, current.String())
current.Reset()
}
// Chinese character as single token
tokens = append(tokens, string(r))
} else {
// Delimiter
if current.Len() > 0 {
tokens = append(tokens, current.String())
current.Reset()
}
}
}
if current.Len() > 0 {
tokens = append(tokens, current.String())
}
return tokens
}
@@ -154,7 +154,6 @@ func (p *PluginSearchParallel) OnEvent(ctx context.Context,
// Merge results from both searches (no concurrent access now) // Merge results from both searches (no concurrent access now)
chatManage.SearchResult = append(chunkChatManage.SearchResult, entityChatManage.SearchResult...) chatManage.SearchResult = append(chunkChatManage.SearchResult, entityChatManage.SearchResult...)
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
// Log any errors but don't fail the pipeline if at least one search succeeded // Log any errors but don't fail the pipeline if at least one search succeeded
if chunkSearchErr != nil { if chunkSearchErr != nil {
+172 -42
View File
@@ -1315,6 +1315,34 @@ type ProcessChunksOptions struct {
QuestionCount int QuestionCount int
EnableMultimodel bool EnableMultimodel bool
StoredImages []docparser.StoredImage StoredImages []docparser.StoredImage
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
ParentChunks []types.ParsedParentChunk
}
// buildParentChildConfigs derives parent and child SplitterConfig from ChunkingConfig.
// The base config (already validated with defaults) is used for separators.
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
parentSize := cc.ParentChunkSize
if parentSize <= 0 {
parentSize = 1024
}
childSize := cc.ChildChunkSize
if childSize <= 0 {
childSize = 256
}
parent = chunker.SplitterConfig{
ChunkSize: parentSize,
ChunkOverlap: base.ChunkOverlap, // reuse configured overlap for parents
Separators: base.Separators,
}
child = chunker.SplitterConfig{
ChunkSize: childSize,
ChunkOverlap: childSize / 5, // ~20% overlap for child chunks
Separators: base.Separators,
}
return
} }
// processChunks processes chunks and creates embeddings for knowledge content // processChunks processes chunks and creates embeddings for knowledge content
@@ -1447,8 +1475,44 @@ func (s *knowledgeService) processChunks(ctx context.Context,
} }
} }
// 重新分配容量,考虑图片相关的Chunk // === Parent-Child Chunking: create parent chunks first ===
insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount) hasParentChild := len(options.ParentChunks) > 0
var parentDBChunks []*types.Chunk // indexed by ParsedParentChunk position
if hasParentChild {
parentDBChunks = make([]*types.Chunk, len(options.ParentChunks))
for i, pc := range options.ParentChunks {
parentDBChunks[i] = &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: pc.Content,
ChunkIndex: pc.Seq,
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: pc.Start,
EndAt: pc.End,
ChunkType: types.ChunkTypeParentText,
}
}
// Set prev/next links for parent chunks
for i := range parentDBChunks {
if i > 0 {
parentDBChunks[i-1].NextChunkID = parentDBChunks[i].ID
parentDBChunks[i].PreChunkID = parentDBChunks[i-1].ID
}
}
logger.Infof(ctx, "Created %d parent chunks for parent-child strategy", len(parentDBChunks))
}
// 重新分配容量,考虑图片相关的Chunk + parent chunks
parentCount := len(options.ParentChunks)
insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount+parentCount)
// Add parent chunks first (they go into DB but NOT into the vector index)
if hasParentChild {
insertChunks = append(insertChunks, parentDBChunks...)
}
for idx, chunkData := range chunks { for idx, chunkData := range chunks {
if strings.TrimSpace(chunkData.Content) == "" { if strings.TrimSpace(chunkData.Content) == "" {
@@ -1470,6 +1534,12 @@ func (s *knowledgeService) processChunks(ctx context.Context,
EndAt: int(chunkData.End), EndAt: int(chunkData.End),
ChunkType: types.ChunkTypeText, ChunkType: types.ChunkTypeText,
} }
// Wire up ParentChunkID for child chunks
if hasParentChild && chunkData.ParentIndex >= 0 && chunkData.ParentIndex < len(parentDBChunks) {
textChunk.ParentChunkID = parentDBChunks[chunkData.ParentIndex].ID
}
chunks[idx].ChunkID = textChunk.ID chunks[idx].ChunkID = textChunk.ID
insertChunks = append(insertChunks, textChunk) insertChunks = append(insertChunks, textChunk)
} }
@@ -1479,30 +1549,43 @@ func (s *knowledgeService) processChunks(ctx context.Context,
return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex
}) })
// 仅为文本类型的Chunk设置前后关系 // 仅为文本类型的Chunk设置前后关系(child chunks only, parents already linked above)
textChunks := make([]*types.Chunk, 0, len(chunks)) textChunks := make([]*types.Chunk, 0, len(chunks))
for _, chunk := range insertChunks { for _, chunk := range insertChunks {
if chunk.ChunkType == types.ChunkTypeText { if chunk.ChunkType == types.ChunkTypeText && chunk.ParentChunkID != "" {
// This is a child chunk in parent-child mode
textChunks = append(textChunks, chunk)
} else if chunk.ChunkType == types.ChunkTypeText && !hasParentChild {
// Normal flat chunk (no parent-child mode)
textChunks = append(textChunks, chunk) textChunks = append(textChunks, chunk)
} }
} }
// 设置文本Chunk之间的前后关系 // 设置文本Chunk之间的前后关系 (skip if parent-child, children don't need prev/next links)
for i, chunk := range textChunks { if !hasParentChild {
if i > 0 { for i, chunk := range textChunks {
textChunks[i-1].NextChunkID = chunk.ID if i > 0 {
} textChunks[i-1].NextChunkID = chunk.ID
if i < len(textChunks)-1 { }
textChunks[i+1].PreChunkID = chunk.ID if i < len(textChunks)-1 {
textChunks[i+1].PreChunkID = chunk.ID
}
} }
} }
// Create index information for each chunk (without generated questions for now) // Create index information — only for child/flat chunks, NOT parent chunks.
indexInfoList := make([]*types.IndexInfo, 0, len(insertChunks)) // Parent chunks are stored for context retrieval but do not need vector embeddings.
for _, chunk := range insertChunks { // Prepend the document title to improve semantic alignment between
// Add original chunk content to index // question-style queries and statement-style chunk content.
indexInfoList := make([]*types.IndexInfo, 0, len(textChunks))
titlePrefix := ""
if t := strings.TrimSpace(knowledge.Title); t != "" {
titlePrefix = t + "\n"
}
for _, chunk := range textChunks {
indexContent := titlePrefix + chunk.Content
indexInfoList = append(indexInfoList, &types.IndexInfo{ indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content, Content: indexContent,
SourceID: chunk.ID, SourceID: chunk.ID,
SourceType: types.ChunkSourceType, SourceType: types.ChunkSourceType,
ChunkID: chunk.ID, ChunkID: chunk.ID,
@@ -3031,7 +3114,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
tagIDMapping := map[string]string{} // srcTagID -> dstTagID tagIDMapping := map[string]string{} // srcTagID -> dstTagID
targetChunks := make([]*types.Chunk, 0, 10) targetChunks := make([]*types.Chunk, 0, 10)
chunkType := []types.ChunkType{ chunkType := []types.ChunkType{
types.ChunkTypeText, types.ChunkTypeSummary, types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary,
types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, types.ChunkTypeImageCaption, types.ChunkTypeImageOCR,
} }
for { for {
@@ -6626,24 +6709,47 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
chunkCfg.Separators = []string{"\n\n", "\n", "。"} chunkCfg.Separators = []string{"\n\n", "\n", "。"}
} }
splitChunks := chunker.SplitText(clean, chunkCfg) var parsed []types.ParsedChunk
parsed := make([]types.ParsedChunk, len(splitChunks)) var opts ProcessChunksOptions
for i, c := range splitChunks {
parsed[i] = types.ParsedChunk{ if kb.ChunkingConfig.EnableParentChild {
Content: c.Content, parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
Seq: c.Seq, pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg)
Start: c.Start, parsed = make([]types.ParsedChunk, len(pcResult.Children))
End: c.End, for i, c := range pcResult.Children {
parsed[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
for i, p := range pcResult.Parents {
parentChunks[i] = types.ParsedParentChunk{Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End}
}
opts.ParentChunks = parentChunks
} else {
splitChunks := chunker.SplitText(clean, chunkCfg)
parsed = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
parsed[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
}
} }
} }
if doSync { if doSync {
s.processChunks(ctx, kb, knowledge, parsed) s.processChunks(ctx, kb, knowledge, parsed, opts)
return return
} }
newCtx := logger.CloneContext(ctx) newCtx := logger.CloneContext(ctx)
go s.processChunks(newCtx, kb, knowledge, parsed) go s.processChunks(newCtx, kb, knowledge, parsed, opts)
} }
func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowledge *types.Knowledge) error { func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowledge *types.Knowledge) error {
@@ -7198,25 +7304,49 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
chunkCfg.Separators = []string{"\n\n", "\n", "。"} chunkCfg.Separators = []string{"\n\n", "\n", "。"}
} }
splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg) processOpts := ProcessChunksOptions{
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
}
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
// Step 4: Process chunks (vectorize + index + enqueue async tasks)
s.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration, EnableQuestionGeneration: payload.EnableQuestionGeneration,
QuestionCount: payload.QuestionCount, QuestionCount: payload.QuestionCount,
EnableMultimodel: payload.EnableMultimodel, EnableMultimodel: payload.EnableMultimodel,
StoredImages: storedImages, StoredImages: storedImages,
}) }
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
chunks[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
for i, p := range pcResult.Parents {
parentChunks[i] = types.ParsedParentChunk{Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End}
}
processOpts.ParentChunks = parentChunks
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
len(pcResult.Parents), len(pcResult.Children), knowledge.ID)
} else {
splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg)
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
}
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
}
// Step 4: Process chunks (vectorize + index + enqueue async tasks)
s.processChunks(ctx, kb, knowledge, chunks, processOpts)
return nil return nil
} }
+58 -49
View File
@@ -616,7 +616,9 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
return nil, err return nil, err
} }
matchCount := params.MatchCount * 3 // Use 5x over-retrieval to ensure sufficient candidates for RRF fusion and reranking.
// Minimum 50 to handle large knowledge bases with diverse content.
matchCount := max(params.MatchCount*5, 50)
// Add vector retrieval params if supported // Add vector retrieval params if supported
if retrieveEngine.SupportRetriever(types.VectorRetrieverType) && !params.DisableVectorMatch { if retrieveEngine.SupportRetriever(types.VectorRetrieverType) && !params.DisableVectorMatch {
@@ -790,14 +792,16 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
} }
} }
// Compute RRF scores // Compute weighted RRF scores (vector retrieval weighted higher for semantic relevance)
const vectorWeight = 0.7
const keywordWeight = 0.3
for chunkID := range chunkInfoMap { for chunkID := range chunkInfoMap {
rrfScore := 0.0 rrfScore := 0.0
if rank, ok := vectorRanks[chunkID]; ok { if rank, ok := vectorRanks[chunkID]; ok {
rrfScore += 1.0 / float64(rrfK+rank) rrfScore += vectorWeight / float64(rrfK+rank)
} }
if rank, ok := keywordRanks[chunkID]; ok { if rank, ok := keywordRanks[chunkID]; ok {
rrfScore += 1.0 / float64(rrfK+rank) rrfScore += keywordWeight / float64(rrfK+rank)
} }
rrfScores[chunkID] = rrfScore rrfScores[chunkID] = rrfScore
} }
@@ -858,7 +862,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
deduplicatedChunks = deduplicatedChunks[:params.MatchCount] deduplicatedChunks = deduplicatedChunks[:params.MatchCount]
} }
return s.processSearchResults(ctx, deduplicatedChunks) return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment)
} }
// iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found // iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found
@@ -1100,6 +1104,7 @@ func (s *knowledgeBaseService) matchesNegativeQuestions(queryTextLower string, n
// processSearchResults handles the processing of search results, optimizing database queries // processSearchResults handles the processing of search results, optimizing database queries
func (s *knowledgeBaseService) processSearchResults(ctx context.Context, func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
chunks []*types.IndexWithScore, chunks []*types.IndexWithScore,
skipEnrichment bool,
) ([]*types.SearchResult, error) { ) ([]*types.SearchResult, error) {
if len(chunks) == 0 { if len(chunks) == 0 {
return nil, nil return nil, nil
@@ -1157,34 +1162,36 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
chunkMap[chunk.ID] = chunk chunkMap[chunk.ID] = chunk
processedChunkIDs[chunk.ID] = true processedChunkIDs[chunk.ID] = true
// Collect parent chunks if !skipEnrichment {
if chunk.ParentChunkID != "" && !processedChunkIDs[chunk.ParentChunkID] { // Collect parent chunks
additionalChunkIDs = append(additionalChunkIDs, chunk.ParentChunkID) if chunk.ParentChunkID != "" && !processedChunkIDs[chunk.ParentChunkID] {
processedChunkIDs[chunk.ParentChunkID] = true additionalChunkIDs = append(additionalChunkIDs, chunk.ParentChunkID)
processedChunkIDs[chunk.ParentChunkID] = true
// Pass score to parent // Pass score to parent
chunkScores[chunk.ParentChunkID] = chunkScores[chunk.ID] chunkScores[chunk.ParentChunkID] = chunkScores[chunk.ID]
chunkMatchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk chunkMatchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk
}
// Collect related chunks
relationChunkIDs := s.collectRelatedChunkIDs(chunk, processedChunkIDs)
for _, chunkID := range relationChunkIDs {
additionalChunkIDs = append(additionalChunkIDs, chunkID)
chunkMatchTypes[chunkID] = types.MatchTypeRelationChunk
}
// Add nearby chunks (prev and next)
if slices.Contains([]string{types.ChunkTypeText}, chunk.ChunkType) {
if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] {
additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID)
processedChunkIDs[chunk.NextChunkID] = true
chunkMatchTypes[chunk.NextChunkID] = types.MatchTypeNearByChunk
} }
if chunk.PreChunkID != "" && !processedChunkIDs[chunk.PreChunkID] {
additionalChunkIDs = append(additionalChunkIDs, chunk.PreChunkID) // Collect related chunks
processedChunkIDs[chunk.PreChunkID] = true relationChunkIDs := s.collectRelatedChunkIDs(chunk, processedChunkIDs)
chunkMatchTypes[chunk.PreChunkID] = types.MatchTypeNearByChunk for _, chunkID := range relationChunkIDs {
additionalChunkIDs = append(additionalChunkIDs, chunkID)
chunkMatchTypes[chunkID] = types.MatchTypeRelationChunk
}
// Add nearby chunks (prev and next)
if slices.Contains([]string{types.ChunkTypeText}, chunk.ChunkType) {
if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] {
additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID)
processedChunkIDs[chunk.NextChunkID] = true
chunkMatchTypes[chunk.NextChunkID] = types.MatchTypeNearByChunk
}
if chunk.PreChunkID != "" && !processedChunkIDs[chunk.PreChunkID] {
additionalChunkIDs = append(additionalChunkIDs, chunk.PreChunkID)
processedChunkIDs[chunk.PreChunkID] = true
chunkMatchTypes[chunk.PreChunkID] = types.MatchTypeNearByChunk
}
} }
} }
} }
@@ -1235,26 +1242,28 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
} }
// Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input // Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input
for chunkID, chunk := range chunkMap { if !skipEnrichment {
if addedChunkIDs[chunkID] || !s.isValidTextChunk(chunk) { for chunkID, chunk := range chunkMap {
continue if addedChunkIDs[chunkID] || !s.isValidTextChunk(chunk) {
}
score, hasScore := chunkScores[chunkID]
if !hasScore || score <= 0 {
score = 0.0
}
if knowledge, ok := knowledgeMap[chunk.KnowledgeID]; ok {
matchType := types.MatchTypeParentChunk
if specificType, exists := chunkMatchTypes[chunkID]; exists {
matchType = specificType
} else {
logger.Warnf(ctx, "Unkonwn match type for chunk: %s", chunkID)
continue continue
} }
matchedContent := chunkMatchedContents[chunkID]
searchResults = append(searchResults, s.buildSearchResult(chunk, knowledge, score, matchType, matchedContent)) score, hasScore := chunkScores[chunkID]
if !hasScore || score <= 0 {
score = 0.0
}
if knowledge, ok := knowledgeMap[chunk.KnowledgeID]; ok {
matchType := types.MatchTypeParentChunk
if specificType, exists := chunkMatchTypes[chunkID]; exists {
matchType = specificType
} else {
logger.Warnf(ctx, "Unkonwn match type for chunk: %s", chunkID)
continue
}
matchedContent := chunkMatchedContents[chunkID]
searchResults = append(searchResults, s.buildSearchResult(chunk, knowledge, score, matchType, matchedContent))
}
} }
} }
logger.Infof(ctx, "Search results processed, total: %d", len(searchResults)) logger.Infof(ctx, "Search results processed, total: %d", len(searchResults))
+10
View File
@@ -96,6 +96,9 @@ type KBModelConfigRequest struct {
ChunkOverlap int `json:"chunkOverlap"` ChunkOverlap int `json:"chunkOverlap"`
Separators []string `json:"separators"` Separators []string `json:"separators"`
ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"` ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"`
EnableParentChild bool `json:"enableParentChild"`
ParentChunkSize int `json:"parentChunkSize,omitempty"`
ChildChunkSize int `json:"childChunkSize,omitempty"`
} `json:"documentSplitting"` } `json:"documentSplitting"`
// 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置) // 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置)
@@ -289,6 +292,13 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators
} }
kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules
kb.ChunkingConfig.EnableParentChild = req.DocumentSplitting.EnableParentChild
if req.DocumentSplitting.ParentChunkSize > 0 {
kb.ChunkingConfig.ParentChunkSize = req.DocumentSplitting.ParentChunkSize
}
if req.DocumentSplitting.ChildChunkSize > 0 {
kb.ChunkingConfig.ChildChunkSize = req.DocumentSplitting.ChildChunkSize
}
// 更新多模态配置 // 更新多模态配置
if req.Multimodal.Enabled { if req.Multimodal.Enabled {
+47 -1
View File
@@ -36,7 +36,7 @@ type SplitterConfig struct {
func DefaultConfig() SplitterConfig { func DefaultConfig() SplitterConfig {
return SplitterConfig{ return SplitterConfig{
ChunkSize: 512, ChunkSize: 512,
ChunkOverlap: 50, ChunkOverlap: 128,
Separators: []string{"\n\n", "\n", "。"}, Separators: []string{"\n\n", "\n", "。"},
} }
} }
@@ -316,6 +316,52 @@ func isSeparatorOnly(s string) bool {
return true return true
} }
// ParentChildResult holds the two-level chunking output.
// Parent chunks provide context (large window), child chunks are used for
// embedding/retrieval (small window). Each child carries its ParentIndex so
// the caller can wire up ParentChunkID after DB insertion.
type ParentChildResult struct {
Parents []Chunk
Children []ChildChunk
}
// ChildChunk extends Chunk with a reference to its parent.
type ChildChunk struct {
Chunk
ParentIndex int // index into ParentChildResult.Parents
}
// SplitTextParentChild performs two-level chunking:
// 1. Split text into large parent chunks (parentCfg).
// 2. Split each parent into smaller child chunks (childCfg) for embedding.
//
// The child Seq is globally unique across the entire document.
func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
parents := SplitText(text, parentCfg)
if len(parents) == 0 {
return ParentChildResult{}
}
var children []ChildChunk
childSeq := 0
for pi, parent := range parents {
subs := SplitText(parent.Content, childCfg)
for _, sub := range subs {
// Adjust offsets: sub positions are relative to parent content,
// shift to document-level offsets.
sub.Seq = childSeq
sub.Start += parent.Start
sub.End = sub.Start + runeLen(sub.Content)
children = append(children, ChildChunk{
Chunk: sub,
ParentIndex: pi,
})
childSeq++
}
}
return ParentChildResult{Parents: parents, Children: children}
}
// ExtractImageRefs extracts markdown image references from text. // ExtractImageRefs extracts markdown image references from text.
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
+46 -8
View File
@@ -4,6 +4,9 @@ import (
"crypto/md5" "crypto/md5"
"encoding/hex" "encoding/hex"
"strings" "strings"
"unicode"
"github.com/Tencent/WeKnora/internal/types"
) )
// BuildContentSignature creates a normalized MD5 signature for content to detect duplicates. // BuildContentSignature creates a normalized MD5 signature for content to detect duplicates.
@@ -20,20 +23,55 @@ func BuildContentSignature(content string) string {
return hex.EncodeToString(hash[:]) return hex.EncodeToString(hash[:])
} }
// TokenizeSimple tokenizes text into a set of words (simple whitespace-based). // containsChinese checks whether text contains any CJK unified ideographs.
// Returns a map where keys are lowercase tokens with length > 1. func containsChinese(text string) bool {
for _, r := range text {
if unicode.Is(unicode.Han, r) {
return true
}
}
return false
}
// TokenizeSimple tokenizes text into a set of unique tokens.
// For text containing Chinese characters, it uses jieba segmentation for accurate word boundaries.
// For pure non-Chinese text, it falls back to whitespace-based splitting.
// Returns a map where keys are lowercase tokens with rune length > 1.
func TokenizeSimple(text string) map[string]struct{} { func TokenizeSimple(text string) map[string]struct{} {
text = strings.ToLower(text) text = strings.ToLower(strings.TrimSpace(text))
fields := strings.Fields(text) if text == "" {
set := make(map[string]struct{}, len(fields)) return nil
for _, f := range fields { }
if len(f) > 1 {
set[f] = struct{}{} var words []string
if containsChinese(text) {
// Use jieba for Chinese text segmentation (search mode for finer granularity)
words = types.Jieba.CutForSearch(text, true)
} else {
words = strings.Fields(text)
}
set := make(map[string]struct{}, len(words))
for _, w := range words {
w = strings.TrimSpace(w)
// Filter out single-rune tokens and pure punctuation/whitespace
if len([]rune(w)) > 1 && !isAllPunct(w) {
set[w] = struct{}{}
} }
} }
return set return set
} }
// isAllPunct checks if a string consists entirely of punctuation or whitespace.
func isAllPunct(s string) bool {
for _, r := range s {
if !unicode.IsPunct(r) && !unicode.IsSpace(r) && !unicode.IsSymbol(r) {
return false
}
}
return true
}
// Jaccard calculates Jaccard similarity between two token sets. // Jaccard calculates Jaccard similarity between two token sets.
// Returns a value between 0 and 1, where 1 means identical sets. // Returns a value between 0 and 1, where 1 means identical sets.
func Jaccard(a, b map[string]struct{}) float64 { func Jaccard(a, b map[string]struct{}) float64 {
+2
View File
@@ -14,6 +14,8 @@ type ChunkType = string
const ( const (
// ChunkTypeText 表示普通的文本 Chunk // ChunkTypeText 表示普通的文本 Chunk
ChunkTypeText ChunkType = "text" ChunkTypeText ChunkType = "text"
// ChunkTypeParentText 表示父子分块策略中的父文本 Chunk(仅用于上下文,不参与向量索引)
ChunkTypeParentText ChunkType = "parent_text"
// ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk // ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk
ChunkTypeImageOCR ChunkType = "image_ocr" ChunkTypeImageOCR ChunkType = "image_ocr"
// ChunkTypeImageCaption 表示图片描述的 Chunk // ChunkTypeImageCaption 表示图片描述的 Chunk
+15
View File
@@ -67,6 +67,21 @@ type ParsedChunk struct {
End int End int
Images []ParsedImage Images []ParsedImage
ChunkID string // populated by processChunks with the actual DB UUID ChunkID string // populated by processChunks with the actual DB UUID
// ParentIndex is set when using parent-child chunking strategy.
// -1 (or unset/0 for flat chunks) means this is a top-level chunk.
// >= 0 means this is a child chunk referencing the parent at this index
// in the ParentChunks slice of ProcessChunksOptions.
ParentIndex int
}
// ParsedParentChunk represents a parent chunk in the parent-child strategy.
// Parent chunks are stored in DB for context retrieval but NOT vector-indexed.
type ParsedParentChunk struct {
Content string
Seq int
Start int
End int
} }
type ParsedImage struct { type ParsedImage struct {
+10
View File
@@ -117,6 +117,16 @@ type ChunkingConfig struct {
// ParserEngineRules configures which parser engine to use for each file type. // ParserEngineRules configures which parser engine to use for each file type.
// When empty, the builtin engine is used for all types. // When empty, the builtin engine is used for all types.
ParserEngineRules []ParserEngineRule `yaml:"parser_engine_rules,omitempty" json:"parser_engine_rules,omitempty"` ParserEngineRules []ParserEngineRule `yaml:"parser_engine_rules,omitempty" json:"parser_engine_rules,omitempty"`
// EnableParentChild enables two-level parent-child chunking strategy.
// When enabled, large parent chunks provide context while small child chunks
// are used for vector matching. Retrieval matches on child but returns parent content.
EnableParentChild bool `yaml:"enable_parent_child,omitempty" json:"enable_parent_child,omitempty"`
// ParentChunkSize is the size of parent chunks (default: 1024).
// Only used when EnableParentChild is true.
ParentChunkSize int `yaml:"parent_chunk_size,omitempty" json:"parent_chunk_size,omitempty"`
// ChildChunkSize is the size of child chunks used for embedding (default: 256).
// Only used when EnableParentChild is true.
ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"`
} }
// ResolveParserEngine returns the engine name for the given file type // ResolveParserEngine returns the engine name for the given file type
+4
View File
@@ -142,6 +142,10 @@ type SearchParams struct {
KnowledgeIDs []string `json:"knowledge_ids"` KnowledgeIDs []string `json:"knowledge_ids"`
TagIDs []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering) TagIDs []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering)
OnlyRecommended bool `json:"only_recommended"` OnlyRecommended bool `json:"only_recommended"`
// SkipContextEnrichment skips fetching parent, nearby, and relation chunks
// in processSearchResults. Used by the chat pipeline where context assembly
// is handled separately in the merge stage.
SkipContextEnrichment bool `json:"skip_context_enrichment,omitempty"`
} }
// Value implements the driver.Valuer interface, used to convert SearchResult to database value // Value implements the driver.Valuer interface, used to convert SearchResult to database value