diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index 070bbe32..87f30e2d 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -83,6 +83,9 @@ export interface KBModelConfigRequest { chunkOverlap: number separators: string[] parserEngineRules?: { file_types: string[]; engine: string }[] + enableParentChild?: boolean + parentChunkSize?: number + childChunkSize?: number } multimodal: { enabled: boolean diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index b8e040a4..333653c8 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -4,8 +4,8 @@ import { marked } from "marked"; import hljs from "highlight.js"; import "highlight.js/styles/github.css"; import mermaid from "mermaid"; -import { onMounted, ref, nextTick, onUnmounted, onUpdated, watch } from "vue"; -import { downKnowledgeDetails, deleteGeneratedQuestion } from "@/api/knowledge-base/index"; +import { onMounted, ref, nextTick, onUnmounted, watch } from "vue"; +import { downKnowledgeDetails, deleteGeneratedQuestion, getChunkByIdOnly } from "@/api/knowledge-base/index"; import { MessagePlugin, DialogPlugin } from "tdesign-vue-next"; import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL, hydrateProtectedFileImages } from '@/utils/security'; import { openMermaidFullscreen } from '@/utils/mermaidViewer'; @@ -267,24 +267,39 @@ const isMarkdownFile = (fileType?: string): boolean => { const markdownTypes = ['md', 'markdown']; return markdownTypes.includes(fileType.toLowerCase()); }; +const runMarkdownPostRenderPipeline = async () => { + await nextTick(); + const renderRoot = mdContentWrap.value as ParentNode; + await hydrateProtectedFileImages(renderRoot); + const images = renderRoot?.querySelectorAll?.('img.markdown-image') as NodeListOf | undefined; + if (images) { + images.forEach(async item => { + const isValid = await checkImage(item.src); + if (!isValid) { + item.remove(); + } + }) + } + // 渲染 Mermaid 图表 + await renderMermaidDiagrams(); +}; + watch(() => props.details.md, (newVal) => { - nextTick(async () => { - const renderRoot = (doc as ParentNode) || mdContentWrap.value; - await hydrateProtectedFileImages(renderRoot); - const images = renderRoot?.querySelectorAll?.('img.markdown-image') as NodeListOf | undefined; - if (images) { - images.forEach(async item => { - const isValid = await checkImage(item.src); - if (!isValid) { - item.remove(); - } - }) - } - // 渲染 Mermaid 图表 - await renderMermaidDiagrams(); - }) + runMarkdownPostRenderPipeline(); }, { immediate: true, deep: true }) +watch(() => viewMode.value, (mode) => { + if ((mode === 'chunks' || mode === 'merged') && props.visible) { + runMarkdownPostRenderPipeline(); + } +}); + +watch(() => props.visible, (visible) => { + if (visible && (viewMode.value === 'chunks' || viewMode.value === 'merged')) { + runMarkdownPostRenderPipeline(); + } +}); + // 渲染 Mermaid 图表的函数 const renderMermaidDiagrams = async () => { try { @@ -557,6 +572,50 @@ const isDeleting = (chunkIndex: number, questionId: string) => { return deletingQuestion.value?.chunkIndex === chunkIndex && deletingQuestion.value?.questionId === questionId; }; +// 父 Chunk 上下文展开状态 +const parentContextExpanded = ref>(new Set()); +const parentContextCache = ref>(new Map()); +const parentContextLoading = ref>(new Set()); + +const hasParentChunk = (item: any) => !!item?.parent_chunk_id; + +const isParentExpanded = (index: number) => parentContextExpanded.value.has(index); + +const toggleParentContext = async (item: any, index: number) => { + if (parentContextExpanded.value.has(index)) { + parentContextExpanded.value.delete(index); + parentContextExpanded.value = new Set(parentContextExpanded.value); + return; + } + + const parentId = item.parent_chunk_id; + if (!parentContextCache.value.has(parentId)) { + parentContextLoading.value.add(index); + parentContextLoading.value = new Set(parentContextLoading.value); + try { + const result: any = await getChunkByIdOnly(parentId); + if (result.success && result.data) { + parentContextCache.value.set(parentId, result.data.content || ''); + parentContextCache.value = new Map(parentContextCache.value); + } + } catch (err) { + MessagePlugin.error(t('knowledgeBase.parentContextLoadFailed') || '加载父上下文失败'); + return; + } finally { + parentContextLoading.value.delete(index); + parentContextLoading.value = new Set(parentContextLoading.value); + } + } + + parentContextExpanded.value.add(index); + parentContextExpanded.value = new Set(parentContextExpanded.value); + await runMarkdownPostRenderPipeline(); +}; + +const getParentContent = (item: any) => { + return parentContextCache.value.get(item.parent_chunk_id) || ''; +}; + const downloadFile = () => { downKnowledgeDetails(props.details.id) .then((result) => { @@ -703,6 +762,14 @@ const handleDetailsScroll = () => {
{{ $t('knowledgeBase.segment') || '片段' }} {{ index + 1 }}
+ + {{ $t('knowledgeBase.childChunk') || '子块' }} + {
+ +
+
+ + + {{ $t('knowledgeBase.viewParentContext') || '查看父块上下文' }} +
+
+
+
+
+
@@ -1058,6 +1137,42 @@ const handleDetailsScroll = () => { } } +// 父 Chunk 上下文样式 +.parent-context-section { + margin-top: 10px; + padding-top: 8px; + border-top: 1px dashed #d0d5dd; +} + +.parent-context-toggle { + display: flex; + align-items: center; + gap: 6px; + cursor: pointer; + color: #07c05f; + font-size: 12px; + font-weight: 500; + padding: 4px 0; + transition: color 0.2s ease; + + &:hover { + color: #07c05f; + } +} + +.parent-context-content { + margin-top: 8px; + padding: 10px 12px; + background: #f0f5ff; + border-radius: 4px; + border-left: 3px solid #07c05f; + + .md-content { + color: #4e5969; + font-size: 13px; + } +} + // 生成的问题样式 .questions-section { margin-top: 12px; diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 296c1f0a..6befae00 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -142,6 +142,9 @@ export default { loadOriginalFailed: 'Failed to load original file content', questions: 'Questions', generatedQuestions: 'Generated Questions', + childChunk: 'Child Chunk', + viewParentContext: 'View Parent Context', + parentContextLoadFailed: 'Failed to load parent context', confirmDeleteQuestion: 'Are you sure you want to delete this question? The corresponding vector index will also be removed.', legacyQuestionCannotDelete: 'Legacy format questions cannot be deleted. Please regenerate questions.', notInitialized: 'Knowledge base is not initialized. Please configure models in settings before uploading files', @@ -1415,7 +1418,13 @@ export default { semicolonCn: 'Chinese semicolon (；)', semicolonEn: 'Semicolon (;)', space: 'Space ( )' - } + }, + parentChildLabel: 'Parent-Child Chunking', + parentChildDescription: 'Enable two-level parent-child chunking strategy. Large parent chunks provide context while small child chunks are used for vector matching.', + parentChunkSizeLabel: 'Parent Chunk Size', + parentChunkSizeDescription: 'Size of parent chunks that provide context (256-4096)', + childChunkSizeLabel: 'Child Chunk Size', + childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)' }, advanced: { title: 'Advanced Settings', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 9825e8c5..bdcc4bdd 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -143,6 +143,9 @@ export default { loadOriginalFailed: "원본 파일 내용 로드 실패", questions: "질문", generatedQuestions: "생성된 질문", + childChunk: "자식 청크", + viewParentContext: "부모 컨텍스트 보기", + parentContextLoadFailed: "부모 컨텍스트 로드 실패", confirmDeleteQuestion: "이 질문을 삭제하시겠습니까? 삭제 시 해당 벡터 인덱스도 함께 제거됩니다.", legacyQuestionCannotDelete: "이전 형식의 질문은 삭제할 수 없습니다. 질문을 다시 생성하세요", @@ -1860,6 +1863,12 @@ export default { semicolonEn: "영어 세미콜론 (;)", space: "공백 ( )", }, + parentChildLabel: "부모-자식 청킹", + parentChildDescription: "2단계 부모-자식 청킹 전략을 활성화합니다. 큰 부모 청크는 컨텍스트를 제공하고, 작은 자식 청크는 벡터 매칭에 사용됩니다.", + parentChunkSizeLabel: "부모 청크 크기", + parentChunkSizeDescription: "컨텍스트를 제공하는 부모 청크의 문자 수 (256-4096)", + childChunkSizeLabel: "자식 청크 크기", + childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)", }, advanced: { title: "고급 설정", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index bb10d663..fa630fc6 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -118,6 +118,9 @@ export default { loadOriginalFailed: 'Не удалось загрузить содержимое исходного файла', questions: 'Вопросы', generatedQuestions: 'Сгенерированные вопросы', + childChunk: 'Дочерний блок', + viewParentContext: 'Просмотр родительского контекста', + parentContextLoadFailed: 'Не удалось загрузить родительский контекст', confirmDeleteQuestion: 'Вы уверены, что хотите удалить этот вопрос? Соответствующий векторный индекс также будет удален.', legacyQuestionCannotDelete: 'Вопросы в устаревшем формате нельзя удалить. Пожалуйста, сгенерируйте вопросы заново.', notInitialized: 'База знаний не инициализирована. Пожалуйста, настройте модели в разделе настроек перед загрузкой файлов', @@ -1253,7 +1256,13 @@ export default { semicolonCn: 'Китайская точка с запятой (；)', semicolonEn: 'Точка с запятой (;)', space: 'Пробел ( )' - } + }, + parentChildLabel: 'Родительско-дочернее разбиение', + parentChildDescription: 'Включить двухуровневую стратегию разбиения. Большие родительские блоки обеспечивают контекст, а маленькие дочерние блоки используются для векторного поиска.', + parentChunkSizeLabel: 'Размер родительского блока', + parentChunkSizeDescription: 'Размер родительских блоков для контекста (256-4096)', + childChunkSizeLabel: 'Размер дочернего блока', + childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)' }, advanced: { title: 'Расширенные настройки', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 806f9adf..ede48fc6 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -141,6 +141,9 @@ export default { loadOriginalFailed: "加载原文件内容失败", questions: "问题", generatedQuestions: "生成的问题", + childChunk: "子块", + viewParentContext: "查看父块上下文", + parentContextLoadFailed: "加载父上下文失败", confirmDeleteQuestion: "确定要删除这个问题吗？删除后将同时移除对应的向量索引。", legacyQuestionCannotDelete: "旧格式问题无法删除，请重新生成问题", docActionUnsupported: "当前知识库类型不支持该操作", @@ -1820,6 +1823,12 @@ export default { semicolonEn: "英文分号 (;)", space: "空格 ( )", }, + parentChildLabel: "父子分块", + parentChildDescription: "启用两级父子分块策略。大的父块提供上下文，小的子块用于向量匹配检索。", + parentChunkSizeLabel: "父块大小", + parentChunkSizeDescription: "提供上下文的父块字符数（256-4096）", + childChunkSizeLabel: "子块大小", + childChunkSizeDescription: "用于向量匹配的子块字符数（64-1024）", }, advanced: { title: "高级设置", diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index f46ebff0..b172d9e7 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -296,7 +296,10 @@ const initFormData = (type: 'document' | 'faq' = 'document') => { chunkSize: 512, chunkOverlap: 100, separators: ['\n\n', '\n', '。', '！', '？', ';', '；'], - parserEngineRules: undefined as any + parserEngineRules: undefined as any, + enableParentChild: false, + parentChunkSize: 1024, + childChunkSize: 256 }, storageProvider: 'local' as string, multimodalConfig: { @@ -373,7 +376,10 @@ const loadKBData = async () => { chunkSize: kb.chunking_config?.chunk_size || 512, chunkOverlap: kb.chunking_config?.chunk_overlap || 100, separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '！', '？', ';', '；'], - parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined + parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined, + enableParentChild: kb.chunking_config?.enable_parent_child || false, + parentChunkSize: kb.chunking_config?.parent_chunk_size || 1024, + childChunkSize: kb.chunking_config?.child_chunk_size || 256 }, storageProvider: (kb.storage_config?.provider || 'local') as string, multimodalConfig: { @@ -503,6 +509,9 @@ const buildSubmitData = () => { chunk_overlap: formData.value.chunkingConfig.chunkOverlap, separators: formData.value.chunkingConfig.separators, enable_multimodal: formData.value.multimodalConfig.enabled, + enable_parent_child: formData.value.chunkingConfig.enableParentChild, + parent_chunk_size: formData.value.chunkingConfig.parentChunkSize, + child_chunk_size: formData.value.chunkingConfig.childChunkSize, ...(formData.value.chunkingConfig.parserEngineRules?.length ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } : {}) @@ -603,7 +612,10 @@ const handleSubmit = async () => { chunkSize: data.chunking_config.chunk_size, chunkOverlap: data.chunking_config.chunk_overlap, separators: data.chunking_config.separators, - parserEngineRules: data.chunking_config.parser_engine_rules || undefined + parserEngineRules: data.chunking_config.parser_engine_rules || undefined, + enableParentChild: data.chunking_config.enable_parent_child || false, + parentChunkSize: data.chunking_config.parent_chunk_size || 1024, + childChunkSize: data.chunking_config.child_chunk_size || 256 }, multimodal: { enabled: !!data.vlm_config?.enabled diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index d7a90572..ff937d78 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -69,6 +69,64 @@ />
+ + +
+
+ +

{{ $t('knowledgeEditor.chunking.parentChildDescription') }}

+
+
+ +
+
+ + +
+
+ +

{{ $t('knowledgeEditor.chunking.parentChunkSizeDescription') }}

+
+
+
+ + {{ localParentChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }} +
+
+
+ + +
+
+ +

{{ $t('knowledgeEditor.chunking.childChunkSizeDescription') }}

+
+
+
+ + {{ localChildChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }} +
+
+
@@ -87,6 +145,9 @@ interface ChunkingConfig { chunkOverlap: number separators: string[] parserEngineRules?: ParserEngineRule[] + enableParentChild: boolean + parentChunkSize: number + childChunkSize: number } interface Props { @@ -102,6 +163,9 @@ const emit = defineEmits<{ const localChunkSize = ref(props.config.chunkSize) const localChunkOverlap = ref(props.config.chunkOverlap) const localSeparators = ref([...props.config.separators]) +const localEnableParentChild = ref(props.config.enableParentChild ?? false) +const localParentChunkSize = ref(props.config.parentChunkSize || 1024) +const localChildChunkSize = ref(props.config.childChunkSize || 256) const { t } = useI18n() const separatorOptions = computed(() => [ @@ -119,18 +183,27 @@ watch(() => props.config, (newConfig) => { localChunkSize.value = newConfig.chunkSize localChunkOverlap.value = newConfig.chunkOverlap localSeparators.value = [...newConfig.separators] + localEnableParentChild.value = newConfig.enableParentChild ?? false + localParentChunkSize.value = newConfig.parentChunkSize || 1024 + localChildChunkSize.value = newConfig.childChunkSize || 256 }, { deep: true }) const handleChunkSizeChange = () => { emitUpdate() } const handleChunkOverlapChange = () => { emitUpdate() } const handleSeparatorsChange = () => { emitUpdate() } +const handleParentChildChange = () => { emitUpdate() } +const handleParentChunkSizeChange = () => { emitUpdate() } +const handleChildChunkSizeChange = () => { emitUpdate() } const emitUpdate = () => { emit('update:config', { chunkSize: localChunkSize.value, chunkOverlap: localChunkOverlap.value, separators: localSeparators.value, - parserEngineRules: props.config.parserEngineRules + parserEngineRules: props.config.parserEngineRules, + enableParentChild: localEnableParentChild.value, + parentChunkSize: localParentChunkSize.value, + childChunkSize: localChildChunkSize.value }) } diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index 0ff3daca..cdefad73 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -3,22 +3,26 @@ package chatpipline import ( "context" "encoding/json" + "fmt" "sort" "strings" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) // PluginMerge handles merging of search result chunks type PluginMerge struct { - chunkRepo interfaces.ChunkRepository + chunkRepo interfaces.ChunkRepository + chunkService interfaces.ChunkService // for parent chunk resolution } // NewPluginMerge creates and registers a new PluginMerge instance -func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository) *PluginMerge { +func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository, chunkService interfaces.ChunkService) *PluginMerge { res := &PluginMerge{ - chunkRepo: chunkRepo, + chunkRepo: chunkRepo, + chunkService: chunkService, } eventManager.Register(res) return res @@ -45,6 +49,32 @@ func (p *PluginMerge) OnEvent(ctx context.Context, "reason": "empty_rerank_result", }) searchResult = chatManage.SearchResult + // Sort by score descending so dedup keeps highest-scored entries + sort.Slice(searchResult, func(i, j int) bool { + return searchResult[i].Score > searchResult[j].Score + }) + } + + // Deduplicate after rerank so higher-scored duplicates are preferred + beforeDedup := len(searchResult) + searchResult = removeDuplicateResults(searchResult) + pipelineInfo(ctx, "Merge", "dedup_summary", map[string]interface{}{ + "before": beforeDedup, + "after": len(searchResult), + }) + + // Inject relevant results from chat history with similarity filtering. + // History references were produced for a previous query, so we only keep + // those that are textually relevant to the current query to avoid injecting + // stale or off-topic context. + historyResults := filterHistoryResults(ctx, chatManage, searchResult) + if len(historyResults) > 0 { + pipelineInfo(ctx, "Merge", "history_inject", map[string]interface{}{ + "session_id": chatManage.SessionID, + "history_hits": len(historyResults), + }) + searchResult = append(searchResult, historyResults...) + searchResult = removeDuplicateResults(searchResult) } pipelineInfo(ctx, "Merge", "candidate_ready", map[string]interface{}{ @@ -60,6 +90,9 @@ func (p *PluginMerge) OnEvent(ctx context.Context, return next() } + // Resolve parent chunks: replace child content with fuller parent content + searchResult = p.resolveParentChunks(ctx, chatManage, searchResult) + // Group chunks by their knowledge source ID knowledgeGroup := make(map[string]map[string][]*types.SearchResult) for _, chunk := range searchResult { @@ -145,6 +178,161 @@ func (p *PluginMerge) OnEvent(ctx context.Context, return next() } +// resolveParentChunks replaces child chunk content with parent chunk content +// for results that have ParentChunkID set. This provides fuller context +// for small child chunks used in parent-child chunking strategy. +func (p *PluginMerge) resolveParentChunks( + ctx context.Context, + chatManage *types.ChatManage, + results []*types.SearchResult, +) []*types.SearchResult { + if len(results) == 0 || p.chunkRepo == nil { + return results + } + + tenantID, _ := types.TenantIDFromContext(ctx) + if tenantID == 0 && chatManage != nil { + tenantID = chatManage.TenantID + } + if tenantID == 0 { + pipelineWarn(ctx, "Merge", "parent_resolve_skip", map[string]interface{}{ + "reason": "missing_tenant", + }) + return results + } + + // Collect unique parent chunk IDs + parentIDs := make(map[string]struct{}) + for _, r := range results { + if r.ParentChunkID != "" { + parentIDs[r.ParentChunkID] = struct{}{} + } + } + + if len(parentIDs) == 0 { + return results + } + + // Batch fetch parent chunks + ids := make([]string, 0, len(parentIDs)) + for id := range parentIDs { + ids = append(ids, id) + } + parentChunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, ids) + if err != nil { + pipelineWarn(ctx, "Merge", "parent_resolve_failed", map[string]interface{}{ + "error": err.Error(), + }) + return results + } + + parentMap := make(map[string]*types.Chunk, len(parentChunks)) + for _, c := range parentChunks { + parentMap[c.ID] = c + } + + // Replace child content with parent content + for _, r := range results { + if r.ParentChunkID == "" { + continue + } + parent, ok := parentMap[r.ParentChunkID] + if !ok || parent.Content == "" { + continue + } + pipelineInfo(ctx, "Merge", "parent_resolve", map[string]interface{}{ + "child_id": r.ID, + "parent_id": r.ParentChunkID, + "child_len": runeLen(r.Content), + "parent_len": runeLen(parent.Content), + }) + r.Content = parent.Content + r.StartAt = parent.StartAt + r.EndAt = parent.EndAt + // Track the original child as a sub-chunk + if !containsID(r.SubChunkID, r.ID) { + r.SubChunkID = append(r.SubChunkID, r.ID) + } + } + + return results +} + +// filterHistoryResults retrieves history references and filters them by +// textual similarity to the current query. Only references that are above +// a Jaccard similarity threshold are kept, and their scores are discounted +// to reflect that they were not directly retrieved for the current query. +// Results already present in currentResults (by chunk ID) are excluded. +func filterHistoryResults( + ctx context.Context, + chatManage *types.ChatManage, + currentResults []*types.SearchResult, +) []*types.SearchResult { + const ( + // minSimilarity is the minimum Jaccard similarity between the current + // query and a history chunk's content for it to be injected. + minSimilarity = 0.15 + // historyScoreDiscount reduces the original score of history results + // to rank them below freshly-retrieved results of similar relevance. + historyScoreDiscount = 0.6 + // maxHistoryResults caps the number of history results injected to + // avoid overwhelming the context with stale references. + maxHistoryResults = 3 + ) + + raw := getSearchResultFromHistory(chatManage) + if len(raw) == 0 { + return nil + } + + // Build a set of chunk IDs already in current results for fast dedup + existingIDs := make(map[string]struct{}, len(currentResults)) + for _, r := range currentResults { + existingIDs[r.ID] = struct{}{} + } + + // Use RewriteQuery if available (it's the cleaned-up retrieval query), + // otherwise fall back to the original query. + query := chatManage.RewriteQuery + if query == "" { + query = chatManage.Query + } + queryTokens := searchutil.TokenizeSimple(query) + + var filtered []*types.SearchResult + for _, r := range raw { + if _, exists := existingIDs[r.ID]; exists { + continue + } + contentTokens := searchutil.TokenizeSimple(r.Content) + sim := searchutil.Jaccard(queryTokens, contentTokens) + if sim < minSimilarity { + pipelineInfo(ctx, "Merge", "history_filter_drop", map[string]interface{}{ + "chunk_id": r.ID, + "similarity": sim, + }) + continue + } + r.MatchType = types.MatchTypeHistory + r.Score = r.Score * historyScoreDiscount + r.Metadata = ensureMetadata(r.Metadata) + r.Metadata["history_similarity"] = strings.TrimRight(strings.TrimRight( + fmt.Sprintf("%.4f", sim), "0"), ".") + filtered = append(filtered, r) + + pipelineInfo(ctx, "Merge", "history_filter_keep", map[string]interface{}{ + "chunk_id": r.ID, + "similarity": sim, + "new_score": r.Score, + }) + + if len(filtered) >= maxHistoryResults { + break + } + } + return filtered +} + // mergeImageInfo 合并两个chunk的ImageInfo func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error { // 如果source没有ImageInfo，不需要合并 diff --git a/internal/application/service/chat_pipline/query_expansion.go b/internal/application/service/chat_pipline/query_expansion.go new file mode 100644 index 00000000..f9bccba2 --- /dev/null +++ b/internal/application/service/chat_pipline/query_expansion.go @@ -0,0 +1,253 @@ +package chatpipline + +import ( + "context" + "regexp" + "strings" + "sync" + "unicode" + + "github.com/Tencent/WeKnora/internal/types" +) + +// runQueryExpansion performs query expansion when initial recall is low. +// It generates query variants and runs concurrent retrieval across search targets. +func (p *PluginSearch) runQueryExpansion(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult { + pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{ + "current": len(chatManage.SearchResult), + "threshold": chatManage.EmbeddingTopK, + }) + expansions := p.expandQueries(ctx, chatManage) + if len(expansions) == 0 { + return nil + } + + pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{ + "variants": len(expansions), + }) + expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2) + expKwTh := chatManage.KeywordThreshold * 0.8 + + // Concurrent expansion retrieval across queries and search targets + expResults := make([]*types.SearchResult, 0, expTopK*len(expansions)) + var muExp sync.Mutex + var wgExp sync.WaitGroup + jobs := len(expansions) * len(chatManage.SearchTargets) + capSem := 16 + if jobs < capSem { + capSem = jobs + } + if capSem <= 0 { + capSem = 1 + } + sem := make(chan struct{}, capSem) + pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{ + "jobs": jobs, + "cap": capSem, + }) + for _, q := range expansions { + for _, target := range chatManage.SearchTargets { + wgExp.Add(1) + go func(q string, t *types.SearchTarget) { + defer wgExp.Done() + sem <- struct{}{} + defer func() { <-sem }() + paramsExp := types.SearchParams{ + QueryText: q, + VectorThreshold: chatManage.VectorThreshold, + KeywordThreshold: expKwTh, + MatchCount: expTopK, + DisableVectorMatch: false, + DisableKeywordsMatch: false, + SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage + } + // Apply knowledge ID filter if this is a partial KB search + if t.Type == types.SearchTargetTypeKnowledge { + paramsExp.KnowledgeIDs = t.KnowledgeIDs + } + res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp) + if err != nil { + pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{ + "kb_id": t.KnowledgeBaseID, + "error": err.Error(), + }) + return + } + if len(res) > 0 { + for _, r := range res { + r.KnowledgeBaseID = t.KnowledgeBaseID + } + pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{ + "kb_id": t.KnowledgeBaseID, + "query": q, + "hits": len(res), + }) + muExp.Lock() + expResults = append(expResults, res...) + muExp.Unlock() + } + }(q, target) + } + } + wgExp.Wait() + + if len(expResults) > 0 { + pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{ + "added": len(expResults), + }) + } + return expResults +} + +// expandQueries generates query variants locally without LLM to improve keyword recall. +// Uses simple techniques: word reordering, stopword removal, key phrase extraction. +func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string { + query := strings.TrimSpace(chatManage.RewriteQuery) + if query == "" { + return nil + } + + expansions := make([]string, 0, 5) + seen := make(map[string]struct{}) + seen[strings.ToLower(query)] = struct{}{} + if q := strings.ToLower(chatManage.Query); q != "" { + seen[q] = struct{}{} + } + + addIfNew := func(s string) { + s = strings.TrimSpace(s) + if s == "" || len(s) < 3 { + return + } + key := strings.ToLower(s) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + expansions = append(expansions, s) + } + + // 1. Remove common stopwords and create keyword-only variant + keywords := extractKeywords(query) + if len(keywords) >= 2 { + addIfNew(strings.Join(keywords, " ")) + } + + // 2. Extract quoted phrases or key segments + phrases := extractPhrases(query) + for _, phrase := range phrases { + addIfNew(phrase) + } + + // 3. Split by common delimiters and use longest segment + segments := splitByDelimiters(query) + for _, seg := range segments { + if len(seg) > 5 { + addIfNew(seg) + } + } + + // 4. Remove question words (什么/如何/怎么/为什么/哪个 etc.) + cleaned := removeQuestionWords(query) + if cleaned != query { + addIfNew(cleaned) + } + + // Limit to 5 expansions + if len(expansions) > 5 { + expansions = expansions[:5] + } + + pipelineInfo(ctx, "Search", "local_expansion_result", map[string]interface{}{ + "variants": len(expansions), + }) + return expansions +} + +// Common Chinese and English stopwords +var stopwords = map[string]struct{}{ + "的": {}, "是": {}, "在": {}, "了": {}, "和": {}, "与": {}, "或": {}, + "a": {}, "an": {}, "the": {}, "is": {}, "are": {}, "was": {}, "were": {}, + "be": {}, "been": {}, "being": {}, "have": {}, "has": {}, "had": {}, + "do": {}, "does": {}, "did": {}, "will": {}, "would": {}, "could": {}, + "should": {}, "may": {}, "might": {}, "must": {}, "can": {}, + "to": {}, "of": {}, "in": {}, "for": {}, "on": {}, "with": {}, "at": {}, + "by": {}, "from": {}, "as": {}, "into": {}, "through": {}, "about": {}, + "what": {}, "how": {}, "why": {}, "when": {}, "where": {}, "which": {}, + "who": {}, "whom": {}, "whose": {}, +} + +// Question words in Chinese +var questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`) + +func extractKeywords(text string) []string { + words := tokenize(text) + keywords := make([]string, 0, len(words)) + for _, w := range words { + lower := strings.ToLower(w) + if _, isStop := stopwords[lower]; !isStop && len(w) > 1 { + keywords = append(keywords, w) + } + } + return keywords +} + +func extractPhrases(text string) []string { + // Extract quoted content + var phrases []string + re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`) + matches := re.FindAllStringSubmatch(text, -1) + for _, m := range matches { + if len(m) > 1 && len(m[1]) > 2 { + phrases = append(phrases, m[1]) + } + } + return phrases +} + +func splitByDelimiters(text string) []string { + // Split by common delimiters + re := regexp.MustCompile(`[,，;；、。！？!?\s]+`) + parts := re.Split(text, -1) + var result []string + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + result = append(result, p) + } + } + return result +} + +func removeQuestionWords(text string) string { + return strings.TrimSpace(questionWords.ReplaceAllString(text, "")) +} + +func tokenize(text string) []string { + var tokens []string + var current strings.Builder + + for _, r := range text { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + current.WriteRune(r) + } else if unicode.Is(unicode.Han, r) { + // Flush current token + if current.Len() > 0 { + tokens = append(tokens, current.String()) + current.Reset() + } + // Chinese character as single token + tokens = append(tokens, string(r)) + } else { + // Delimiter + if current.Len() > 0 { + tokens = append(tokens, current.String()) + current.Reset() + } + } + } + if current.Len() > 0 { + tokens = append(tokens, current.String()) + } + return tokens +} diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 8b7094d7..3c355800 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -264,6 +264,20 @@ func (p *PluginRerank) rerank(ctx context.Context, rankFilter = append(rankFilter, result) } } + + // Fallback: if threshold filtering removed all results, keep top-N as safety net + // This prevents returning empty results when all scores are below threshold + if len(rankFilter) == 0 && len(rerankResp) > 0 { + fallbackN := min(3, len(rerankResp)) + rankFilter = rerankResp[:fallbackN] + pipelineInfo(ctx, "Rerank", "fallback_topn", map[string]interface{}{ + "reason": "all_below_threshold", + "threshold": chatManage.RerankThreshold, + "fallback_n": fallbackN, + "top_score": rerankResp[0].RelevanceScore, + }) + } + return rankFilter } diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 302ff414..8d9ed23d 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -3,10 +3,8 @@ package chatpipline import ( "context" "fmt" - "regexp" "strings" "sync" - "unicode" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" @@ -126,107 +124,13 @@ func (p *PluginSearch) OnEvent(ctx context.Context, } // If recall is low, attempt query expansion with keyword-focused search - if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK/2) { - pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{ - "current": len(chatManage.SearchResult), - "threshold": chatManage.EmbeddingTopK / 2, - }) - expansions := p.expandQueries(ctx, chatManage) - if len(expansions) > 0 { - pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{ - "variants": len(expansions), - }) - expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2) - expKwTh := chatManage.KeywordThreshold * 0.8 - // Concurrent expansion retrieval across queries and search targets - expResults := make([]*types.SearchResult, 0, expTopK*len(expansions)) - var muExp sync.Mutex - var wgExp sync.WaitGroup - jobs := len(expansions) * len(chatManage.SearchTargets) - capSem := 16 - if jobs < capSem { - capSem = jobs - } - if capSem <= 0 { - capSem = 1 - } - sem := make(chan struct{}, capSem) - pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{ - "jobs": jobs, - "cap": capSem, - }) - for _, q := range expansions { - for _, target := range chatManage.SearchTargets { - wgExp.Add(1) - go func(q string, t *types.SearchTarget) { - defer wgExp.Done() - sem <- struct{}{} - defer func() { <-sem }() - paramsExp := types.SearchParams{ - QueryText: q, - VectorThreshold: chatManage.VectorThreshold, - KeywordThreshold: expKwTh, - MatchCount: expTopK, - DisableVectorMatch: true, - DisableKeywordsMatch: false, - } - // Apply knowledge ID filter if this is a partial KB search - if t.Type == types.SearchTargetTypeKnowledge { - paramsExp.KnowledgeIDs = t.KnowledgeIDs - } - res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp) - if err != nil { - pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{ - "kb_id": t.KnowledgeBaseID, - "error": err.Error(), - }) - return - } - if len(res) > 0 { - for _, r := range res { - r.KnowledgeBaseID = t.KnowledgeBaseID - } - pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{ - "kb_id": t.KnowledgeBaseID, - "query": q, - "hits": len(res), - }) - muExp.Lock() - expResults = append(expResults, res...) - muExp.Unlock() - } - }(q, target) - } - } - wgExp.Wait() - if len(expResults) > 0 { - // Scores already normalized in HybridSearch - pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{ - "added": len(expResults), - }) - chatManage.SearchResult = append(chatManage.SearchResult, expResults...) - } + if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) { + expResults := p.runQueryExpansion(ctx, chatManage) + if len(expResults) > 0 { + chatManage.SearchResult = append(chatManage.SearchResult, expResults...) } } - // Add relevant results from chat history - historyResult := p.getSearchResultFromHistory(chatManage) - if historyResult != nil { - pipelineInfo(ctx, "Search", "history_hits", map[string]interface{}{ - "session_id": chatManage.SessionID, - "history_hits": len(historyResult), - }) - chatManage.SearchResult = append(chatManage.SearchResult, historyResult...) - } - - // Remove duplicate results - before := len(chatManage.SearchResult) - chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) - pipelineInfo(ctx, "Search", "dedup_summary", map[string]interface{}{ - "before": before, - "after": len(chatManage.SearchResult), - }) - // Log final scores after all processing for i, r := range chatManage.SearchResult { pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{ @@ -253,7 +157,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context, } // getSearchResultFromHistory retrieves relevant knowledge references from chat history -func (p *PluginSearch) getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult { +func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult { if len(chatManage.History) == 0 { return nil } @@ -275,21 +179,11 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult contentSig := make(map[string]string) // sig -> first chunk ID var uniqueResults []*types.SearchResult for _, r := range results { - keys := []string{r.ID} - if r.ParentChunkID != "" { - keys = append(keys, "parent:"+r.ParentChunkID) - } - dup := false - dupKey := "" - for _, k := range keys { - if seen[k] { - dup = true - dupKey = k - break - } - } - if dup { - logger.Debugf(context.Background(), "Dedup: chunk %s removed due to key: %s", r.ID, dupKey) + // Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID + // as duplicates, because different child chunks of the same parent carry + // different content segments that may all be relevant. + if seen[r.ID] { + logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID) continue } sig := buildContentSignature(r.Content) @@ -300,9 +194,7 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult } contentSig[sig] = r.ID } - for _, k := range keys { - seen[k] = true - } + seen[r.ID] = true uniqueResults = append(uniqueResults, r) } return uniqueResults @@ -370,10 +262,11 @@ func (p *PluginSearch) searchByTargets( // Build params for rewrite query params := types.SearchParams{ - QueryText: strings.TrimSpace(chatManage.RewriteQuery), - VectorThreshold: chatManage.VectorThreshold, - KeywordThreshold: chatManage.KeywordThreshold, - MatchCount: chatManage.EmbeddingTopK, + QueryText: strings.TrimSpace(chatManage.RewriteQuery), + VectorThreshold: chatManage.VectorThreshold, + KeywordThreshold: chatManage.KeywordThreshold, + MatchCount: chatManage.EmbeddingTopK, + SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage } // Apply knowledge ID filter if this is a partial KB search if t.Type == types.SearchTargetTypeKnowledge { @@ -546,156 +439,3 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types }) return res } - -// expandQueries generates query variants locally without LLM to improve keyword recall -// Uses simple techniques: word reordering, stopword removal, key phrase extraction -func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string { - query := strings.TrimSpace(chatManage.RewriteQuery) - if query == "" { - return nil - } - - expansions := make([]string, 0, 5) - seen := make(map[string]struct{}) - seen[strings.ToLower(query)] = struct{}{} - if q := strings.ToLower(chatManage.Query); q != "" { - seen[q] = struct{}{} - } - - addIfNew := func(s string) { - s = strings.TrimSpace(s) - if s == "" || len(s) < 3 { - return - } - key := strings.ToLower(s) - if _, ok := seen[key]; ok { - return - } - seen[key] = struct{}{} - expansions = append(expansions, s) - } - - // 1. Remove common stopwords and create keyword-only variant - keywords := extractKeywords(query) - if len(keywords) >= 2 { - addIfNew(strings.Join(keywords, " ")) - } - - // 2. Extract quoted phrases or key segments - phrases := extractPhrases(query) - for _, phrase := range phrases { - addIfNew(phrase) - } - - // 3. Split by common delimiters and use longest segment - segments := splitByDelimiters(query) - for _, seg := range segments { - if len(seg) > 5 { - addIfNew(seg) - } - } - - // 4. Remove question words (什么/如何/怎么/为什么/哪个 etc.) - cleaned := removeQuestionWords(query) - if cleaned != query { - addIfNew(cleaned) - } - - // Limit to 5 expansions - if len(expansions) > 5 { - expansions = expansions[:5] - } - - pipelineInfo(ctx, "Search", "local_expansion_result", map[string]interface{}{ - "variants": len(expansions), - }) - return expansions -} - -// Common Chinese and English stopwords -var stopwords = map[string]struct{}{ - "的": {}, "是": {}, "在": {}, "了": {}, "和": {}, "与": {}, "或": {}, - "a": {}, "an": {}, "the": {}, "is": {}, "are": {}, "was": {}, "were": {}, - "be": {}, "been": {}, "being": {}, "have": {}, "has": {}, "had": {}, - "do": {}, "does": {}, "did": {}, "will": {}, "would": {}, "could": {}, - "should": {}, "may": {}, "might": {}, "must": {}, "can": {}, - "to": {}, "of": {}, "in": {}, "for": {}, "on": {}, "with": {}, "at": {}, - "by": {}, "from": {}, "as": {}, "into": {}, "through": {}, "about": {}, - "what": {}, "how": {}, "why": {}, "when": {}, "where": {}, "which": {}, - "who": {}, "whom": {}, "whose": {}, -} - -// Question words in Chinese -var questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`) - -func extractKeywords(text string) []string { - words := tokenize(text) - keywords := make([]string, 0, len(words)) - for _, w := range words { - lower := strings.ToLower(w) - if _, isStop := stopwords[lower]; !isStop && len(w) > 1 { - keywords = append(keywords, w) - } - } - return keywords -} - -func extractPhrases(text string) []string { - // Extract quoted content - var phrases []string - re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`) - matches := re.FindAllStringSubmatch(text, -1) - for _, m := range matches { - if len(m) > 1 && len(m[1]) > 2 { - phrases = append(phrases, m[1]) - } - } - return phrases -} - -func splitByDelimiters(text string) []string { - // Split by common delimiters - re := regexp.MustCompile(`[,，;；、。！？!?\s]+`) - parts := re.Split(text, -1) - var result []string - for _, p := range parts { - p = strings.TrimSpace(p) - if p != "" { - result = append(result, p) - } - } - return result -} - -func removeQuestionWords(text string) string { - return strings.TrimSpace(questionWords.ReplaceAllString(text, "")) -} - -func tokenize(text string) []string { - var tokens []string - var current strings.Builder - - for _, r := range text { - if unicode.IsLetter(r) || unicode.IsDigit(r) { - current.WriteRune(r) - } else if unicode.Is(unicode.Han, r) { - // Flush current token - if current.Len() > 0 { - tokens = append(tokens, current.String()) - current.Reset() - } - // Chinese character as single token - tokens = append(tokens, string(r)) - } else { - // Delimiter - if current.Len() > 0 { - tokens = append(tokens, current.String()) - current.Reset() - } - } - } - if current.Len() > 0 { - tokens = append(tokens, current.String()) - } - return tokens -} diff --git a/internal/application/service/chat_pipline/search_parallel.go b/internal/application/service/chat_pipline/search_parallel.go index 054b7e48..895fbb83 100644 --- a/internal/application/service/chat_pipline/search_parallel.go +++ b/internal/application/service/chat_pipline/search_parallel.go @@ -154,7 +154,6 @@ func (p *PluginSearchParallel) OnEvent(ctx context.Context, // Merge results from both searches (no concurrent access now) chatManage.SearchResult = append(chunkChatManage.SearchResult, entityChatManage.SearchResult...) - chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) // Log any errors but don't fail the pipeline if at least one search succeeded if chunkSearchErr != nil { diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index b5842780..1171b9c6 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -1315,6 +1315,34 @@ type ProcessChunksOptions struct { QuestionCount int EnableMultimodel bool StoredImages []docparser.StoredImage + // ParentChunks holds parent chunk data when parent-child chunking is enabled. + // When set, the chunks passed to processChunks are child chunks, and each + // child's ParentIndex references an entry in this slice. + ParentChunks []types.ParsedParentChunk +} + +// buildParentChildConfigs derives parent and child SplitterConfig from ChunkingConfig. +// The base config (already validated with defaults) is used for separators. +func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) { + parentSize := cc.ParentChunkSize + if parentSize <= 0 { + parentSize = 1024 + } + childSize := cc.ChildChunkSize + if childSize <= 0 { + childSize = 256 + } + parent = chunker.SplitterConfig{ + ChunkSize: parentSize, + ChunkOverlap: base.ChunkOverlap, // reuse configured overlap for parents + Separators: base.Separators, + } + child = chunker.SplitterConfig{ + ChunkSize: childSize, + ChunkOverlap: childSize / 5, // ~20% overlap for child chunks + Separators: base.Separators, + } + return } // processChunks processes chunks and creates embeddings for knowledge content @@ -1447,8 +1475,44 @@ func (s *knowledgeService) processChunks(ctx context.Context, } } - // 重新分配容量，考虑图片相关的Chunk - insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount) + // === Parent-Child Chunking: create parent chunks first === + hasParentChild := len(options.ParentChunks) > 0 + var parentDBChunks []*types.Chunk // indexed by ParsedParentChunk position + if hasParentChild { + parentDBChunks = make([]*types.Chunk, len(options.ParentChunks)) + for i, pc := range options.ParentChunks { + parentDBChunks[i] = &types.Chunk{ + ID: uuid.New().String(), + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: pc.Content, + ChunkIndex: pc.Seq, + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StartAt: pc.Start, + EndAt: pc.End, + ChunkType: types.ChunkTypeParentText, + } + } + // Set prev/next links for parent chunks + for i := range parentDBChunks { + if i > 0 { + parentDBChunks[i-1].NextChunkID = parentDBChunks[i].ID + parentDBChunks[i].PreChunkID = parentDBChunks[i-1].ID + } + } + logger.Infof(ctx, "Created %d parent chunks for parent-child strategy", len(parentDBChunks)) + } + + // 重新分配容量，考虑图片相关的Chunk + parent chunks + parentCount := len(options.ParentChunks) + insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount+parentCount) + // Add parent chunks first (they go into DB but NOT into the vector index) + if hasParentChild { + insertChunks = append(insertChunks, parentDBChunks...) + } for idx, chunkData := range chunks { if strings.TrimSpace(chunkData.Content) == "" { @@ -1470,6 +1534,12 @@ func (s *knowledgeService) processChunks(ctx context.Context, EndAt: int(chunkData.End), ChunkType: types.ChunkTypeText, } + + // Wire up ParentChunkID for child chunks + if hasParentChild && chunkData.ParentIndex >= 0 && chunkData.ParentIndex < len(parentDBChunks) { + textChunk.ParentChunkID = parentDBChunks[chunkData.ParentIndex].ID + } + chunks[idx].ChunkID = textChunk.ID insertChunks = append(insertChunks, textChunk) } @@ -1479,30 +1549,43 @@ func (s *knowledgeService) processChunks(ctx context.Context, return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex }) - // 仅为文本类型的Chunk设置前后关系 + // 仅为文本类型的Chunk设置前后关系（child chunks only, parents already linked above） textChunks := make([]*types.Chunk, 0, len(chunks)) for _, chunk := range insertChunks { - if chunk.ChunkType == types.ChunkTypeText { + if chunk.ChunkType == types.ChunkTypeText && chunk.ParentChunkID != "" { + // This is a child chunk in parent-child mode + textChunks = append(textChunks, chunk) + } else if chunk.ChunkType == types.ChunkTypeText && !hasParentChild { + // Normal flat chunk (no parent-child mode) textChunks = append(textChunks, chunk) } } - // 设置文本Chunk之间的前后关系 - for i, chunk := range textChunks { - if i > 0 { - textChunks[i-1].NextChunkID = chunk.ID - } - if i < len(textChunks)-1 { - textChunks[i+1].PreChunkID = chunk.ID + // 设置文本Chunk之间的前后关系 (skip if parent-child, children don't need prev/next links) + if !hasParentChild { + for i, chunk := range textChunks { + if i > 0 { + textChunks[i-1].NextChunkID = chunk.ID + } + if i < len(textChunks)-1 { + textChunks[i+1].PreChunkID = chunk.ID + } } } - // Create index information for each chunk (without generated questions for now) - indexInfoList := make([]*types.IndexInfo, 0, len(insertChunks)) - for _, chunk := range insertChunks { - // Add original chunk content to index + // Create index information — only for child/flat chunks, NOT parent chunks. + // Parent chunks are stored for context retrieval but do not need vector embeddings. + // Prepend the document title to improve semantic alignment between + // question-style queries and statement-style chunk content. + indexInfoList := make([]*types.IndexInfo, 0, len(textChunks)) + titlePrefix := "" + if t := strings.TrimSpace(knowledge.Title); t != "" { + titlePrefix = t + "\n" + } + for _, chunk := range textChunks { + indexContent := titlePrefix + chunk.Content indexInfoList = append(indexInfoList, &types.IndexInfo{ - Content: chunk.Content, + Content: indexContent, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, @@ -3031,7 +3114,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl tagIDMapping := map[string]string{} // srcTagID -> dstTagID targetChunks := make([]*types.Chunk, 0, 10) chunkType := []types.ChunkType{ - types.ChunkTypeText, types.ChunkTypeSummary, + types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary, types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, } for { @@ -6626,24 +6709,47 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context, chunkCfg.Separators = []string{"\n\n", "\n", "。"} } - splitChunks := chunker.SplitText(clean, chunkCfg) - parsed := make([]types.ParsedChunk, len(splitChunks)) - for i, c := range splitChunks { - parsed[i] = types.ParsedChunk{ - Content: c.Content, - Seq: c.Seq, - Start: c.Start, - End: c.End, + var parsed []types.ParsedChunk + var opts ProcessChunksOptions + + if kb.ChunkingConfig.EnableParentChild { + parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg) + pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg) + parsed = make([]types.ParsedChunk, len(pcResult.Children)) + for i, c := range pcResult.Children { + parsed[i] = types.ParsedChunk{ + Content: c.Content, + Seq: c.Seq, + Start: c.Start, + End: c.End, + ParentIndex: c.ParentIndex, + } + } + parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents)) + for i, p := range pcResult.Parents { + parentChunks[i] = types.ParsedParentChunk{Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End} + } + opts.ParentChunks = parentChunks + } else { + splitChunks := chunker.SplitText(clean, chunkCfg) + parsed = make([]types.ParsedChunk, len(splitChunks)) + for i, c := range splitChunks { + parsed[i] = types.ParsedChunk{ + Content: c.Content, + Seq: c.Seq, + Start: c.Start, + End: c.End, + } } } if doSync { - s.processChunks(ctx, kb, knowledge, parsed) + s.processChunks(ctx, kb, knowledge, parsed, opts) return } newCtx := logger.CloneContext(ctx) - go s.processChunks(newCtx, kb, knowledge, parsed) + go s.processChunks(newCtx, kb, knowledge, parsed, opts) } func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowledge *types.Knowledge) error { @@ -7198,25 +7304,49 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e chunkCfg.Separators = []string{"\n\n", "\n", "。"} } - splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg) - chunks = make([]types.ParsedChunk, len(splitChunks)) - for i, c := range splitChunks { - chunks[i] = types.ParsedChunk{ - Content: c.Content, - Seq: c.Seq, - Start: c.Start, - End: c.End, - } - } - logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID) - - // Step 4: Process chunks (vectorize + index + enqueue async tasks) - s.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{ + processOpts := ProcessChunksOptions{ EnableQuestionGeneration: payload.EnableQuestionGeneration, QuestionCount: payload.QuestionCount, EnableMultimodel: payload.EnableMultimodel, StoredImages: storedImages, - }) + } + + if kb.ChunkingConfig.EnableParentChild { + parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg) + pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg) + chunks = make([]types.ParsedChunk, len(pcResult.Children)) + for i, c := range pcResult.Children { + chunks[i] = types.ParsedChunk{ + Content: c.Content, + Seq: c.Seq, + Start: c.Start, + End: c.End, + ParentIndex: c.ParentIndex, + } + } + parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents)) + for i, p := range pcResult.Parents { + parentChunks[i] = types.ParsedParentChunk{Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End} + } + processOpts.ParentChunks = parentChunks + logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s", + len(pcResult.Parents), len(pcResult.Children), knowledge.ID) + } else { + splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg) + chunks = make([]types.ParsedChunk, len(splitChunks)) + for i, c := range splitChunks { + chunks[i] = types.ParsedChunk{ + Content: c.Content, + Seq: c.Seq, + Start: c.Start, + End: c.End, + } + } + logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID) + } + + // Step 4: Process chunks (vectorize + index + enqueue async tasks) + s.processChunks(ctx, kb, knowledge, chunks, processOpts) return nil } diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index ca7ed209..c85fd0c8 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -616,7 +616,9 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, return nil, err } - matchCount := params.MatchCount * 3 + // Use 5x over-retrieval to ensure sufficient candidates for RRF fusion and reranking. + // Minimum 50 to handle large knowledge bases with diverse content. + matchCount := max(params.MatchCount*5, 50) // Add vector retrieval params if supported if retrieveEngine.SupportRetriever(types.VectorRetrieverType) && !params.DisableVectorMatch { @@ -790,14 +792,16 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, } } - // Compute RRF scores + // Compute weighted RRF scores (vector retrieval weighted higher for semantic relevance) + const vectorWeight = 0.7 + const keywordWeight = 0.3 for chunkID := range chunkInfoMap { rrfScore := 0.0 if rank, ok := vectorRanks[chunkID]; ok { - rrfScore += 1.0 / float64(rrfK+rank) + rrfScore += vectorWeight / float64(rrfK+rank) } if rank, ok := keywordRanks[chunkID]; ok { - rrfScore += 1.0 / float64(rrfK+rank) + rrfScore += keywordWeight / float64(rrfK+rank) } rrfScores[chunkID] = rrfScore } @@ -858,7 +862,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, deduplicatedChunks = deduplicatedChunks[:params.MatchCount] } - return s.processSearchResults(ctx, deduplicatedChunks) + return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment) } // iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found @@ -1100,6 +1104,7 @@ func (s *knowledgeBaseService) matchesNegativeQuestions(queryTextLower string, n // processSearchResults handles the processing of search results, optimizing database queries func (s *knowledgeBaseService) processSearchResults(ctx context.Context, chunks []*types.IndexWithScore, + skipEnrichment bool, ) ([]*types.SearchResult, error) { if len(chunks) == 0 { return nil, nil @@ -1157,34 +1162,36 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, chunkMap[chunk.ID] = chunk processedChunkIDs[chunk.ID] = true - // Collect parent chunks - if chunk.ParentChunkID != "" && !processedChunkIDs[chunk.ParentChunkID] { - additionalChunkIDs = append(additionalChunkIDs, chunk.ParentChunkID) - processedChunkIDs[chunk.ParentChunkID] = true + if !skipEnrichment { + // Collect parent chunks + if chunk.ParentChunkID != "" && !processedChunkIDs[chunk.ParentChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.ParentChunkID) + processedChunkIDs[chunk.ParentChunkID] = true - // Pass score to parent - chunkScores[chunk.ParentChunkID] = chunkScores[chunk.ID] - chunkMatchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk - } - - // Collect related chunks - relationChunkIDs := s.collectRelatedChunkIDs(chunk, processedChunkIDs) - for _, chunkID := range relationChunkIDs { - additionalChunkIDs = append(additionalChunkIDs, chunkID) - chunkMatchTypes[chunkID] = types.MatchTypeRelationChunk - } - - // Add nearby chunks (prev and next) - if slices.Contains([]string{types.ChunkTypeText}, chunk.ChunkType) { - if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] { - additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID) - processedChunkIDs[chunk.NextChunkID] = true - chunkMatchTypes[chunk.NextChunkID] = types.MatchTypeNearByChunk + // Pass score to parent + chunkScores[chunk.ParentChunkID] = chunkScores[chunk.ID] + chunkMatchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk } - if chunk.PreChunkID != "" && !processedChunkIDs[chunk.PreChunkID] { - additionalChunkIDs = append(additionalChunkIDs, chunk.PreChunkID) - processedChunkIDs[chunk.PreChunkID] = true - chunkMatchTypes[chunk.PreChunkID] = types.MatchTypeNearByChunk + + // Collect related chunks + relationChunkIDs := s.collectRelatedChunkIDs(chunk, processedChunkIDs) + for _, chunkID := range relationChunkIDs { + additionalChunkIDs = append(additionalChunkIDs, chunkID) + chunkMatchTypes[chunkID] = types.MatchTypeRelationChunk + } + + // Add nearby chunks (prev and next) + if slices.Contains([]string{types.ChunkTypeText}, chunk.ChunkType) { + if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID) + processedChunkIDs[chunk.NextChunkID] = true + chunkMatchTypes[chunk.NextChunkID] = types.MatchTypeNearByChunk + } + if chunk.PreChunkID != "" && !processedChunkIDs[chunk.PreChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.PreChunkID) + processedChunkIDs[chunk.PreChunkID] = true + chunkMatchTypes[chunk.PreChunkID] = types.MatchTypeNearByChunk + } } } } @@ -1235,26 +1242,28 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, } // Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input - for chunkID, chunk := range chunkMap { - if addedChunkIDs[chunkID] || !s.isValidTextChunk(chunk) { - continue - } - - score, hasScore := chunkScores[chunkID] - if !hasScore || score <= 0 { - score = 0.0 - } - - if knowledge, ok := knowledgeMap[chunk.KnowledgeID]; ok { - matchType := types.MatchTypeParentChunk - if specificType, exists := chunkMatchTypes[chunkID]; exists { - matchType = specificType - } else { - logger.Warnf(ctx, "Unkonwn match type for chunk: %s", chunkID) + if !skipEnrichment { + for chunkID, chunk := range chunkMap { + if addedChunkIDs[chunkID] || !s.isValidTextChunk(chunk) { continue } - matchedContent := chunkMatchedContents[chunkID] - searchResults = append(searchResults, s.buildSearchResult(chunk, knowledge, score, matchType, matchedContent)) + + score, hasScore := chunkScores[chunkID] + if !hasScore || score <= 0 { + score = 0.0 + } + + if knowledge, ok := knowledgeMap[chunk.KnowledgeID]; ok { + matchType := types.MatchTypeParentChunk + if specificType, exists := chunkMatchTypes[chunkID]; exists { + matchType = specificType + } else { + logger.Warnf(ctx, "Unkonwn match type for chunk: %s", chunkID) + continue + } + matchedContent := chunkMatchedContents[chunkID] + searchResults = append(searchResults, s.buildSearchResult(chunk, knowledge, score, matchType, matchedContent)) + } } } logger.Infof(ctx, "Search results processed, total: %d", len(searchResults)) diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 2c29b356..ac2aeeef 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -96,6 +96,9 @@ type KBModelConfigRequest struct { ChunkOverlap int `json:"chunkOverlap"` Separators []string `json:"separators"` ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"` + EnableParentChild bool `json:"enableParentChild"` + ParentChunkSize int `json:"parentChunkSize,omitempty"` + ChildChunkSize int `json:"childChunkSize,omitempty"` } `json:"documentSplitting"` // 多模态配置（仅模型相关；存储引擎在 storageProvider 中配置） @@ -289,6 +292,13 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators } kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules + kb.ChunkingConfig.EnableParentChild = req.DocumentSplitting.EnableParentChild + if req.DocumentSplitting.ParentChunkSize > 0 { + kb.ChunkingConfig.ParentChunkSize = req.DocumentSplitting.ParentChunkSize + } + if req.DocumentSplitting.ChildChunkSize > 0 { + kb.ChunkingConfig.ChildChunkSize = req.DocumentSplitting.ChildChunkSize + } // 更新多模态配置 if req.Multimodal.Enabled { diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 94de8031..37dff63d 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -36,7 +36,7 @@ type SplitterConfig struct { func DefaultConfig() SplitterConfig { return SplitterConfig{ ChunkSize: 512, - ChunkOverlap: 50, + ChunkOverlap: 128, Separators: []string{"\n\n", "\n", "。"}, } } @@ -316,6 +316,52 @@ func isSeparatorOnly(s string) bool { return true } +// ParentChildResult holds the two-level chunking output. +// Parent chunks provide context (large window), child chunks are used for +// embedding/retrieval (small window). Each child carries its ParentIndex so +// the caller can wire up ParentChunkID after DB insertion. +type ParentChildResult struct { + Parents []Chunk + Children []ChildChunk +} + +// ChildChunk extends Chunk with a reference to its parent. +type ChildChunk struct { + Chunk + ParentIndex int // index into ParentChildResult.Parents +} + +// SplitTextParentChild performs two-level chunking: +// 1. Split text into large parent chunks (parentCfg). +// 2. Split each parent into smaller child chunks (childCfg) for embedding. +// +// The child Seq is globally unique across the entire document. +func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { + parents := SplitText(text, parentCfg) + if len(parents) == 0 { + return ParentChildResult{} + } + + var children []ChildChunk + childSeq := 0 + for pi, parent := range parents { + subs := SplitText(parent.Content, childCfg) + for _, sub := range subs { + // Adjust offsets: sub positions are relative to parent content, + // shift to document-level offsets. + sub.Seq = childSeq + sub.Start += parent.Start + sub.End = sub.Start + runeLen(sub.Content) + children = append(children, ChildChunk{ + Chunk: sub, + ParentIndex: pi, + }) + childSeq++ + } + } + return ParentChildResult{Parents: parents, Children: children} +} + // ExtractImageRefs extracts markdown image references from text. var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) diff --git a/internal/searchutil/textutil.go b/internal/searchutil/textutil.go index 382fd859..5e6b4ade 100644 --- a/internal/searchutil/textutil.go +++ b/internal/searchutil/textutil.go @@ -4,6 +4,9 @@ import ( "crypto/md5" "encoding/hex" "strings" + "unicode" + + "github.com/Tencent/WeKnora/internal/types" ) // BuildContentSignature creates a normalized MD5 signature for content to detect duplicates. @@ -20,20 +23,55 @@ func BuildContentSignature(content string) string { return hex.EncodeToString(hash[:]) } -// TokenizeSimple tokenizes text into a set of words (simple whitespace-based). -// Returns a map where keys are lowercase tokens with length > 1. +// containsChinese checks whether text contains any CJK unified ideographs. +func containsChinese(text string) bool { + for _, r := range text { + if unicode.Is(unicode.Han, r) { + return true + } + } + return false +} + +// TokenizeSimple tokenizes text into a set of unique tokens. +// For text containing Chinese characters, it uses jieba segmentation for accurate word boundaries. +// For pure non-Chinese text, it falls back to whitespace-based splitting. +// Returns a map where keys are lowercase tokens with rune length > 1. func TokenizeSimple(text string) map[string]struct{} { - text = strings.ToLower(text) - fields := strings.Fields(text) - set := make(map[string]struct{}, len(fields)) - for _, f := range fields { - if len(f) > 1 { - set[f] = struct{}{} + text = strings.ToLower(strings.TrimSpace(text)) + if text == "" { + return nil + } + + var words []string + if containsChinese(text) { + // Use jieba for Chinese text segmentation (search mode for finer granularity) + words = types.Jieba.CutForSearch(text, true) + } else { + words = strings.Fields(text) + } + + set := make(map[string]struct{}, len(words)) + for _, w := range words { + w = strings.TrimSpace(w) + // Filter out single-rune tokens and pure punctuation/whitespace + if len([]rune(w)) > 1 && !isAllPunct(w) { + set[w] = struct{}{} } } return set } +// isAllPunct checks if a string consists entirely of punctuation or whitespace. +func isAllPunct(s string) bool { + for _, r := range s { + if !unicode.IsPunct(r) && !unicode.IsSpace(r) && !unicode.IsSymbol(r) { + return false + } + } + return true +} + // Jaccard calculates Jaccard similarity between two token sets. // Returns a value between 0 and 1, where 1 means identical sets. func Jaccard(a, b map[string]struct{}) float64 { diff --git a/internal/types/chunk.go b/internal/types/chunk.go index eb16a057..f283cc86 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -14,6 +14,8 @@ type ChunkType = string const ( // ChunkTypeText 表示普通的文本 Chunk ChunkTypeText ChunkType = "text" + // ChunkTypeParentText 表示父子分块策略中的父文本 Chunk（仅用于上下文，不参与向量索引） + ChunkTypeParentText ChunkType = "parent_text" // ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk ChunkTypeImageOCR ChunkType = "image_ocr" // ChunkTypeImageCaption 表示图片描述的 Chunk diff --git a/internal/types/docparser.go b/internal/types/docparser.go index e47a9872..1fb46eef 100644 --- a/internal/types/docparser.go +++ b/internal/types/docparser.go @@ -67,6 +67,21 @@ type ParsedChunk struct { End int Images []ParsedImage ChunkID string // populated by processChunks with the actual DB UUID + + // ParentIndex is set when using parent-child chunking strategy. + // -1 (or unset/0 for flat chunks) means this is a top-level chunk. + // >= 0 means this is a child chunk referencing the parent at this index + // in the ParentChunks slice of ProcessChunksOptions. + ParentIndex int +} + +// ParsedParentChunk represents a parent chunk in the parent-child strategy. +// Parent chunks are stored in DB for context retrieval but NOT vector-indexed. +type ParsedParentChunk struct { + Content string + Seq int + Start int + End int } type ParsedImage struct { diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 6f68b026..451c10b7 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -117,6 +117,16 @@ type ChunkingConfig struct { // ParserEngineRules configures which parser engine to use for each file type. // When empty, the builtin engine is used for all types. ParserEngineRules []ParserEngineRule `yaml:"parser_engine_rules,omitempty" json:"parser_engine_rules,omitempty"` + // EnableParentChild enables two-level parent-child chunking strategy. + // When enabled, large parent chunks provide context while small child chunks + // are used for vector matching. Retrieval matches on child but returns parent content. + EnableParentChild bool `yaml:"enable_parent_child,omitempty" json:"enable_parent_child,omitempty"` + // ParentChunkSize is the size of parent chunks (default: 1024). + // Only used when EnableParentChild is true. + ParentChunkSize int `yaml:"parent_chunk_size,omitempty" json:"parent_chunk_size,omitempty"` + // ChildChunkSize is the size of child chunks used for embedding (default: 256). + // Only used when EnableParentChild is true. + ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"` } // ResolveParserEngine returns the engine name for the given file type diff --git a/internal/types/search.go b/internal/types/search.go index 68b12ec7..2bbe2a7b 100644 --- a/internal/types/search.go +++ b/internal/types/search.go @@ -142,6 +142,10 @@ type SearchParams struct { KnowledgeIDs []string `json:"knowledge_ids"` TagIDs []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering) OnlyRecommended bool `json:"only_recommended"` + // SkipContextEnrichment skips fetching parent, nearby, and relation chunks + // in processSearchResults. Used by the chat pipeline where context assembly + // is handled separately in the merge stage. + SkipContextEnrichment bool `json:"skip_context_enrichment,omitempty"` } // Value implements the driver.Valuer interface, used to convert SearchResult to database value