From bc2cb96d705a8647c8c1d5829a2fbea72a8e8862 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 11 Mar 2026 21:16:23 +0800 Subject: [PATCH] feat: enhance multimodal support and image handling - Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment. --- config/config.yaml | 66 ++------ frontend/src/i18n/locales/en-US.ts | 10 +- frontend/src/i18n/locales/zh-CN.ts | 10 +- frontend/src/views/agent/AgentEditorModal.vue | 15 +- .../knowledge/KnowledgeBaseEditorModal.vue | 151 ++++++++++++++++-- .../knowledge/settings/KBAdvancedSettings.vue | 71 -------- .../retriever/postgres/repository.go | 28 +++- .../service/chat_pipline/into_chat_message.go | 24 +++ .../service/chat_pipline/rerank.go | 73 ++++----- .../service/chat_pipline/rewrite.go | 72 ++++++++- .../service/chat_pipline/search.go | 41 ++--- internal/application/service/knowledgebase.go | 14 +- internal/handler/session/qa.go | 67 ++++++-- internal/logger/logger.go | 6 +- internal/models/chat/image_resolve.go | 2 + internal/models/rerank/aliyun_reranker.go | 5 +- internal/models/rerank/jina_reranker.go | 6 +- internal/models/rerank/logging.go | 42 +++++ internal/models/rerank/remote_api.go | 6 +- internal/models/rerank/zhipu_reranker.go | 5 +- 20 files changed, 467 insertions(+), 247 deletions(-) create mode 100644 internal/models/rerank/logging.go diff --git a/config/config.yaml b/config/config.yaml index 43516130..83db7c53 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -48,75 +48,27 @@ conversation: - Otherwise, output the rewritten question directly (no prefix = default to search) - IMPORTANT: This is a knowledge base Q&A system. Default to SEARCH. Only add [NO_SEARCH] when you are very confident the question has nothing to do with the knowledge base. - When to output [NO_SEARCH] (ONLY these narrow cases): + When to output [NO_SEARCH]: - Pure greetings, thanks, or farewell with no question ("谢谢", "你好", "再见") - Requests to summarize or manipulate the previous conversation itself ("总结一下我们的对话") - - When unsure, do NOT add the prefix (default to search) + - Pure image understanding with NO intent to search documents: describing, summarizing, translating, or extracting content from the image itself ("这张图片是什么", "描述一下图片内容", "帮我翻译图中文字", "图里的表格数据是什么", "帮我识别一下这张图") + - Follow-up questions that clearly refer to previous conversation content (especially previously uploaded images) and can be answered from dialogue context directly ("第一张图再详细描述一下", "第二张门上的字是什么意思", "这个再展开讲讲") When NOT to output [NO_SEARCH] (common mistakes to avoid): - - User uploads an image and asks to find/search/look up something → SEARCH (the image content is the search query) - - User asks about image content that may relate to documents ("这个报错怎么解决") → SEARCH - - User asks a question with an image, even if it seems like a simple image description → SEARCH (the knowledge base may contain relevant information) + - User uploads an image and asks to FIND/SEARCH related content in knowledge base → SEARCH (the image content is the search query) + - User asks how to fix/solve/handle something shown in the image ("这个报错怎么解决") → SEARCH (knowledge base may have solutions) + - User asks a question that implies comparing image content against documents → SEARCH ## Task 3: Image Analysis (only when images are attached) If the user's message includes images, after the rewritten question, add a separator "---" on a new line, then provide a concise description of the image content. This description is used as a text fallback for models that cannot process images directly. + If the image contains OCR text, also extract the key text information (OCR) from the image and include it in the description. ## Output Format - Text only (needs search): rewritten question - Text only (no search): [NO_SEARCH] rewritten question - - With images (needs search): rewritten question\n---\nimage description - - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description + - With images (needs search): rewritten question\n---\nimage description (including extracted OCR if present) + - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description (including extracted OCR if present) - ## Few-shot Examples - - Example 1 (text, needs search): - Conversation history: - User: What features does Slack have? - Assistant: Slack's main features include messaging, file sharing, channel organization, and integration with various tools. - User question: Is it secure? - Output: Is Slack secure? - - Example 2 (text, needs search): - Conversation history: - User: My laptop battery drains too fast, what should I do? - Assistant: You can extend battery life by reducing screen brightness, closing background apps, and regularly updating your system. - User question: Would that affect the user experience? - Output: Would reducing screen brightness and closing background apps affect the user experience? - - Example 3 (text, no search - greeting): - Conversation history: - User: How do I install WeKnora? - Assistant: You can install WeKnora using docker compose... - User question: Thanks! - Output: [NO_SEARCH] Thanks! - - Example 4 (text, no search - summarize conversation): - Conversation history: - User: What is RAG? - Assistant: RAG stands for Retrieval Augmented Generation... - User question: Summarize what we discussed - Output: [NO_SEARCH] Summarize our discussion about RAG - - Example 5 (image, needs search - user wants to find related info): - User question: 找一下这张图 [image attached] - Output: 找一下这张图片相关的内容 - --- - 图片中显示了一个系统架构图,包含前端、后端和数据库三层。 - - Example 6 (image, needs search - error screenshot): - Conversation history: - User: How do I configure the search settings? - Assistant: You can configure search settings in the admin panel... - User question: I got this error, how to fix it? [image attached] - Output: How to fix this search configuration error? - --- - The image shows an error dialog with the message "Invalid rerank model ID: model not found". - - Example 7 (image, needs search - image as query): - User question: What is this? [image attached] - Output: What is the content shown in this image? - --- - The image shows a product specification table with model numbers and parameters. rewrite_prompt_user: | ## Conversation History {{conversation}} diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index e4c38d4b..8ec019e6 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1409,6 +1409,7 @@ export default { advanced: 'Advanced Settings', faq: 'FAQ Settings', graph: 'Knowledge Graph', + multimodal: 'Multimodal', share: 'Sharing' }, basic: { @@ -1610,9 +1611,13 @@ export default { childChunkSizeLabel: 'Child Chunk Size', childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)' }, + multimodal: { + title: 'Multimodal Configuration', + description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images', + }, advanced: { title: 'Advanced Settings', - description: 'Configure question generation, multimodal features', + description: 'Configure question generation and other advanced features', questionGeneration: { label: 'AI Question Generation', description: 'Generate related questions for each chunk using LLM during document parsing to improve retrieval recall. Enabling this will increase document parsing time.', @@ -3248,6 +3253,9 @@ export default { selectPlaceholder: 'Select MCP services', }, imageUpload: { + navLabel: 'Multimodal', + sectionTitle: 'Multimodal Configuration', + sectionDesc: 'Configure image upload and vision-language model for multimodal conversations', label: 'Image Upload', desc: 'Allow users to upload images for multimodal Q&A in conversations', vlmModel: 'VLM Model', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index b1f39bf2..07b37d40 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1868,6 +1868,7 @@ export default { advanced: "高级设置", faq: "FAQ 设置", graph: "知识图谱", + multimodal: "多模态", share: "共享管理", }, basic: { @@ -2066,9 +2067,13 @@ export default { childChunkSizeLabel: "子块大小", childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)", }, + multimodal: { + title: "多模态配置", + description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索", + }, advanced: { title: "高级设置", - description: "配置问题生成、多模态等高级功能", + description: "配置问题生成等高级功能", questionGeneration: { label: "AI 问题生成", description: "解析文档时调用大模型为每个分块生成相关问题,提高检索召回率。启用后会增加文档解析耗时。", @@ -3246,6 +3251,9 @@ export default { selectPlaceholder: "选择 MCP 服务", }, imageUpload: { + navLabel: "多模态", + sectionTitle: "多模态配置", + sectionDesc: "配置图片上传和视觉语言模型,启用后用户可在对话中上传图片", label: "图片上传", desc: "启用后用户可在对话中上传图片进行多模态问答", vlmModel: "VLM 模型", diff --git a/frontend/src/views/agent/AgentEditorModal.vue b/frontend/src/views/agent/AgentEditorModal.vue index e5c4bbc0..0293cff8 100644 --- a/frontend/src/views/agent/AgentEditorModal.vue +++ b/frontend/src/views/agent/AgentEditorModal.vue @@ -319,6 +319,17 @@ + + + + +
+
+

{{ $t('agentEditor.imageUpload.sectionTitle') }}

+

{{ $t('agentEditor.imageUpload.sectionDesc') }}

+
+ +
@@ -1357,6 +1368,8 @@ const navItems = computed(() => { } // 网络搜索(独立菜单) items.push({ key: 'websearch', icon: 'internet', label: t('agent.editor.webSearchConfig') }); + // 多模态配置(图片上传) + items.push({ key: 'multimodal', icon: 'image', label: t('agentEditor.imageUpload.navLabel') }); // 多轮对话(仅普通模式显示,Agent模式内部自动控制) if (!isAgentMode.value) { items.push({ key: 'conversation', icon: 'chat', label: t('agent.editor.conversationSettings') }); @@ -2727,7 +2740,7 @@ const handleSave = async () => { // 校验 VLM 模型(当图片上传启用时必填) if (formData.value.config.image_upload_enabled && !formData.value.config.vlm_model_id) { MessagePlugin.error(t('agentEditor.imageUpload.vlmModelRequired')); - currentSection.value = 'model'; + currentSection.value = 'multimodal'; return; } diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 07c04b34..8228b9f9 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -155,15 +155,58 @@ />
+ +
+
+
+

{{ $t('knowledgeEditor.multimodal.title') }}

+

{{ $t('knowledgeEditor.multimodal.description') }}

+
+ +
+ +
+
+ +

{{ $t('knowledgeEditor.advanced.multimodal.description') }}

+
+
+ +
+
+ + +
+
+ +

{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}

+
+
+ +
+
+
+
+
+
@@ -203,6 +246,7 @@ import KBParserSettings from './settings/KBParserSettings.vue' import KBStorageSettings from './settings/KBStorageSettings.vue' import KBChunkingSettings from './settings/KBChunkingSettings.vue' import KBAdvancedSettings from './settings/KBAdvancedSettings.vue' +import ModelSelector from '@/components/ModelSelector.vue' import GraphSettings from './settings/GraphSettings.vue' import KBShareSettings from './settings/KBShareSettings.vue' import { useI18n } from 'vue-i18n' @@ -244,6 +288,7 @@ const navItems = computed(() => { { key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') }, { key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') }, { key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') }, + { key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') }, { key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') } ) } @@ -431,12 +476,22 @@ const handleParserEngineRulesUpdate = (rules: any[]) => { } } -const handleMultimodalUpdate = (config: any) => { - if (formData.value) { - formData.value.multimodalConfig = { ...config } +const handleMultimodalToggle = () => { + if (formData.value && !formData.value.multimodalConfig.enabled) { + formData.value.multimodalConfig.vllmModelId = '' } } +const handleMultimodalVLLMChange = (modelId: string) => { + if (formData.value) { + formData.value.multimodalConfig.vllmModelId = modelId + } +} + +const handleAddVLLMModel = () => { + uiStore.openSettings('models', 'vllm') +} + const handleStorageProviderUpdate = (value: string) => { if (formData.value) { formData.value.storageProvider = value || 'local' @@ -480,13 +535,10 @@ const validateForm = (): boolean => { } // 验证多模态配置(如果启用) - if (formData.value.multimodalConfig.enabled) { - const validation = (advancedSettingsRef.value as any)?.validateMultimodal?.() - if (validation && !validation.valid) { - MessagePlugin.warning(validation.message || t('knowledgeEditor.messages.multimodalInvalid')) - currentSection.value = 'advanced' - return false - } + if (formData.value.multimodalConfig.enabled && !formData.value.multimodalConfig.vllmModelId) { + MessagePlugin.warning(t('knowledgeEditor.messages.multimodalInvalid')) + currentSection.value = 'multimodal' + return false } if (formData.value.type === 'faq' && !formData.value.faqConfig?.indexMode) { @@ -1014,5 +1066,80 @@ watch( } } } + +// 多模态配置内联样式(与子组件 KBStorageSettings/KBAdvancedSettings 一致) +.kb-multimodal-settings { + width: 100%; + + .section-header { + margin-bottom: 32px; + + h2 { + font-size: 20px; + font-weight: 600; + color: var(--td-text-color-primary); + margin: 0 0 8px 0; + } + + .section-description { + font-size: 14px; + color: var(--td-text-color-secondary); + margin: 0; + line-height: 1.5; + } + } + + .settings-group { + display: flex; + flex-direction: column; + } + + .setting-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + padding: 20px 0; + border-bottom: 1px solid var(--td-component-stroke); + + &:last-child { + border-bottom: none; + } + } + + .setting-info { + flex: 1; + max-width: 65%; + padding-right: 24px; + + label { + font-size: 15px; + font-weight: 500; + color: var(--td-text-color-primary); + display: block; + margin-bottom: 4px; + } + + .desc { + font-size: 13px; + color: var(--td-text-color-secondary); + margin: 0; + line-height: 1.5; + } + } + + .setting-control { + flex-shrink: 0; + min-width: 280px; + display: flex; + justify-content: flex-end; + align-items: center; + } + + .required { + color: var(--td-error-color); + margin-left: 2px; + font-weight: 500; + } +} diff --git a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue index 7be04239..9512c64b 100644 --- a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue +++ b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue @@ -42,57 +42,12 @@
- -
-
- -

{{ $t('knowledgeEditor.advanced.multimodal.description') }}

-
-
- -
-
- - -
-
-
- -

{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}

-
-
- -
-
-
-
- diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index 0f8cda4a..cf357e08 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -233,10 +233,19 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context, logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) + const maxKeywordResultLog = 8 for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords) - logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", - i, results[i].ChunkID, results[i].Score) + if i < maxKeywordResultLog { + logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", + i, results[i].ChunkID, results[i].Score) + } + } + if len(results) > maxKeywordResultLog { + logger.GetLogger(ctx).Debugf( + "[Postgres] Keywords result summary: total=%d logged=%d truncated=%d", + len(results), maxKeywordResultLog, len(results)-maxKeywordResultLog, + ) } return []*types.RetrieveResult{ { @@ -387,10 +396,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context, logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) + const maxVectorResultLog = 8 for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding) - logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", - i, results[i].ChunkID, results[i].Score) + if i < maxVectorResultLog { + logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", + i, results[i].ChunkID, results[i].Score) + } + } + if len(results) > maxVectorResultLog { + logger.GetLogger(ctx).Debugf( + "[Postgres] Vector search result summary: total=%d logged=%d truncated=%d", + len(results), maxVectorResultLog, len(results)-maxVectorResultLog, + ) } return []*types.RetrieveResult{ { @@ -494,7 +512,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context, // Create new vector index, copy the content and vector of the source index targetVector := &pgVector{ Content: sourceVector.Content, - SourceID: targetSourceID, // Handle SourceID transformation properly + SourceID: targetSourceID, // Handle SourceID transformation properly SourceType: sourceVector.SourceType, ChunkID: targetChunkID, // Update to target chunk ID KnowledgeID: targetKnowledgeID, // Update to target knowledge ID diff --git a/internal/application/service/chat_pipline/into_chat_message.go b/internal/application/service/chat_pipline/into_chat_message.go index d9d422cd..a81275f6 100644 --- a/internal/application/service/chat_pipline/into_chat_message.go +++ b/internal/application/service/chat_pipline/into_chat_message.go @@ -77,6 +77,30 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context, // Prepare weekday names weekdayName := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} + // Intent-based no-search path: bypass "reference materials" template entirely. + if chatManage.SkipKBSearch { + // Prefer rewritten query in no-search mode; fallback to original query. + userContent := safeQuery + if rewrite := strings.TrimSpace(chatManage.RewriteQuery); rewrite != "" { + if safeRewrite, ok := utils.ValidateInput(rewrite); ok { + userContent = safeRewrite + } else { + pipelineWarn(ctx, "IntoChatMessage", "invalid_rewrite_query_fallback", map[string]interface{}{ + "session_id": chatManage.SessionID, + }) + } + } + if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision { + userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText + } + chatManage.UserContent = userContent + pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{ + "session_id": chatManage.SessionID, + "user_content_len": len(chatManage.UserContent), + }) + return next() + } + var contextsBuilder strings.Builder // Build contexts string based on FAQ priority strategy diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 3b6cd0c1..94560316 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -121,15 +121,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context, "result_cnt": len(rerankResp), }) - // Log input scores before reranking for debugging - for i, sr := range chatManage.SearchResult { - pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{ - "index": i, - "chunk_id": sr.ID, - "score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) - } + logRerankInputScoreSample(ctx, chatManage.SearchResult) for i := range chatManage.SearchResult { chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata) @@ -162,13 +154,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context, }) } - pipelineInfo(ctx, "Rerank", "composite_calc", map[string]interface{}{ - "chunk_id": sr.ID, - "base_score": fmt.Sprintf("%.4f", base), - "model_score": fmt.Sprintf("%.4f", modelScore), - "final_score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) reranked = append(reranked, sr) } @@ -179,13 +164,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context, // Assign high model score for direct load items modelScore := 1.0 sr.Score = compositeScore(sr, modelScore, base) - pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{ - "chunk_id": sr.ID, - "base_score": fmt.Sprintf("%.4f", base), - "model_score": fmt.Sprintf("%.4f", modelScore), - "final_score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) reranked = append(reranked, sr) } final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7) @@ -237,18 +215,26 @@ func (p *PluginRerank) rerank(ctx context.Context, pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{ "threshold": chatManage.RerankThreshold, }) - for i := range min(5, len(rerankResp)) { + logged := min(5, len(rerankResp)) + for i := range logged { if rerankResp[i].Index < len(candidates) { pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{ - "rank": i + 1, - "score": rerankResp[i].RelevanceScore, - "chunk_id": candidates[rerankResp[i].Index].ID, - "match_type": candidates[rerankResp[i].Index].MatchType, - "chunk_type": candidates[rerankResp[i].Index].ChunkType, - "content": candidates[rerankResp[i].Index].Content, + "rank": i + 1, + "score": rerankResp[i].RelevanceScore, + "chunk_id": candidates[rerankResp[i].Index].ID, + "match_type": candidates[rerankResp[i].Index].MatchType, + "chunk_type": candidates[rerankResp[i].Index].ChunkType, + "content_len": len(candidates[rerankResp[i].Index].Content), }) } } + if len(rerankResp) > logged { + pipelineInfo(ctx, "Rerank", "top_score_summary", map[string]interface{}{ + "total": len(rerankResp), + "logged": logged, + "truncated": len(rerankResp) - logged, + }) + } // Filter results based on threshold rankFilter := []rerank.RankResult{} @@ -529,11 +515,26 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string } combinedText += strings.Join(enrichments, "\n") - pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{ - "content_len": len(result.Content), - "cleaned_len": len(cleanPassageForRerank(result.Content)), - "enrichment_len": len(strings.Join(enrichments, "\n")), - }) - return combinedText } + +func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) { + const maxLogRows = 8 + limit := min(maxLogRows, len(results)) + for i := 0; i < limit; i++ { + sr := results[i] + pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{ + "index": i, + "chunk_id": sr.ID, + "score": fmt.Sprintf("%.4f", sr.Score), + "match_type": sr.MatchType, + }) + } + if len(results) > limit { + pipelineInfo(ctx, "Rerank", "input_score_summary", map[string]interface{}{ + "total": len(results), + "logged": limit, + "truncated": len(results) - limit, + }) + } +} diff --git a/internal/application/service/chat_pipline/rewrite.go b/internal/application/service/chat_pipline/rewrite.go index 8b31c96d..1a0ba737 100644 --- a/internal/application/service/chat_pipline/rewrite.go +++ b/internal/application/service/chat_pipline/rewrite.go @@ -11,9 +11,11 @@ import ( "time" "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" ) // PluginRewrite is a plugin for rewriting user queries @@ -26,10 +28,10 @@ type PluginRewrite struct { // reg is a regular expression used to match and remove content between tags var reg = regexp.MustCompile(`(?s).*?`) +var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`) const ( - noSearchPrefix = "[NO_SEARCH]" - imageDescSeparator = "\n---\n" + noSearchPrefix = "[NO_SEARCH]" ) // NewPluginRewrite creates a new query rewriting plugin instance @@ -118,8 +120,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, maxTokens = 500 } + // --- Emit progress event for image analysis --- + var toolCallID string + if useImages && chatManage.EventBus != nil { + toolCallID = uuid.New().String() + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolCall), + SessionID: chatManage.SessionID, + Data: event.AgentToolCallData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + }, + }) + } + // --- Call model --- thinking := false + vlmStart := time.Now() response, err := rewriteModel.Chat(ctx, []chat.Message{ {Role: "system", Content: systemContent}, userMsg, @@ -129,6 +146,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, Thinking: &thinking, }) if err != nil { + if toolCallID != "" && chatManage.EventBus != nil { + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolResult), + SessionID: chatManage.SessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "图片分析失败", + Success: false, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{ "session_id": chatManage.SessionID, "error": err.Error(), @@ -136,9 +166,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, return next() } + // --- Emit completion event for image analysis --- + if toolCallID != "" && chatManage.EventBus != nil { + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolResult), + SessionID: chatManage.SessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "已分析图片内容", + Success: true, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } + // --- Parse structured output --- p.parseRewriteOutput(chatManage, response.Content) - pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{ "session_id": chatManage.SessionID, "rewrite_query": chatManage.RewriteQuery, @@ -261,6 +305,13 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [ if chatManage.RewritePromptSystem != "" { systemPrompt = chatManage.RewritePromptSystem } + // Strengthen context inheritance in multi-turn conversation: + // for follow-up questions that clearly refer to previous turns (especially + // uploaded-image understanding), prefer NO_SEARCH over KB retrieval. + systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" + + "- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" + + "- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" + + "- In these follow-up cases, output with [NO_SEARCH] prefix." conversationText := formatConversationHistory(historyList) currentTime := time.Now().Format("2006-01-02 15:04:05") @@ -296,11 +347,16 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix)) } - // 2. Split rewritten query and image description - if idx := strings.Index(content, imageDescSeparator); idx >= 0 { - chatManage.RewriteQuery = strings.TrimSpace(content[:idx]) - chatManage.ImageOCRText = strings.TrimSpace(content[idx+len(imageDescSeparator):]) - } else if content != "" { + // 2. Split rewritten query and image description. + // Be tolerant to model output variants like: + // - "query\n---\nimage_desc" (expected) + // - "query---\nimage_desc" (missing leading newline before separator) + if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 { + chatManage.RewriteQuery = strings.TrimSpace(m[1]) + chatManage.ImageOCRText = strings.TrimSpace(m[2]) + return + } + if content != "" { chatManage.RewriteQuery = content } } diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 8d9ed23d..4422f2cb 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -113,15 +113,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context, chatManage.SearchResult = allResults - // Log all search results with scores before any processing - for i, r := range chatManage.SearchResult { - pipelineInfo(ctx, "Search", "result_score_before_normalize", map[string]interface{}{ - "index": i, - "chunk_id": r.ID, - "score": fmt.Sprintf("%.4f", r.Score), - "match_type": r.MatchType, - }) - } + logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult) // If recall is low, attempt query expansion with keyword-focused search if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) { @@ -131,15 +123,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context, } } - // Log final scores after all processing - for i, r := range chatManage.SearchResult { - pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{ - "index": i, - "chunk_id": r.ID, - "score": fmt.Sprintf("%.4f", r.Score), - "match_type": r.MatchType, - }) - } + logSearchScoreSample(ctx, "final_score", chatManage.SearchResult) // Return if we have results if len(chatManage.SearchResult) != 0 { @@ -204,6 +188,27 @@ func buildContentSignature(content string) string { return searchutil.BuildContentSignature(content) } +func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) { + const maxLogRows = 8 + limit := min(maxLogRows, len(results)) + for i := 0; i < limit; i++ { + r := results[i] + pipelineInfo(ctx, "Search", action, map[string]interface{}{ + "index": i, + "chunk_id": r.ID, + "score": fmt.Sprintf("%.4f", r.Score), + "match_type": r.MatchType, + }) + } + if len(results) > limit { + pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{ + "total": len(results), + "logged": limit, + "truncated": len(results) - limit, + }) + } +} + // searchByTargets performs KB searches using pre-computed SearchTargets // This is the main search method that uses the unified search targets func (p *PluginSearch) searchByTargets( diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index c5c9314b..c83c4258 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -1231,6 +1231,9 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, // Build final search results - preserve original order from input chunks var searchResults []*types.SearchResult addedChunkIDs := make(map[string]bool) + const maxInvalidChunkLog = 8 + invalidChunkCnt := 0 + invalidChunkSamples := make([]string, 0, maxInvalidChunkLog) // First pass: Add results in the original order from input chunks for _, inputChunk := range chunks { @@ -1240,7 +1243,10 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, continue } if !s.isValidTextChunk(chunk) { - logger.Debugf(ctx, "Chunk is not valid text chunk: %s, type: %s", chunk.ID, chunk.ChunkType) + invalidChunkCnt++ + if len(invalidChunkSamples) < maxInvalidChunkLog { + invalidChunkSamples = append(invalidChunkSamples, chunk.ID+":"+chunk.ChunkType) + } continue } if addedChunkIDs[chunk.ID] { @@ -1257,6 +1263,12 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, logger.Warnf(ctx, "Knowledge not found for chunk: %s, knowledge_id: %s", chunk.ID, chunk.KnowledgeID) } } + if invalidChunkCnt > 0 { + logger.Debugf(ctx, + "Skip non-text chunks in search results: total=%d sampled=%d samples=%v", + invalidChunkCnt, len(invalidChunkSamples), invalidChunkSamples, + ) + } // Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input if !skipEnrichment { diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index 308dd17b..ad566399 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -165,21 +165,10 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err)) } - // Decide whether to run VLM analysis here or defer. - // Agent mode defers to the async execution flow (so SSE stream is up and progress is visible). - // Normal mode defers when the RAG pipeline will run (rewrite step handles it). - // Only run here for Normal mode pure-chat paths (no KB, no web search). - isAgentEntry := logPrefix == "AgentQA" - hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0 - agentWillResolveKBs := !hasRequestKBs && - !customAgent.Config.RetrieveKBOnlyWhenMentioned && - customAgent.Config.KBSelectionMode != "" && - customAgent.Config.KBSelectionMode != "none" - willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled - if !willDeferVLM { - agentVLMModelID := customAgent.Config.VLMModelID - h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query) - } + // VLM analysis is always deferred to after SSE stream is up: + // - Agent mode: runs in async execution flow with tool_call/tool_result events + // - Normal RAG mode: runs in the pipeline rewrite step with progress events + // - Normal pure-chat mode: runs in the async goroutine with progress events } // Build request context @@ -478,6 +467,54 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo } }() + // For pure-chat path (no KB, no web search), VLM analysis isn't handled by the + // rewrite pipeline step, so run it here with progress events. + // RAG paths (with KB/web search, including agent-resolved KBs) defer VLM analysis + // to the rewrite step to avoid duplicate progress events. + hasRequestKBs := len(reqCtx.knowledgeBaseIDs) > 0 || len(reqCtx.knowledgeIDs) > 0 + agentWillResolveKBs := false + if !hasRequestKBs && reqCtx.customAgent != nil && !reqCtx.customAgent.Config.RetrieveKBOnlyWhenMentioned { + mode := reqCtx.customAgent.Config.KBSelectionMode + switch mode { + case "all": + agentWillResolveKBs = true + case "selected", "": + // "" keeps backward-compatible behavior (same as selected). + agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0 + case "none": + agentWillResolveKBs = false + default: + agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0 + } + } + isPureChatPath := !hasRequestKBs && !agentWillResolveKBs && !reqCtx.webSearchEnabled + if isPureChatPath && len(reqCtx.images) > 0 && + reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" { + toolCallID := uuid.New().String() + streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{ + Type: event.EventAgentToolCall, + SessionID: sessionID, + Data: event.AgentToolCallData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + }, + }) + vlmStart := time.Now() + h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images, + reqCtx.customAgent.Config.VLMModelID, reqCtx.query) + streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{ + Type: event.EventAgentToolResult, + SessionID: sessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "已分析图片内容", + Success: true, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } + imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images) err := h.sessionService.KnowledgeQA( streamCtx.asyncCtx, diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 7fbfc13c..ed78113b 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -82,10 +82,10 @@ func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) { // request_id 优先输出 if v, ok := entry.Data["request_id"]; ok { if f.ForceColor { - fields += fmt.Sprintf("%srequest_id%s=%s%v%s ", - colorCyan, colorReset, colorBlue, v, colorReset) + fields += fmt.Sprintf("%s%v%s ", + colorBlue, v, colorReset) } else { - fields += fmt.Sprintf("request_id=%v ", v) + fields += fmt.Sprintf("%v ", v) } } diff --git a/internal/models/chat/image_resolve.go b/internal/models/chat/image_resolve.go index d7f41577..c4503b81 100644 --- a/internal/models/chat/image_resolve.go +++ b/internal/models/chat/image_resolve.go @@ -3,6 +3,7 @@ package chat import ( "encoding/base64" "fmt" + "log" "net/http" "os" "path/filepath" @@ -55,6 +56,7 @@ func readLocalStorageBytes(storagePath string) []byte { localPath := filepath.Join(baseDir, filepath.FromSlash(relPath)) data, err := os.ReadFile(localPath) if err != nil { + log.Printf("[image-resolve] failed to read local file %s: %v", localPath, err) return nil } return data diff --git a/internal/models/rerank/aliyun_reranker.go b/internal/models/rerank/aliyun_reranker.go index 3643b7f7..75103c87 100644 --- a/internal/models/rerank/aliyun_reranker.go +++ b/internal/models/rerank/aliyun_reranker.go @@ -112,10 +112,7 @@ func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []s req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging - logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", - r.baseURL, r.apiKey, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/jina_reranker.go b/internal/models/rerank/jina_reranker.go index 4b7b1947..5067123e 100644 --- a/internal/models/rerank/jina_reranker.go +++ b/internal/models/rerank/jina_reranker.go @@ -80,11 +80,7 @@ func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []str req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging (API key masked for security) - logger.GetLogger(ctx).Infof( - "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", - r.baseURL, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/logging.go b/internal/models/rerank/logging.go new file mode 100644 index 00000000..de38c111 --- /dev/null +++ b/internal/models/rerank/logging.go @@ -0,0 +1,42 @@ +package rerank + +import ( + "encoding/json" + "fmt" + "strings" + "unicode/utf8" +) + +const ( + maxLogDocuments = 3 + maxLogTextRunes = 120 +) + +func buildRerankRequestDebug(model, endpoint, query string, documents []string) string { + previews := make([]string, 0, maxLogDocuments) + for i, doc := range documents { + if i >= maxLogDocuments { + break + } + previews = append(previews, compactForLog(doc, maxLogTextRunes)) + } + + previewJSON, _ := json.Marshal(previews) + return fmt.Sprintf( + "rerank request endpoint=%s model=%s query_preview=%q query_runes=%d documents=%d preview_docs=%s", + endpoint, + model, + compactForLog(query, maxLogTextRunes), + utf8.RuneCountInString(query), + len(documents), + string(previewJSON), + ) +} + +func compactForLog(text string, maxRunes int) string { + normalized := strings.Join(strings.Fields(strings.TrimSpace(text)), " ") + if utf8.RuneCountInString(normalized) <= maxRunes { + return normalized + } + return string([]rune(normalized)[:maxRunes]) + "...(truncated)" +} diff --git a/internal/models/rerank/remote_api.go b/internal/models/rerank/remote_api.go index 945b9fc4..d9b77771 100644 --- a/internal/models/rerank/remote_api.go +++ b/internal/models/rerank/remote_api.go @@ -82,11 +82,7 @@ func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []s req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging (API key masked for security) - logger.GetLogger(ctx).Infof( - "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", - r.baseURL, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/zhipu_reranker.go b/internal/models/rerank/zhipu_reranker.go index c9bbd7b4..ad0d801c 100644 --- a/internal/models/rerank/zhipu_reranker.go +++ b/internal/models/rerank/zhipu_reranker.go @@ -93,10 +93,7 @@ func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []st req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging - logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", - r.baseURL, r.apiKey, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) resp, err := r.client.Do(req) if err != nil {