diff --git a/config/config.yaml b/config/config.yaml index 43516130..83db7c53 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -48,75 +48,27 @@ conversation: - Otherwise, output the rewritten question directly (no prefix = default to search) - IMPORTANT: This is a knowledge base Q&A system. Default to SEARCH. Only add [NO_SEARCH] when you are very confident the question has nothing to do with the knowledge base. - When to output [NO_SEARCH] (ONLY these narrow cases): + When to output [NO_SEARCH]: - Pure greetings, thanks, or farewell with no question ("谢谢", "你好", "再见") - Requests to summarize or manipulate the previous conversation itself ("总结一下我们的对话") - - When unsure, do NOT add the prefix (default to search) + - Pure image understanding with NO intent to search documents: describing, summarizing, translating, or extracting content from the image itself ("这张图片是什么", "描述一下图片内容", "帮我翻译图中文字", "图里的表格数据是什么", "帮我识别一下这张图") + - Follow-up questions that clearly refer to previous conversation content (especially previously uploaded images) and can be answered from dialogue context directly ("第一张图再详细描述一下", "第二张门上的字是什么意思", "这个再展开讲讲") When NOT to output [NO_SEARCH] (common mistakes to avoid): - - User uploads an image and asks to find/search/look up something → SEARCH (the image content is the search query) - - User asks about image content that may relate to documents ("这个报错怎么解决") → SEARCH - - User asks a question with an image, even if it seems like a simple image description → SEARCH (the knowledge base may contain relevant information) + - User uploads an image and asks to FIND/SEARCH related content in knowledge base → SEARCH (the image content is the search query) + - User asks how to fix/solve/handle something shown in the image ("这个报错怎么解决") → SEARCH (knowledge base may have solutions) + - User asks a question that implies comparing image content against documents → SEARCH ## Task 3: Image Analysis (only when images are attached) If the user's message includes images, after the rewritten question, add a separator "---" on a new line, then provide a concise description of the image content. This description is used as a text fallback for models that cannot process images directly. + If the image contains OCR text, also extract the key text information (OCR) from the image and include it in the description. ## Output Format - Text only (needs search): rewritten question - Text only (no search): [NO_SEARCH] rewritten question - - With images (needs search): rewritten question\n---\nimage description - - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description + - With images (needs search): rewritten question\n---\nimage description (including extracted OCR if present) + - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description (including extracted OCR if present) - ## Few-shot Examples - - Example 1 (text, needs search): - Conversation history: - User: What features does Slack have? - Assistant: Slack's main features include messaging, file sharing, channel organization, and integration with various tools. - User question: Is it secure? - Output: Is Slack secure? - - Example 2 (text, needs search): - Conversation history: - User: My laptop battery drains too fast, what should I do? - Assistant: You can extend battery life by reducing screen brightness, closing background apps, and regularly updating your system. - User question: Would that affect the user experience? - Output: Would reducing screen brightness and closing background apps affect the user experience? - - Example 3 (text, no search - greeting): - Conversation history: - User: How do I install WeKnora? - Assistant: You can install WeKnora using docker compose... - User question: Thanks! - Output: [NO_SEARCH] Thanks! - - Example 4 (text, no search - summarize conversation): - Conversation history: - User: What is RAG? - Assistant: RAG stands for Retrieval Augmented Generation... - User question: Summarize what we discussed - Output: [NO_SEARCH] Summarize our discussion about RAG - - Example 5 (image, needs search - user wants to find related info): - User question: 找一下这张图 [image attached] - Output: 找一下这张图片相关的内容 - --- - 图片中显示了一个系统架构图,包含前端、后端和数据库三层。 - - Example 6 (image, needs search - error screenshot): - Conversation history: - User: How do I configure the search settings? - Assistant: You can configure search settings in the admin panel... - User question: I got this error, how to fix it? [image attached] - Output: How to fix this search configuration error? - --- - The image shows an error dialog with the message "Invalid rerank model ID: model not found". - - Example 7 (image, needs search - image as query): - User question: What is this? [image attached] - Output: What is the content shown in this image? - --- - The image shows a product specification table with model numbers and parameters. rewrite_prompt_user: | ## Conversation History {{conversation}} diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index e4c38d4b..8ec019e6 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1409,6 +1409,7 @@ export default { advanced: 'Advanced Settings', faq: 'FAQ Settings', graph: 'Knowledge Graph', + multimodal: 'Multimodal', share: 'Sharing' }, basic: { @@ -1610,9 +1611,13 @@ export default { childChunkSizeLabel: 'Child Chunk Size', childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)' }, + multimodal: { + title: 'Multimodal Configuration', + description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images', + }, advanced: { title: 'Advanced Settings', - description: 'Configure question generation, multimodal features', + description: 'Configure question generation and other advanced features', questionGeneration: { label: 'AI Question Generation', description: 'Generate related questions for each chunk using LLM during document parsing to improve retrieval recall. Enabling this will increase document parsing time.', @@ -3248,6 +3253,9 @@ export default { selectPlaceholder: 'Select MCP services', }, imageUpload: { + navLabel: 'Multimodal', + sectionTitle: 'Multimodal Configuration', + sectionDesc: 'Configure image upload and vision-language model for multimodal conversations', label: 'Image Upload', desc: 'Allow users to upload images for multimodal Q&A in conversations', vlmModel: 'VLM Model', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index b1f39bf2..07b37d40 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1868,6 +1868,7 @@ export default { advanced: "高级设置", faq: "FAQ 设置", graph: "知识图谱", + multimodal: "多模态", share: "共享管理", }, basic: { @@ -2066,9 +2067,13 @@ export default { childChunkSizeLabel: "子块大小", childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)", }, + multimodal: { + title: "多模态配置", + description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索", + }, advanced: { title: "高级设置", - description: "配置问题生成、多模态等高级功能", + description: "配置问题生成等高级功能", questionGeneration: { label: "AI 问题生成", description: "解析文档时调用大模型为每个分块生成相关问题,提高检索召回率。启用后会增加文档解析耗时。", @@ -3246,6 +3251,9 @@ export default { selectPlaceholder: "选择 MCP 服务", }, imageUpload: { + navLabel: "多模态", + sectionTitle: "多模态配置", + sectionDesc: "配置图片上传和视觉语言模型,启用后用户可在对话中上传图片", label: "图片上传", desc: "启用后用户可在对话中上传图片进行多模态问答", vlmModel: "VLM 模型", diff --git a/frontend/src/views/agent/AgentEditorModal.vue b/frontend/src/views/agent/AgentEditorModal.vue index e5c4bbc0..0293cff8 100644 --- a/frontend/src/views/agent/AgentEditorModal.vue +++ b/frontend/src/views/agent/AgentEditorModal.vue @@ -319,6 +319,17 @@ + + + + +
+
+

{{ $t('agentEditor.imageUpload.sectionTitle') }}

+

{{ $t('agentEditor.imageUpload.sectionDesc') }}

+
+ +
@@ -1357,6 +1368,8 @@ const navItems = computed(() => { } // 网络搜索(独立菜单) items.push({ key: 'websearch', icon: 'internet', label: t('agent.editor.webSearchConfig') }); + // 多模态配置(图片上传) + items.push({ key: 'multimodal', icon: 'image', label: t('agentEditor.imageUpload.navLabel') }); // 多轮对话(仅普通模式显示,Agent模式内部自动控制) if (!isAgentMode.value) { items.push({ key: 'conversation', icon: 'chat', label: t('agent.editor.conversationSettings') }); @@ -2727,7 +2740,7 @@ const handleSave = async () => { // 校验 VLM 模型(当图片上传启用时必填) if (formData.value.config.image_upload_enabled && !formData.value.config.vlm_model_id) { MessagePlugin.error(t('agentEditor.imageUpload.vlmModelRequired')); - currentSection.value = 'model'; + currentSection.value = 'multimodal'; return; } diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 07c04b34..8228b9f9 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -155,15 +155,58 @@ />
+ +
+
+
+

{{ $t('knowledgeEditor.multimodal.title') }}

+

{{ $t('knowledgeEditor.multimodal.description') }}

+
+ +
+ +
+
+ +

{{ $t('knowledgeEditor.advanced.multimodal.description') }}

+
+
+ +
+
+ + +
+
+ +

{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}

+
+
+ +
+
+
+
+
+
@@ -203,6 +246,7 @@ import KBParserSettings from './settings/KBParserSettings.vue' import KBStorageSettings from './settings/KBStorageSettings.vue' import KBChunkingSettings from './settings/KBChunkingSettings.vue' import KBAdvancedSettings from './settings/KBAdvancedSettings.vue' +import ModelSelector from '@/components/ModelSelector.vue' import GraphSettings from './settings/GraphSettings.vue' import KBShareSettings from './settings/KBShareSettings.vue' import { useI18n } from 'vue-i18n' @@ -244,6 +288,7 @@ const navItems = computed(() => { { key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') }, { key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') }, { key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') }, + { key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') }, { key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') } ) } @@ -431,12 +476,22 @@ const handleParserEngineRulesUpdate = (rules: any[]) => { } } -const handleMultimodalUpdate = (config: any) => { - if (formData.value) { - formData.value.multimodalConfig = { ...config } +const handleMultimodalToggle = () => { + if (formData.value && !formData.value.multimodalConfig.enabled) { + formData.value.multimodalConfig.vllmModelId = '' } } +const handleMultimodalVLLMChange = (modelId: string) => { + if (formData.value) { + formData.value.multimodalConfig.vllmModelId = modelId + } +} + +const handleAddVLLMModel = () => { + uiStore.openSettings('models', 'vllm') +} + const handleStorageProviderUpdate = (value: string) => { if (formData.value) { formData.value.storageProvider = value || 'local' @@ -480,13 +535,10 @@ const validateForm = (): boolean => { } // 验证多模态配置(如果启用) - if (formData.value.multimodalConfig.enabled) { - const validation = (advancedSettingsRef.value as any)?.validateMultimodal?.() - if (validation && !validation.valid) { - MessagePlugin.warning(validation.message || t('knowledgeEditor.messages.multimodalInvalid')) - currentSection.value = 'advanced' - return false - } + if (formData.value.multimodalConfig.enabled && !formData.value.multimodalConfig.vllmModelId) { + MessagePlugin.warning(t('knowledgeEditor.messages.multimodalInvalid')) + currentSection.value = 'multimodal' + return false } if (formData.value.type === 'faq' && !formData.value.faqConfig?.indexMode) { @@ -1014,5 +1066,80 @@ watch( } } } + +// 多模态配置内联样式(与子组件 KBStorageSettings/KBAdvancedSettings 一致) +.kb-multimodal-settings { + width: 100%; + + .section-header { + margin-bottom: 32px; + + h2 { + font-size: 20px; + font-weight: 600; + color: var(--td-text-color-primary); + margin: 0 0 8px 0; + } + + .section-description { + font-size: 14px; + color: var(--td-text-color-secondary); + margin: 0; + line-height: 1.5; + } + } + + .settings-group { + display: flex; + flex-direction: column; + } + + .setting-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + padding: 20px 0; + border-bottom: 1px solid var(--td-component-stroke); + + &:last-child { + border-bottom: none; + } + } + + .setting-info { + flex: 1; + max-width: 65%; + padding-right: 24px; + + label { + font-size: 15px; + font-weight: 500; + color: var(--td-text-color-primary); + display: block; + margin-bottom: 4px; + } + + .desc { + font-size: 13px; + color: var(--td-text-color-secondary); + margin: 0; + line-height: 1.5; + } + } + + .setting-control { + flex-shrink: 0; + min-width: 280px; + display: flex; + justify-content: flex-end; + align-items: center; + } + + .required { + color: var(--td-error-color); + margin-left: 2px; + font-weight: 500; + } +} diff --git a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue index 7be04239..9512c64b 100644 --- a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue +++ b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue @@ -42,57 +42,12 @@
- -
-
- -

{{ $t('knowledgeEditor.advanced.multimodal.description') }}

-
-
- -
-
- - -
-
-
- -

{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}

-
-
- -
-
-
-
- diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index 0f8cda4a..cf357e08 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -233,10 +233,19 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context, logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) + const maxKeywordResultLog = 8 for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords) - logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", - i, results[i].ChunkID, results[i].Score) + if i < maxKeywordResultLog { + logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", + i, results[i].ChunkID, results[i].Score) + } + } + if len(results) > maxKeywordResultLog { + logger.GetLogger(ctx).Debugf( + "[Postgres] Keywords result summary: total=%d logged=%d truncated=%d", + len(results), maxKeywordResultLog, len(results)-maxKeywordResultLog, + ) } return []*types.RetrieveResult{ { @@ -387,10 +396,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context, logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) + const maxVectorResultLog = 8 for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding) - logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", - i, results[i].ChunkID, results[i].Score) + if i < maxVectorResultLog { + logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", + i, results[i].ChunkID, results[i].Score) + } + } + if len(results) > maxVectorResultLog { + logger.GetLogger(ctx).Debugf( + "[Postgres] Vector search result summary: total=%d logged=%d truncated=%d", + len(results), maxVectorResultLog, len(results)-maxVectorResultLog, + ) } return []*types.RetrieveResult{ { @@ -494,7 +512,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context, // Create new vector index, copy the content and vector of the source index targetVector := &pgVector{ Content: sourceVector.Content, - SourceID: targetSourceID, // Handle SourceID transformation properly + SourceID: targetSourceID, // Handle SourceID transformation properly SourceType: sourceVector.SourceType, ChunkID: targetChunkID, // Update to target chunk ID KnowledgeID: targetKnowledgeID, // Update to target knowledge ID diff --git a/internal/application/service/chat_pipline/into_chat_message.go b/internal/application/service/chat_pipline/into_chat_message.go index d9d422cd..a81275f6 100644 --- a/internal/application/service/chat_pipline/into_chat_message.go +++ b/internal/application/service/chat_pipline/into_chat_message.go @@ -77,6 +77,30 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context, // Prepare weekday names weekdayName := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} + // Intent-based no-search path: bypass "reference materials" template entirely. + if chatManage.SkipKBSearch { + // Prefer rewritten query in no-search mode; fallback to original query. + userContent := safeQuery + if rewrite := strings.TrimSpace(chatManage.RewriteQuery); rewrite != "" { + if safeRewrite, ok := utils.ValidateInput(rewrite); ok { + userContent = safeRewrite + } else { + pipelineWarn(ctx, "IntoChatMessage", "invalid_rewrite_query_fallback", map[string]interface{}{ + "session_id": chatManage.SessionID, + }) + } + } + if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision { + userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText + } + chatManage.UserContent = userContent + pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{ + "session_id": chatManage.SessionID, + "user_content_len": len(chatManage.UserContent), + }) + return next() + } + var contextsBuilder strings.Builder // Build contexts string based on FAQ priority strategy diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 3b6cd0c1..94560316 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -121,15 +121,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context, "result_cnt": len(rerankResp), }) - // Log input scores before reranking for debugging - for i, sr := range chatManage.SearchResult { - pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{ - "index": i, - "chunk_id": sr.ID, - "score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) - } + logRerankInputScoreSample(ctx, chatManage.SearchResult) for i := range chatManage.SearchResult { chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata) @@ -162,13 +154,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context, }) } - pipelineInfo(ctx, "Rerank", "composite_calc", map[string]interface{}{ - "chunk_id": sr.ID, - "base_score": fmt.Sprintf("%.4f", base), - "model_score": fmt.Sprintf("%.4f", modelScore), - "final_score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) reranked = append(reranked, sr) } @@ -179,13 +164,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context, // Assign high model score for direct load items modelScore := 1.0 sr.Score = compositeScore(sr, modelScore, base) - pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{ - "chunk_id": sr.ID, - "base_score": fmt.Sprintf("%.4f", base), - "model_score": fmt.Sprintf("%.4f", modelScore), - "final_score": fmt.Sprintf("%.4f", sr.Score), - "match_type": sr.MatchType, - }) reranked = append(reranked, sr) } final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7) @@ -237,18 +215,26 @@ func (p *PluginRerank) rerank(ctx context.Context, pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{ "threshold": chatManage.RerankThreshold, }) - for i := range min(5, len(rerankResp)) { + logged := min(5, len(rerankResp)) + for i := range logged { if rerankResp[i].Index < len(candidates) { pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{ - "rank": i + 1, - "score": rerankResp[i].RelevanceScore, - "chunk_id": candidates[rerankResp[i].Index].ID, - "match_type": candidates[rerankResp[i].Index].MatchType, - "chunk_type": candidates[rerankResp[i].Index].ChunkType, - "content": candidates[rerankResp[i].Index].Content, + "rank": i + 1, + "score": rerankResp[i].RelevanceScore, + "chunk_id": candidates[rerankResp[i].Index].ID, + "match_type": candidates[rerankResp[i].Index].MatchType, + "chunk_type": candidates[rerankResp[i].Index].ChunkType, + "content_len": len(candidates[rerankResp[i].Index].Content), }) } } + if len(rerankResp) > logged { + pipelineInfo(ctx, "Rerank", "top_score_summary", map[string]interface{}{ + "total": len(rerankResp), + "logged": logged, + "truncated": len(rerankResp) - logged, + }) + } // Filter results based on threshold rankFilter := []rerank.RankResult{} @@ -529,11 +515,26 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string } combinedText += strings.Join(enrichments, "\n") - pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{ - "content_len": len(result.Content), - "cleaned_len": len(cleanPassageForRerank(result.Content)), - "enrichment_len": len(strings.Join(enrichments, "\n")), - }) - return combinedText } + +func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) { + const maxLogRows = 8 + limit := min(maxLogRows, len(results)) + for i := 0; i < limit; i++ { + sr := results[i] + pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{ + "index": i, + "chunk_id": sr.ID, + "score": fmt.Sprintf("%.4f", sr.Score), + "match_type": sr.MatchType, + }) + } + if len(results) > limit { + pipelineInfo(ctx, "Rerank", "input_score_summary", map[string]interface{}{ + "total": len(results), + "logged": limit, + "truncated": len(results) - limit, + }) + } +} diff --git a/internal/application/service/chat_pipline/rewrite.go b/internal/application/service/chat_pipline/rewrite.go index 8b31c96d..1a0ba737 100644 --- a/internal/application/service/chat_pipline/rewrite.go +++ b/internal/application/service/chat_pipline/rewrite.go @@ -11,9 +11,11 @@ import ( "time" "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" ) // PluginRewrite is a plugin for rewriting user queries @@ -26,10 +28,10 @@ type PluginRewrite struct { // reg is a regular expression used to match and remove content between tags var reg = regexp.MustCompile(`(?s).*?`) +var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`) const ( - noSearchPrefix = "[NO_SEARCH]" - imageDescSeparator = "\n---\n" + noSearchPrefix = "[NO_SEARCH]" ) // NewPluginRewrite creates a new query rewriting plugin instance @@ -118,8 +120,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, maxTokens = 500 } + // --- Emit progress event for image analysis --- + var toolCallID string + if useImages && chatManage.EventBus != nil { + toolCallID = uuid.New().String() + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolCall), + SessionID: chatManage.SessionID, + Data: event.AgentToolCallData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + }, + }) + } + // --- Call model --- thinking := false + vlmStart := time.Now() response, err := rewriteModel.Chat(ctx, []chat.Message{ {Role: "system", Content: systemContent}, userMsg, @@ -129,6 +146,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, Thinking: &thinking, }) if err != nil { + if toolCallID != "" && chatManage.EventBus != nil { + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolResult), + SessionID: chatManage.SessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "图片分析失败", + Success: false, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{ "session_id": chatManage.SessionID, "error": err.Error(), @@ -136,9 +166,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context, return next() } + // --- Emit completion event for image analysis --- + if toolCallID != "" && chatManage.EventBus != nil { + chatManage.EventBus.Emit(ctx, types.Event{ + Type: types.EventType(event.EventAgentToolResult), + SessionID: chatManage.SessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "已分析图片内容", + Success: true, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } + // --- Parse structured output --- p.parseRewriteOutput(chatManage, response.Content) - pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{ "session_id": chatManage.SessionID, "rewrite_query": chatManage.RewriteQuery, @@ -261,6 +305,13 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [ if chatManage.RewritePromptSystem != "" { systemPrompt = chatManage.RewritePromptSystem } + // Strengthen context inheritance in multi-turn conversation: + // for follow-up questions that clearly refer to previous turns (especially + // uploaded-image understanding), prefer NO_SEARCH over KB retrieval. + systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" + + "- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" + + "- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" + + "- In these follow-up cases, output with [NO_SEARCH] prefix." conversationText := formatConversationHistory(historyList) currentTime := time.Now().Format("2006-01-02 15:04:05") @@ -296,11 +347,16 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix)) } - // 2. Split rewritten query and image description - if idx := strings.Index(content, imageDescSeparator); idx >= 0 { - chatManage.RewriteQuery = strings.TrimSpace(content[:idx]) - chatManage.ImageOCRText = strings.TrimSpace(content[idx+len(imageDescSeparator):]) - } else if content != "" { + // 2. Split rewritten query and image description. + // Be tolerant to model output variants like: + // - "query\n---\nimage_desc" (expected) + // - "query---\nimage_desc" (missing leading newline before separator) + if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 { + chatManage.RewriteQuery = strings.TrimSpace(m[1]) + chatManage.ImageOCRText = strings.TrimSpace(m[2]) + return + } + if content != "" { chatManage.RewriteQuery = content } } diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 8d9ed23d..4422f2cb 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -113,15 +113,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context, chatManage.SearchResult = allResults - // Log all search results with scores before any processing - for i, r := range chatManage.SearchResult { - pipelineInfo(ctx, "Search", "result_score_before_normalize", map[string]interface{}{ - "index": i, - "chunk_id": r.ID, - "score": fmt.Sprintf("%.4f", r.Score), - "match_type": r.MatchType, - }) - } + logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult) // If recall is low, attempt query expansion with keyword-focused search if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) { @@ -131,15 +123,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context, } } - // Log final scores after all processing - for i, r := range chatManage.SearchResult { - pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{ - "index": i, - "chunk_id": r.ID, - "score": fmt.Sprintf("%.4f", r.Score), - "match_type": r.MatchType, - }) - } + logSearchScoreSample(ctx, "final_score", chatManage.SearchResult) // Return if we have results if len(chatManage.SearchResult) != 0 { @@ -204,6 +188,27 @@ func buildContentSignature(content string) string { return searchutil.BuildContentSignature(content) } +func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) { + const maxLogRows = 8 + limit := min(maxLogRows, len(results)) + for i := 0; i < limit; i++ { + r := results[i] + pipelineInfo(ctx, "Search", action, map[string]interface{}{ + "index": i, + "chunk_id": r.ID, + "score": fmt.Sprintf("%.4f", r.Score), + "match_type": r.MatchType, + }) + } + if len(results) > limit { + pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{ + "total": len(results), + "logged": limit, + "truncated": len(results) - limit, + }) + } +} + // searchByTargets performs KB searches using pre-computed SearchTargets // This is the main search method that uses the unified search targets func (p *PluginSearch) searchByTargets( diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index c5c9314b..c83c4258 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -1231,6 +1231,9 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, // Build final search results - preserve original order from input chunks var searchResults []*types.SearchResult addedChunkIDs := make(map[string]bool) + const maxInvalidChunkLog = 8 + invalidChunkCnt := 0 + invalidChunkSamples := make([]string, 0, maxInvalidChunkLog) // First pass: Add results in the original order from input chunks for _, inputChunk := range chunks { @@ -1240,7 +1243,10 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, continue } if !s.isValidTextChunk(chunk) { - logger.Debugf(ctx, "Chunk is not valid text chunk: %s, type: %s", chunk.ID, chunk.ChunkType) + invalidChunkCnt++ + if len(invalidChunkSamples) < maxInvalidChunkLog { + invalidChunkSamples = append(invalidChunkSamples, chunk.ID+":"+chunk.ChunkType) + } continue } if addedChunkIDs[chunk.ID] { @@ -1257,6 +1263,12 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, logger.Warnf(ctx, "Knowledge not found for chunk: %s, knowledge_id: %s", chunk.ID, chunk.KnowledgeID) } } + if invalidChunkCnt > 0 { + logger.Debugf(ctx, + "Skip non-text chunks in search results: total=%d sampled=%d samples=%v", + invalidChunkCnt, len(invalidChunkSamples), invalidChunkSamples, + ) + } // Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input if !skipEnrichment { diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index 308dd17b..ad566399 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -165,21 +165,10 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err)) } - // Decide whether to run VLM analysis here or defer. - // Agent mode defers to the async execution flow (so SSE stream is up and progress is visible). - // Normal mode defers when the RAG pipeline will run (rewrite step handles it). - // Only run here for Normal mode pure-chat paths (no KB, no web search). - isAgentEntry := logPrefix == "AgentQA" - hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0 - agentWillResolveKBs := !hasRequestKBs && - !customAgent.Config.RetrieveKBOnlyWhenMentioned && - customAgent.Config.KBSelectionMode != "" && - customAgent.Config.KBSelectionMode != "none" - willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled - if !willDeferVLM { - agentVLMModelID := customAgent.Config.VLMModelID - h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query) - } + // VLM analysis is always deferred to after SSE stream is up: + // - Agent mode: runs in async execution flow with tool_call/tool_result events + // - Normal RAG mode: runs in the pipeline rewrite step with progress events + // - Normal pure-chat mode: runs in the async goroutine with progress events } // Build request context @@ -478,6 +467,54 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo } }() + // For pure-chat path (no KB, no web search), VLM analysis isn't handled by the + // rewrite pipeline step, so run it here with progress events. + // RAG paths (with KB/web search, including agent-resolved KBs) defer VLM analysis + // to the rewrite step to avoid duplicate progress events. + hasRequestKBs := len(reqCtx.knowledgeBaseIDs) > 0 || len(reqCtx.knowledgeIDs) > 0 + agentWillResolveKBs := false + if !hasRequestKBs && reqCtx.customAgent != nil && !reqCtx.customAgent.Config.RetrieveKBOnlyWhenMentioned { + mode := reqCtx.customAgent.Config.KBSelectionMode + switch mode { + case "all": + agentWillResolveKBs = true + case "selected", "": + // "" keeps backward-compatible behavior (same as selected). + agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0 + case "none": + agentWillResolveKBs = false + default: + agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0 + } + } + isPureChatPath := !hasRequestKBs && !agentWillResolveKBs && !reqCtx.webSearchEnabled + if isPureChatPath && len(reqCtx.images) > 0 && + reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" { + toolCallID := uuid.New().String() + streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{ + Type: event.EventAgentToolCall, + SessionID: sessionID, + Data: event.AgentToolCallData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + }, + }) + vlmStart := time.Now() + h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images, + reqCtx.customAgent.Config.VLMModelID, reqCtx.query) + streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{ + Type: event.EventAgentToolResult, + SessionID: sessionID, + Data: event.AgentToolResultData{ + ToolCallID: toolCallID, + ToolName: "image_analysis", + Output: "已分析图片内容", + Success: true, + Duration: time.Since(vlmStart).Milliseconds(), + }, + }) + } + imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images) err := h.sessionService.KnowledgeQA( streamCtx.asyncCtx, diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 7fbfc13c..ed78113b 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -82,10 +82,10 @@ func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) { // request_id 优先输出 if v, ok := entry.Data["request_id"]; ok { if f.ForceColor { - fields += fmt.Sprintf("%srequest_id%s=%s%v%s ", - colorCyan, colorReset, colorBlue, v, colorReset) + fields += fmt.Sprintf("%s%v%s ", + colorBlue, v, colorReset) } else { - fields += fmt.Sprintf("request_id=%v ", v) + fields += fmt.Sprintf("%v ", v) } } diff --git a/internal/models/chat/image_resolve.go b/internal/models/chat/image_resolve.go index d7f41577..c4503b81 100644 --- a/internal/models/chat/image_resolve.go +++ b/internal/models/chat/image_resolve.go @@ -3,6 +3,7 @@ package chat import ( "encoding/base64" "fmt" + "log" "net/http" "os" "path/filepath" @@ -55,6 +56,7 @@ func readLocalStorageBytes(storagePath string) []byte { localPath := filepath.Join(baseDir, filepath.FromSlash(relPath)) data, err := os.ReadFile(localPath) if err != nil { + log.Printf("[image-resolve] failed to read local file %s: %v", localPath, err) return nil } return data diff --git a/internal/models/rerank/aliyun_reranker.go b/internal/models/rerank/aliyun_reranker.go index 3643b7f7..75103c87 100644 --- a/internal/models/rerank/aliyun_reranker.go +++ b/internal/models/rerank/aliyun_reranker.go @@ -112,10 +112,7 @@ func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []s req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging - logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", - r.baseURL, r.apiKey, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/jina_reranker.go b/internal/models/rerank/jina_reranker.go index 4b7b1947..5067123e 100644 --- a/internal/models/rerank/jina_reranker.go +++ b/internal/models/rerank/jina_reranker.go @@ -80,11 +80,7 @@ func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []str req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging (API key masked for security) - logger.GetLogger(ctx).Infof( - "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", - r.baseURL, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/logging.go b/internal/models/rerank/logging.go new file mode 100644 index 00000000..de38c111 --- /dev/null +++ b/internal/models/rerank/logging.go @@ -0,0 +1,42 @@ +package rerank + +import ( + "encoding/json" + "fmt" + "strings" + "unicode/utf8" +) + +const ( + maxLogDocuments = 3 + maxLogTextRunes = 120 +) + +func buildRerankRequestDebug(model, endpoint, query string, documents []string) string { + previews := make([]string, 0, maxLogDocuments) + for i, doc := range documents { + if i >= maxLogDocuments { + break + } + previews = append(previews, compactForLog(doc, maxLogTextRunes)) + } + + previewJSON, _ := json.Marshal(previews) + return fmt.Sprintf( + "rerank request endpoint=%s model=%s query_preview=%q query_runes=%d documents=%d preview_docs=%s", + endpoint, + model, + compactForLog(query, maxLogTextRunes), + utf8.RuneCountInString(query), + len(documents), + string(previewJSON), + ) +} + +func compactForLog(text string, maxRunes int) string { + normalized := strings.Join(strings.Fields(strings.TrimSpace(text)), " ") + if utf8.RuneCountInString(normalized) <= maxRunes { + return normalized + } + return string([]rune(normalized)[:maxRunes]) + "...(truncated)" +} diff --git a/internal/models/rerank/remote_api.go b/internal/models/rerank/remote_api.go index 945b9fc4..d9b77771 100644 --- a/internal/models/rerank/remote_api.go +++ b/internal/models/rerank/remote_api.go @@ -82,11 +82,7 @@ func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []s req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging (API key masked for security) - logger.GetLogger(ctx).Infof( - "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", - r.baseURL, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) resp, err := r.client.Do(req) if err != nil { diff --git a/internal/models/rerank/zhipu_reranker.go b/internal/models/rerank/zhipu_reranker.go index c9bbd7b4..ad0d801c 100644 --- a/internal/models/rerank/zhipu_reranker.go +++ b/internal/models/rerank/zhipu_reranker.go @@ -93,10 +93,7 @@ func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []st req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - // Log the curl equivalent for debugging - logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", - r.baseURL, r.apiKey, string(jsonData), - ) + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) resp, err := r.client.Do(req) if err != nil {