diff --git a/config/config.yaml b/config/config.yaml
index 43516130..83db7c53 100644
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -48,75 +48,27 @@ conversation:
- Otherwise, output the rewritten question directly (no prefix = default to search)
- IMPORTANT: This is a knowledge base Q&A system. Default to SEARCH. Only add [NO_SEARCH] when you are very confident the question has nothing to do with the knowledge base.
- When to output [NO_SEARCH] (ONLY these narrow cases):
+ When to output [NO_SEARCH]:
- Pure greetings, thanks, or farewell with no question ("谢谢", "你好", "再见")
- Requests to summarize or manipulate the previous conversation itself ("总结一下我们的对话")
- - When unsure, do NOT add the prefix (default to search)
+ - Pure image understanding with NO intent to search documents: describing, summarizing, translating, or extracting content from the image itself ("这张图片是什么", "描述一下图片内容", "帮我翻译图中文字", "图里的表格数据是什么", "帮我识别一下这张图")
+ - Follow-up questions that clearly refer to previous conversation content (especially previously uploaded images) and can be answered from dialogue context directly ("第一张图再详细描述一下", "第二张门上的字是什么意思", "这个再展开讲讲")
When NOT to output [NO_SEARCH] (common mistakes to avoid):
- - User uploads an image and asks to find/search/look up something → SEARCH (the image content is the search query)
- - User asks about image content that may relate to documents ("这个报错怎么解决") → SEARCH
- - User asks a question with an image, even if it seems like a simple image description → SEARCH (the knowledge base may contain relevant information)
+ - User uploads an image and asks to FIND/SEARCH related content in knowledge base → SEARCH (the image content is the search query)
+ - User asks how to fix/solve/handle something shown in the image ("这个报错怎么解决") → SEARCH (knowledge base may have solutions)
+ - User asks a question that implies comparing image content against documents → SEARCH
## Task 3: Image Analysis (only when images are attached)
If the user's message includes images, after the rewritten question, add a separator "---" on a new line, then provide a concise description of the image content. This description is used as a text fallback for models that cannot process images directly.
+ If the image contains OCR text, also extract the key text information (OCR) from the image and include it in the description.
## Output Format
- Text only (needs search): rewritten question
- Text only (no search): [NO_SEARCH] rewritten question
- - With images (needs search): rewritten question\n---\nimage description
- - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description
+ - With images (needs search): rewritten question\n---\nimage description (including extracted OCR if present)
+ - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description (including extracted OCR if present)
- ## Few-shot Examples
-
- Example 1 (text, needs search):
- Conversation history:
- User: What features does Slack have?
- Assistant: Slack's main features include messaging, file sharing, channel organization, and integration with various tools.
- User question: Is it secure?
- Output: Is Slack secure?
-
- Example 2 (text, needs search):
- Conversation history:
- User: My laptop battery drains too fast, what should I do?
- Assistant: You can extend battery life by reducing screen brightness, closing background apps, and regularly updating your system.
- User question: Would that affect the user experience?
- Output: Would reducing screen brightness and closing background apps affect the user experience?
-
- Example 3 (text, no search - greeting):
- Conversation history:
- User: How do I install WeKnora?
- Assistant: You can install WeKnora using docker compose...
- User question: Thanks!
- Output: [NO_SEARCH] Thanks!
-
- Example 4 (text, no search - summarize conversation):
- Conversation history:
- User: What is RAG?
- Assistant: RAG stands for Retrieval Augmented Generation...
- User question: Summarize what we discussed
- Output: [NO_SEARCH] Summarize our discussion about RAG
-
- Example 5 (image, needs search - user wants to find related info):
- User question: 找一下这张图 [image attached]
- Output: 找一下这张图片相关的内容
- ---
- 图片中显示了一个系统架构图,包含前端、后端和数据库三层。
-
- Example 6 (image, needs search - error screenshot):
- Conversation history:
- User: How do I configure the search settings?
- Assistant: You can configure search settings in the admin panel...
- User question: I got this error, how to fix it? [image attached]
- Output: How to fix this search configuration error?
- ---
- The image shows an error dialog with the message "Invalid rerank model ID: model not found".
-
- Example 7 (image, needs search - image as query):
- User question: What is this? [image attached]
- Output: What is the content shown in this image?
- ---
- The image shows a product specification table with model numbers and parameters.
rewrite_prompt_user: |
## Conversation History
{{conversation}}
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts
index e4c38d4b..8ec019e6 100755
--- a/frontend/src/i18n/locales/en-US.ts
+++ b/frontend/src/i18n/locales/en-US.ts
@@ -1409,6 +1409,7 @@ export default {
advanced: 'Advanced Settings',
faq: 'FAQ Settings',
graph: 'Knowledge Graph',
+ multimodal: 'Multimodal',
share: 'Sharing'
},
basic: {
@@ -1610,9 +1611,13 @@ export default {
childChunkSizeLabel: 'Child Chunk Size',
childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)'
},
+ multimodal: {
+ title: 'Multimodal Configuration',
+ description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images',
+ },
advanced: {
title: 'Advanced Settings',
- description: 'Configure question generation, multimodal features',
+ description: 'Configure question generation and other advanced features',
questionGeneration: {
label: 'AI Question Generation',
description: 'Generate related questions for each chunk using LLM during document parsing to improve retrieval recall. Enabling this will increase document parsing time.',
@@ -3248,6 +3253,9 @@ export default {
selectPlaceholder: 'Select MCP services',
},
imageUpload: {
+ navLabel: 'Multimodal',
+ sectionTitle: 'Multimodal Configuration',
+ sectionDesc: 'Configure image upload and vision-language model for multimodal conversations',
label: 'Image Upload',
desc: 'Allow users to upload images for multimodal Q&A in conversations',
vlmModel: 'VLM Model',
diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts
index b1f39bf2..07b37d40 100755
--- a/frontend/src/i18n/locales/zh-CN.ts
+++ b/frontend/src/i18n/locales/zh-CN.ts
@@ -1868,6 +1868,7 @@ export default {
advanced: "高级设置",
faq: "FAQ 设置",
graph: "知识图谱",
+ multimodal: "多模态",
share: "共享管理",
},
basic: {
@@ -2066,9 +2067,13 @@ export default {
childChunkSizeLabel: "子块大小",
childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)",
},
+ multimodal: {
+ title: "多模态配置",
+ description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索",
+ },
advanced: {
title: "高级设置",
- description: "配置问题生成、多模态等高级功能",
+ description: "配置问题生成等高级功能",
questionGeneration: {
label: "AI 问题生成",
description: "解析文档时调用大模型为每个分块生成相关问题,提高检索召回率。启用后会增加文档解析耗时。",
@@ -3246,6 +3251,9 @@ export default {
selectPlaceholder: "选择 MCP 服务",
},
imageUpload: {
+ navLabel: "多模态",
+ sectionTitle: "多模态配置",
+ sectionDesc: "配置图片上传和视觉语言模型,启用后用户可在对话中上传图片",
label: "图片上传",
desc: "启用后用户可在对话中上传图片进行多模态问答",
vlmModel: "VLM 模型",
diff --git a/frontend/src/views/agent/AgentEditorModal.vue b/frontend/src/views/agent/AgentEditorModal.vue
index e5c4bbc0..0293cff8 100644
--- a/frontend/src/views/agent/AgentEditorModal.vue
+++ b/frontend/src/views/agent/AgentEditorModal.vue
@@ -319,6 +319,17 @@
+
+
+
+
+
+
+
+
@@ -1357,6 +1368,8 @@ const navItems = computed(() => {
}
// 网络搜索(独立菜单)
items.push({ key: 'websearch', icon: 'internet', label: t('agent.editor.webSearchConfig') });
+ // 多模态配置(图片上传)
+ items.push({ key: 'multimodal', icon: 'image', label: t('agentEditor.imageUpload.navLabel') });
// 多轮对话(仅普通模式显示,Agent模式内部自动控制)
if (!isAgentMode.value) {
items.push({ key: 'conversation', icon: 'chat', label: t('agent.editor.conversationSettings') });
@@ -2727,7 +2740,7 @@ const handleSave = async () => {
// 校验 VLM 模型(当图片上传启用时必填)
if (formData.value.config.image_upload_enabled && !formData.value.config.vlm_model_id) {
MessagePlugin.error(t('agentEditor.imageUpload.vlmModelRequired'));
- currentSection.value = 'model';
+ currentSection.value = 'multimodal';
return;
}
diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue
index 07c04b34..8228b9f9 100644
--- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue
+++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue
@@ -155,15 +155,58 @@
/>
+
+
+
+
+
+
+
+
+
+
{{ $t('knowledgeEditor.advanced.multimodal.label') }}
+
{{ $t('knowledgeEditor.advanced.multimodal.description') }}
+
+
+
+
+
+
+
+
+
+
{{ $t('knowledgeEditor.advanced.multimodal.vllmLabel') }} *
+
{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}
+
+
+
+
+
+
+
+
+
@@ -203,6 +246,7 @@ import KBParserSettings from './settings/KBParserSettings.vue'
import KBStorageSettings from './settings/KBStorageSettings.vue'
import KBChunkingSettings from './settings/KBChunkingSettings.vue'
import KBAdvancedSettings from './settings/KBAdvancedSettings.vue'
+import ModelSelector from '@/components/ModelSelector.vue'
import GraphSettings from './settings/GraphSettings.vue'
import KBShareSettings from './settings/KBShareSettings.vue'
import { useI18n } from 'vue-i18n'
@@ -244,6 +288,7 @@ const navItems = computed(() => {
{ key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') },
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
+ { key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') },
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
)
}
@@ -431,12 +476,22 @@ const handleParserEngineRulesUpdate = (rules: any[]) => {
}
}
-const handleMultimodalUpdate = (config: any) => {
- if (formData.value) {
- formData.value.multimodalConfig = { ...config }
+const handleMultimodalToggle = () => {
+ if (formData.value && !formData.value.multimodalConfig.enabled) {
+ formData.value.multimodalConfig.vllmModelId = ''
}
}
+const handleMultimodalVLLMChange = (modelId: string) => {
+ if (formData.value) {
+ formData.value.multimodalConfig.vllmModelId = modelId
+ }
+}
+
+const handleAddVLLMModel = () => {
+ uiStore.openSettings('models', 'vllm')
+}
+
const handleStorageProviderUpdate = (value: string) => {
if (formData.value) {
formData.value.storageProvider = value || 'local'
@@ -480,13 +535,10 @@ const validateForm = (): boolean => {
}
// 验证多模态配置(如果启用)
- if (formData.value.multimodalConfig.enabled) {
- const validation = (advancedSettingsRef.value as any)?.validateMultimodal?.()
- if (validation && !validation.valid) {
- MessagePlugin.warning(validation.message || t('knowledgeEditor.messages.multimodalInvalid'))
- currentSection.value = 'advanced'
- return false
- }
+ if (formData.value.multimodalConfig.enabled && !formData.value.multimodalConfig.vllmModelId) {
+ MessagePlugin.warning(t('knowledgeEditor.messages.multimodalInvalid'))
+ currentSection.value = 'multimodal'
+ return false
}
if (formData.value.type === 'faq' && !formData.value.faqConfig?.indexMode) {
@@ -1014,5 +1066,80 @@ watch(
}
}
}
+
+// 多模态配置内联样式(与子组件 KBStorageSettings/KBAdvancedSettings 一致)
+.kb-multimodal-settings {
+ width: 100%;
+
+ .section-header {
+ margin-bottom: 32px;
+
+ h2 {
+ font-size: 20px;
+ font-weight: 600;
+ color: var(--td-text-color-primary);
+ margin: 0 0 8px 0;
+ }
+
+ .section-description {
+ font-size: 14px;
+ color: var(--td-text-color-secondary);
+ margin: 0;
+ line-height: 1.5;
+ }
+ }
+
+ .settings-group {
+ display: flex;
+ flex-direction: column;
+ }
+
+ .setting-row {
+ display: flex;
+ align-items: flex-start;
+ justify-content: space-between;
+ padding: 20px 0;
+ border-bottom: 1px solid var(--td-component-stroke);
+
+ &:last-child {
+ border-bottom: none;
+ }
+ }
+
+ .setting-info {
+ flex: 1;
+ max-width: 65%;
+ padding-right: 24px;
+
+ label {
+ font-size: 15px;
+ font-weight: 500;
+ color: var(--td-text-color-primary);
+ display: block;
+ margin-bottom: 4px;
+ }
+
+ .desc {
+ font-size: 13px;
+ color: var(--td-text-color-secondary);
+ margin: 0;
+ line-height: 1.5;
+ }
+ }
+
+ .setting-control {
+ flex-shrink: 0;
+ min-width: 280px;
+ display: flex;
+ justify-content: flex-end;
+ align-items: center;
+ }
+
+ .required {
+ color: var(--td-error-color);
+ margin-left: 2px;
+ font-weight: 500;
+ }
+}
diff --git a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue
index 7be04239..9512c64b 100644
--- a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue
+++ b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue
@@ -42,57 +42,12 @@
-
-
-
-
{{ $t('knowledgeEditor.advanced.multimodal.label') }}
-
{{ $t('knowledgeEditor.advanced.multimodal.description') }}
-
-
-
-
-
-
-
-
-
-
-
{{ $t('knowledgeEditor.advanced.multimodal.vllmLabel') }} *
-
{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}
-
-
-
-
-
-
-
-
diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go
index 0f8cda4a..cf357e08 100644
--- a/internal/application/repository/retriever/postgres/repository.go
+++ b/internal/application/repository/retriever/postgres/repository.go
@@ -233,10 +233,19 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
+ const maxKeywordResultLog = 8
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords)
- logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
- i, results[i].ChunkID, results[i].Score)
+ if i < maxKeywordResultLog {
+ logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
+ i, results[i].ChunkID, results[i].Score)
+ }
+ }
+ if len(results) > maxKeywordResultLog {
+ logger.GetLogger(ctx).Debugf(
+ "[Postgres] Keywords result summary: total=%d logged=%d truncated=%d",
+ len(results), maxKeywordResultLog, len(results)-maxKeywordResultLog,
+ )
}
return []*types.RetrieveResult{
{
@@ -387,10 +396,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
+ const maxVectorResultLog = 8
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding)
- logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
- i, results[i].ChunkID, results[i].Score)
+ if i < maxVectorResultLog {
+ logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
+ i, results[i].ChunkID, results[i].Score)
+ }
+ }
+ if len(results) > maxVectorResultLog {
+ logger.GetLogger(ctx).Debugf(
+ "[Postgres] Vector search result summary: total=%d logged=%d truncated=%d",
+ len(results), maxVectorResultLog, len(results)-maxVectorResultLog,
+ )
}
return []*types.RetrieveResult{
{
@@ -494,7 +512,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
// Create new vector index, copy the content and vector of the source index
targetVector := &pgVector{
Content: sourceVector.Content,
- SourceID: targetSourceID, // Handle SourceID transformation properly
+ SourceID: targetSourceID, // Handle SourceID transformation properly
SourceType: sourceVector.SourceType,
ChunkID: targetChunkID, // Update to target chunk ID
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
diff --git a/internal/application/service/chat_pipline/into_chat_message.go b/internal/application/service/chat_pipline/into_chat_message.go
index d9d422cd..a81275f6 100644
--- a/internal/application/service/chat_pipline/into_chat_message.go
+++ b/internal/application/service/chat_pipline/into_chat_message.go
@@ -77,6 +77,30 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
// Prepare weekday names
weekdayName := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
+ // Intent-based no-search path: bypass "reference materials" template entirely.
+ if chatManage.SkipKBSearch {
+ // Prefer rewritten query in no-search mode; fallback to original query.
+ userContent := safeQuery
+ if rewrite := strings.TrimSpace(chatManage.RewriteQuery); rewrite != "" {
+ if safeRewrite, ok := utils.ValidateInput(rewrite); ok {
+ userContent = safeRewrite
+ } else {
+ pipelineWarn(ctx, "IntoChatMessage", "invalid_rewrite_query_fallback", map[string]interface{}{
+ "session_id": chatManage.SessionID,
+ })
+ }
+ }
+ if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
+ userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
+ }
+ chatManage.UserContent = userContent
+ pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{
+ "session_id": chatManage.SessionID,
+ "user_content_len": len(chatManage.UserContent),
+ })
+ return next()
+ }
+
var contextsBuilder strings.Builder
// Build contexts string based on FAQ priority strategy
diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go
index 3b6cd0c1..94560316 100644
--- a/internal/application/service/chat_pipline/rerank.go
+++ b/internal/application/service/chat_pipline/rerank.go
@@ -121,15 +121,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
"result_cnt": len(rerankResp),
})
- // Log input scores before reranking for debugging
- for i, sr := range chatManage.SearchResult {
- pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{
- "index": i,
- "chunk_id": sr.ID,
- "score": fmt.Sprintf("%.4f", sr.Score),
- "match_type": sr.MatchType,
- })
- }
+ logRerankInputScoreSample(ctx, chatManage.SearchResult)
for i := range chatManage.SearchResult {
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
@@ -162,13 +154,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
})
}
- pipelineInfo(ctx, "Rerank", "composite_calc", map[string]interface{}{
- "chunk_id": sr.ID,
- "base_score": fmt.Sprintf("%.4f", base),
- "model_score": fmt.Sprintf("%.4f", modelScore),
- "final_score": fmt.Sprintf("%.4f", sr.Score),
- "match_type": sr.MatchType,
- })
reranked = append(reranked, sr)
}
@@ -179,13 +164,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
// Assign high model score for direct load items
modelScore := 1.0
sr.Score = compositeScore(sr, modelScore, base)
- pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{
- "chunk_id": sr.ID,
- "base_score": fmt.Sprintf("%.4f", base),
- "model_score": fmt.Sprintf("%.4f", modelScore),
- "final_score": fmt.Sprintf("%.4f", sr.Score),
- "match_type": sr.MatchType,
- })
reranked = append(reranked, sr)
}
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
@@ -237,18 +215,26 @@ func (p *PluginRerank) rerank(ctx context.Context,
pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{
"threshold": chatManage.RerankThreshold,
})
- for i := range min(5, len(rerankResp)) {
+ logged := min(5, len(rerankResp))
+ for i := range logged {
if rerankResp[i].Index < len(candidates) {
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
- "rank": i + 1,
- "score": rerankResp[i].RelevanceScore,
- "chunk_id": candidates[rerankResp[i].Index].ID,
- "match_type": candidates[rerankResp[i].Index].MatchType,
- "chunk_type": candidates[rerankResp[i].Index].ChunkType,
- "content": candidates[rerankResp[i].Index].Content,
+ "rank": i + 1,
+ "score": rerankResp[i].RelevanceScore,
+ "chunk_id": candidates[rerankResp[i].Index].ID,
+ "match_type": candidates[rerankResp[i].Index].MatchType,
+ "chunk_type": candidates[rerankResp[i].Index].ChunkType,
+ "content_len": len(candidates[rerankResp[i].Index].Content),
})
}
}
+ if len(rerankResp) > logged {
+ pipelineInfo(ctx, "Rerank", "top_score_summary", map[string]interface{}{
+ "total": len(rerankResp),
+ "logged": logged,
+ "truncated": len(rerankResp) - logged,
+ })
+ }
// Filter results based on threshold
rankFilter := []rerank.RankResult{}
@@ -529,11 +515,26 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
}
combinedText += strings.Join(enrichments, "\n")
- pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{
- "content_len": len(result.Content),
- "cleaned_len": len(cleanPassageForRerank(result.Content)),
- "enrichment_len": len(strings.Join(enrichments, "\n")),
- })
-
return combinedText
}
+
+func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) {
+ const maxLogRows = 8
+ limit := min(maxLogRows, len(results))
+ for i := 0; i < limit; i++ {
+ sr := results[i]
+ pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{
+ "index": i,
+ "chunk_id": sr.ID,
+ "score": fmt.Sprintf("%.4f", sr.Score),
+ "match_type": sr.MatchType,
+ })
+ }
+ if len(results) > limit {
+ pipelineInfo(ctx, "Rerank", "input_score_summary", map[string]interface{}{
+ "total": len(results),
+ "logged": limit,
+ "truncated": len(results) - limit,
+ })
+ }
+}
diff --git a/internal/application/service/chat_pipline/rewrite.go b/internal/application/service/chat_pipline/rewrite.go
index 8b31c96d..1a0ba737 100644
--- a/internal/application/service/chat_pipline/rewrite.go
+++ b/internal/application/service/chat_pipline/rewrite.go
@@ -11,9 +11,11 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/config"
+ "github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/google/uuid"
)
// PluginRewrite is a plugin for rewriting user queries
@@ -26,10 +28,10 @@ type PluginRewrite struct {
// reg is a regular expression used to match and remove content between tags
var reg = regexp.MustCompile(`(?s).*? `)
+var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`)
const (
- noSearchPrefix = "[NO_SEARCH]"
- imageDescSeparator = "\n---\n"
+ noSearchPrefix = "[NO_SEARCH]"
)
// NewPluginRewrite creates a new query rewriting plugin instance
@@ -118,8 +120,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
maxTokens = 500
}
+ // --- Emit progress event for image analysis ---
+ var toolCallID string
+ if useImages && chatManage.EventBus != nil {
+ toolCallID = uuid.New().String()
+ chatManage.EventBus.Emit(ctx, types.Event{
+ Type: types.EventType(event.EventAgentToolCall),
+ SessionID: chatManage.SessionID,
+ Data: event.AgentToolCallData{
+ ToolCallID: toolCallID,
+ ToolName: "image_analysis",
+ },
+ })
+ }
+
// --- Call model ---
thinking := false
+ vlmStart := time.Now()
response, err := rewriteModel.Chat(ctx, []chat.Message{
{Role: "system", Content: systemContent},
userMsg,
@@ -129,6 +146,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
Thinking: &thinking,
})
if err != nil {
+ if toolCallID != "" && chatManage.EventBus != nil {
+ chatManage.EventBus.Emit(ctx, types.Event{
+ Type: types.EventType(event.EventAgentToolResult),
+ SessionID: chatManage.SessionID,
+ Data: event.AgentToolResultData{
+ ToolCallID: toolCallID,
+ ToolName: "image_analysis",
+ Output: "图片分析失败",
+ Success: false,
+ Duration: time.Since(vlmStart).Milliseconds(),
+ },
+ })
+ }
pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
@@ -136,9 +166,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
return next()
}
+ // --- Emit completion event for image analysis ---
+ if toolCallID != "" && chatManage.EventBus != nil {
+ chatManage.EventBus.Emit(ctx, types.Event{
+ Type: types.EventType(event.EventAgentToolResult),
+ SessionID: chatManage.SessionID,
+ Data: event.AgentToolResultData{
+ ToolCallID: toolCallID,
+ ToolName: "image_analysis",
+ Output: "已分析图片内容",
+ Success: true,
+ Duration: time.Since(vlmStart).Milliseconds(),
+ },
+ })
+ }
+
// --- Parse structured output ---
p.parseRewriteOutput(chatManage, response.Content)
-
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
@@ -261,6 +305,13 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
if chatManage.RewritePromptSystem != "" {
systemPrompt = chatManage.RewritePromptSystem
}
+ // Strengthen context inheritance in multi-turn conversation:
+ // for follow-up questions that clearly refer to previous turns (especially
+ // uploaded-image understanding), prefer NO_SEARCH over KB retrieval.
+ systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" +
+ "- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" +
+ "- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" +
+ "- In these follow-up cases, output with [NO_SEARCH] prefix."
conversationText := formatConversationHistory(historyList)
currentTime := time.Now().Format("2006-01-02 15:04:05")
@@ -296,11 +347,16 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str
content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix))
}
- // 2. Split rewritten query and image description
- if idx := strings.Index(content, imageDescSeparator); idx >= 0 {
- chatManage.RewriteQuery = strings.TrimSpace(content[:idx])
- chatManage.ImageOCRText = strings.TrimSpace(content[idx+len(imageDescSeparator):])
- } else if content != "" {
+ // 2. Split rewritten query and image description.
+ // Be tolerant to model output variants like:
+ // - "query\n---\nimage_desc" (expected)
+ // - "query---\nimage_desc" (missing leading newline before separator)
+ if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 {
+ chatManage.RewriteQuery = strings.TrimSpace(m[1])
+ chatManage.ImageOCRText = strings.TrimSpace(m[2])
+ return
+ }
+ if content != "" {
chatManage.RewriteQuery = content
}
}
diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go
index 8d9ed23d..4422f2cb 100644
--- a/internal/application/service/chat_pipline/search.go
+++ b/internal/application/service/chat_pipline/search.go
@@ -113,15 +113,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
chatManage.SearchResult = allResults
- // Log all search results with scores before any processing
- for i, r := range chatManage.SearchResult {
- pipelineInfo(ctx, "Search", "result_score_before_normalize", map[string]interface{}{
- "index": i,
- "chunk_id": r.ID,
- "score": fmt.Sprintf("%.4f", r.Score),
- "match_type": r.MatchType,
- })
- }
+ logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
// If recall is low, attempt query expansion with keyword-focused search
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
@@ -131,15 +123,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
}
}
- // Log final scores after all processing
- for i, r := range chatManage.SearchResult {
- pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{
- "index": i,
- "chunk_id": r.ID,
- "score": fmt.Sprintf("%.4f", r.Score),
- "match_type": r.MatchType,
- })
- }
+ logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
// Return if we have results
if len(chatManage.SearchResult) != 0 {
@@ -204,6 +188,27 @@ func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
+func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
+ const maxLogRows = 8
+ limit := min(maxLogRows, len(results))
+ for i := 0; i < limit; i++ {
+ r := results[i]
+ pipelineInfo(ctx, "Search", action, map[string]interface{}{
+ "index": i,
+ "chunk_id": r.ID,
+ "score": fmt.Sprintf("%.4f", r.Score),
+ "match_type": r.MatchType,
+ })
+ }
+ if len(results) > limit {
+ pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
+ "total": len(results),
+ "logged": limit,
+ "truncated": len(results) - limit,
+ })
+ }
+}
+
// searchByTargets performs KB searches using pre-computed SearchTargets
// This is the main search method that uses the unified search targets
func (p *PluginSearch) searchByTargets(
diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go
index c5c9314b..c83c4258 100644
--- a/internal/application/service/knowledgebase.go
+++ b/internal/application/service/knowledgebase.go
@@ -1231,6 +1231,9 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
// Build final search results - preserve original order from input chunks
var searchResults []*types.SearchResult
addedChunkIDs := make(map[string]bool)
+ const maxInvalidChunkLog = 8
+ invalidChunkCnt := 0
+ invalidChunkSamples := make([]string, 0, maxInvalidChunkLog)
// First pass: Add results in the original order from input chunks
for _, inputChunk := range chunks {
@@ -1240,7 +1243,10 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
continue
}
if !s.isValidTextChunk(chunk) {
- logger.Debugf(ctx, "Chunk is not valid text chunk: %s, type: %s", chunk.ID, chunk.ChunkType)
+ invalidChunkCnt++
+ if len(invalidChunkSamples) < maxInvalidChunkLog {
+ invalidChunkSamples = append(invalidChunkSamples, chunk.ID+":"+chunk.ChunkType)
+ }
continue
}
if addedChunkIDs[chunk.ID] {
@@ -1257,6 +1263,12 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
logger.Warnf(ctx, "Knowledge not found for chunk: %s, knowledge_id: %s", chunk.ID, chunk.KnowledgeID)
}
}
+ if invalidChunkCnt > 0 {
+ logger.Debugf(ctx,
+ "Skip non-text chunks in search results: total=%d sampled=%d samples=%v",
+ invalidChunkCnt, len(invalidChunkSamples), invalidChunkSamples,
+ )
+ }
// Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input
if !skipEnrichment {
diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go
index 308dd17b..ad566399 100644
--- a/internal/handler/session/qa.go
+++ b/internal/handler/session/qa.go
@@ -165,21 +165,10 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err))
}
- // Decide whether to run VLM analysis here or defer.
- // Agent mode defers to the async execution flow (so SSE stream is up and progress is visible).
- // Normal mode defers when the RAG pipeline will run (rewrite step handles it).
- // Only run here for Normal mode pure-chat paths (no KB, no web search).
- isAgentEntry := logPrefix == "AgentQA"
- hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0
- agentWillResolveKBs := !hasRequestKBs &&
- !customAgent.Config.RetrieveKBOnlyWhenMentioned &&
- customAgent.Config.KBSelectionMode != "" &&
- customAgent.Config.KBSelectionMode != "none"
- willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled
- if !willDeferVLM {
- agentVLMModelID := customAgent.Config.VLMModelID
- h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query)
- }
+ // VLM analysis is always deferred to after SSE stream is up:
+ // - Agent mode: runs in async execution flow with tool_call/tool_result events
+ // - Normal RAG mode: runs in the pipeline rewrite step with progress events
+ // - Normal pure-chat mode: runs in the async goroutine with progress events
}
// Build request context
@@ -478,6 +467,54 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
}
}()
+ // For pure-chat path (no KB, no web search), VLM analysis isn't handled by the
+ // rewrite pipeline step, so run it here with progress events.
+ // RAG paths (with KB/web search, including agent-resolved KBs) defer VLM analysis
+ // to the rewrite step to avoid duplicate progress events.
+ hasRequestKBs := len(reqCtx.knowledgeBaseIDs) > 0 || len(reqCtx.knowledgeIDs) > 0
+ agentWillResolveKBs := false
+ if !hasRequestKBs && reqCtx.customAgent != nil && !reqCtx.customAgent.Config.RetrieveKBOnlyWhenMentioned {
+ mode := reqCtx.customAgent.Config.KBSelectionMode
+ switch mode {
+ case "all":
+ agentWillResolveKBs = true
+ case "selected", "":
+ // "" keeps backward-compatible behavior (same as selected).
+ agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
+ case "none":
+ agentWillResolveKBs = false
+ default:
+ agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
+ }
+ }
+ isPureChatPath := !hasRequestKBs && !agentWillResolveKBs && !reqCtx.webSearchEnabled
+ if isPureChatPath && len(reqCtx.images) > 0 &&
+ reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" {
+ toolCallID := uuid.New().String()
+ streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
+ Type: event.EventAgentToolCall,
+ SessionID: sessionID,
+ Data: event.AgentToolCallData{
+ ToolCallID: toolCallID,
+ ToolName: "image_analysis",
+ },
+ })
+ vlmStart := time.Now()
+ h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images,
+ reqCtx.customAgent.Config.VLMModelID, reqCtx.query)
+ streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
+ Type: event.EventAgentToolResult,
+ SessionID: sessionID,
+ Data: event.AgentToolResultData{
+ ToolCallID: toolCallID,
+ ToolName: "image_analysis",
+ Output: "已分析图片内容",
+ Success: true,
+ Duration: time.Since(vlmStart).Milliseconds(),
+ },
+ })
+ }
+
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,
diff --git a/internal/logger/logger.go b/internal/logger/logger.go
index 7fbfc13c..ed78113b 100644
--- a/internal/logger/logger.go
+++ b/internal/logger/logger.go
@@ -82,10 +82,10 @@ func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) {
// request_id 优先输出
if v, ok := entry.Data["request_id"]; ok {
if f.ForceColor {
- fields += fmt.Sprintf("%srequest_id%s=%s%v%s ",
- colorCyan, colorReset, colorBlue, v, colorReset)
+ fields += fmt.Sprintf("%s%v%s ",
+ colorBlue, v, colorReset)
} else {
- fields += fmt.Sprintf("request_id=%v ", v)
+ fields += fmt.Sprintf("%v ", v)
}
}
diff --git a/internal/models/chat/image_resolve.go b/internal/models/chat/image_resolve.go
index d7f41577..c4503b81 100644
--- a/internal/models/chat/image_resolve.go
+++ b/internal/models/chat/image_resolve.go
@@ -3,6 +3,7 @@ package chat
import (
"encoding/base64"
"fmt"
+ "log"
"net/http"
"os"
"path/filepath"
@@ -55,6 +56,7 @@ func readLocalStorageBytes(storagePath string) []byte {
localPath := filepath.Join(baseDir, filepath.FromSlash(relPath))
data, err := os.ReadFile(localPath)
if err != nil {
+ log.Printf("[image-resolve] failed to read local file %s: %v", localPath, err)
return nil
}
return data
diff --git a/internal/models/rerank/aliyun_reranker.go b/internal/models/rerank/aliyun_reranker.go
index 3643b7f7..75103c87 100644
--- a/internal/models/rerank/aliyun_reranker.go
+++ b/internal/models/rerank/aliyun_reranker.go
@@ -112,10 +112,7 @@ func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []s
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
- // Log the curl equivalent for debugging
- logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'",
- r.baseURL, r.apiKey, string(jsonData),
- )
+ logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
resp, err := r.client.Do(req)
if err != nil {
diff --git a/internal/models/rerank/jina_reranker.go b/internal/models/rerank/jina_reranker.go
index 4b7b1947..5067123e 100644
--- a/internal/models/rerank/jina_reranker.go
+++ b/internal/models/rerank/jina_reranker.go
@@ -80,11 +80,7 @@ func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []str
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
- // Log the curl equivalent for debugging (API key masked for security)
- logger.GetLogger(ctx).Infof(
- "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
- r.baseURL, string(jsonData),
- )
+ logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
resp, err := r.client.Do(req)
if err != nil {
diff --git a/internal/models/rerank/logging.go b/internal/models/rerank/logging.go
new file mode 100644
index 00000000..de38c111
--- /dev/null
+++ b/internal/models/rerank/logging.go
@@ -0,0 +1,42 @@
+package rerank
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "unicode/utf8"
+)
+
+const (
+ maxLogDocuments = 3
+ maxLogTextRunes = 120
+)
+
+func buildRerankRequestDebug(model, endpoint, query string, documents []string) string {
+ previews := make([]string, 0, maxLogDocuments)
+ for i, doc := range documents {
+ if i >= maxLogDocuments {
+ break
+ }
+ previews = append(previews, compactForLog(doc, maxLogTextRunes))
+ }
+
+ previewJSON, _ := json.Marshal(previews)
+ return fmt.Sprintf(
+ "rerank request endpoint=%s model=%s query_preview=%q query_runes=%d documents=%d preview_docs=%s",
+ endpoint,
+ model,
+ compactForLog(query, maxLogTextRunes),
+ utf8.RuneCountInString(query),
+ len(documents),
+ string(previewJSON),
+ )
+}
+
+func compactForLog(text string, maxRunes int) string {
+ normalized := strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
+ if utf8.RuneCountInString(normalized) <= maxRunes {
+ return normalized
+ }
+ return string([]rune(normalized)[:maxRunes]) + "...(truncated)"
+}
diff --git a/internal/models/rerank/remote_api.go b/internal/models/rerank/remote_api.go
index 945b9fc4..d9b77771 100644
--- a/internal/models/rerank/remote_api.go
+++ b/internal/models/rerank/remote_api.go
@@ -82,11 +82,7 @@ func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []s
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
- // Log the curl equivalent for debugging (API key masked for security)
- logger.GetLogger(ctx).Infof(
- "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
- r.baseURL, string(jsonData),
- )
+ logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
resp, err := r.client.Do(req)
if err != nil {
diff --git a/internal/models/rerank/zhipu_reranker.go b/internal/models/rerank/zhipu_reranker.go
index c9bbd7b4..ad0d801c 100644
--- a/internal/models/rerank/zhipu_reranker.go
+++ b/internal/models/rerank/zhipu_reranker.go
@@ -93,10 +93,7 @@ func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []st
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
- // Log the curl equivalent for debugging
- logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'",
- r.baseURL, r.apiKey, string(jsonData),
- )
+ logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
resp, err := r.client.Do(req)
if err != nil {