feat: enhance multimodal support and image handling

- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses.
- Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively.
- Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience.
- Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users.

These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
This commit is contained in:
wizardchen
2026-03-11 21:16:23 +08:00
parent 3a197f3333
commit bc2cb96d70
20 changed files with 467 additions and 247 deletions
+52 -15
View File
@@ -165,21 +165,10 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err))
}
// Decide whether to run VLM analysis here or defer.
// Agent mode defers to the async execution flow (so SSE stream is up and progress is visible).
// Normal mode defers when the RAG pipeline will run (rewrite step handles it).
// Only run here for Normal mode pure-chat paths (no KB, no web search).
isAgentEntry := logPrefix == "AgentQA"
hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0
agentWillResolveKBs := !hasRequestKBs &&
!customAgent.Config.RetrieveKBOnlyWhenMentioned &&
customAgent.Config.KBSelectionMode != "" &&
customAgent.Config.KBSelectionMode != "none"
willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled
if !willDeferVLM {
agentVLMModelID := customAgent.Config.VLMModelID
h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query)
}
// VLM analysis is always deferred to after SSE stream is up:
// - Agent mode: runs in async execution flow with tool_call/tool_result events
// - Normal RAG mode: runs in the pipeline rewrite step with progress events
// - Normal pure-chat mode: runs in the async goroutine with progress events
}
// Build request context
@@ -478,6 +467,54 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
}
}()
// For pure-chat path (no KB, no web search), VLM analysis isn't handled by the
// rewrite pipeline step, so run it here with progress events.
// RAG paths (with KB/web search, including agent-resolved KBs) defer VLM analysis
// to the rewrite step to avoid duplicate progress events.
hasRequestKBs := len(reqCtx.knowledgeBaseIDs) > 0 || len(reqCtx.knowledgeIDs) > 0
agentWillResolveKBs := false
if !hasRequestKBs && reqCtx.customAgent != nil && !reqCtx.customAgent.Config.RetrieveKBOnlyWhenMentioned {
mode := reqCtx.customAgent.Config.KBSelectionMode
switch mode {
case "all":
agentWillResolveKBs = true
case "selected", "":
// "" keeps backward-compatible behavior (same as selected).
agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
case "none":
agentWillResolveKBs = false
default:
agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
}
}
isPureChatPath := !hasRequestKBs && !agentWillResolveKBs && !reqCtx.webSearchEnabled
if isPureChatPath && len(reqCtx.images) > 0 &&
reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" {
toolCallID := uuid.New().String()
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolCall,
SessionID: sessionID,
Data: event.AgentToolCallData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
},
})
vlmStart := time.Now()
h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images,
reqCtx.customAgent.Config.VLMModelID, reqCtx.query)
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolResult,
SessionID: sessionID,
Data: event.AgentToolResultData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Output: "已分析图片内容",
Success: true,
Duration: time.Since(vlmStart).Milliseconds(),
},
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,