feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
This commit is contained in:
@@ -165,21 +165,10 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
|
||||
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err))
|
||||
}
|
||||
|
||||
// Decide whether to run VLM analysis here or defer.
|
||||
// Agent mode defers to the async execution flow (so SSE stream is up and progress is visible).
|
||||
// Normal mode defers when the RAG pipeline will run (rewrite step handles it).
|
||||
// Only run here for Normal mode pure-chat paths (no KB, no web search).
|
||||
isAgentEntry := logPrefix == "AgentQA"
|
||||
hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0
|
||||
agentWillResolveKBs := !hasRequestKBs &&
|
||||
!customAgent.Config.RetrieveKBOnlyWhenMentioned &&
|
||||
customAgent.Config.KBSelectionMode != "" &&
|
||||
customAgent.Config.KBSelectionMode != "none"
|
||||
willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled
|
||||
if !willDeferVLM {
|
||||
agentVLMModelID := customAgent.Config.VLMModelID
|
||||
h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query)
|
||||
}
|
||||
// VLM analysis is always deferred to after SSE stream is up:
|
||||
// - Agent mode: runs in async execution flow with tool_call/tool_result events
|
||||
// - Normal RAG mode: runs in the pipeline rewrite step with progress events
|
||||
// - Normal pure-chat mode: runs in the async goroutine with progress events
|
||||
}
|
||||
|
||||
// Build request context
|
||||
@@ -478,6 +467,54 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
|
||||
}
|
||||
}()
|
||||
|
||||
// For pure-chat path (no KB, no web search), VLM analysis isn't handled by the
|
||||
// rewrite pipeline step, so run it here with progress events.
|
||||
// RAG paths (with KB/web search, including agent-resolved KBs) defer VLM analysis
|
||||
// to the rewrite step to avoid duplicate progress events.
|
||||
hasRequestKBs := len(reqCtx.knowledgeBaseIDs) > 0 || len(reqCtx.knowledgeIDs) > 0
|
||||
agentWillResolveKBs := false
|
||||
if !hasRequestKBs && reqCtx.customAgent != nil && !reqCtx.customAgent.Config.RetrieveKBOnlyWhenMentioned {
|
||||
mode := reqCtx.customAgent.Config.KBSelectionMode
|
||||
switch mode {
|
||||
case "all":
|
||||
agentWillResolveKBs = true
|
||||
case "selected", "":
|
||||
// "" keeps backward-compatible behavior (same as selected).
|
||||
agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
|
||||
case "none":
|
||||
agentWillResolveKBs = false
|
||||
default:
|
||||
agentWillResolveKBs = len(reqCtx.customAgent.Config.KnowledgeBases) > 0
|
||||
}
|
||||
}
|
||||
isPureChatPath := !hasRequestKBs && !agentWillResolveKBs && !reqCtx.webSearchEnabled
|
||||
if isPureChatPath && len(reqCtx.images) > 0 &&
|
||||
reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" {
|
||||
toolCallID := uuid.New().String()
|
||||
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
|
||||
Type: event.EventAgentToolCall,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentToolCallData{
|
||||
ToolCallID: toolCallID,
|
||||
ToolName: "image_analysis",
|
||||
},
|
||||
})
|
||||
vlmStart := time.Now()
|
||||
h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images,
|
||||
reqCtx.customAgent.Config.VLMModelID, reqCtx.query)
|
||||
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
|
||||
Type: event.EventAgentToolResult,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentToolResultData{
|
||||
ToolCallID: toolCallID,
|
||||
ToolName: "image_analysis",
|
||||
Output: "已分析图片内容",
|
||||
Success: true,
|
||||
Duration: time.Since(vlmStart).Milliseconds(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
|
||||
err := h.sessionService.KnowledgeQA(
|
||||
streamCtx.asyncCtx,
|
||||
|
||||
Reference in New Issue
Block a user