feat: implement image upload and multimodal support

- Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A.
- Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes.
- Updated the backend to process images alongside text queries, including support for image analysis.
- Introduced new UI components for image previews and management, improving user interaction with uploaded content.
- Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages.

These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses.
This commit is contained in:
wizardchen
2026-03-11 17:17:32 +08:00
parent 8de1cfb6b5
commit 991bd29a14
42 changed files with 1554 additions and 226 deletions
+74 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
// qaRequestContext holds all the common data needed for QA requests
@@ -32,7 +33,8 @@ type qaRequestContext struct {
webSearchEnabled bool
enableMemory bool // Whether memory feature is enabled
mentionedItems types.MentionedItems
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
images []ImageAttachment // Uploaded images with analysis text
}
// parseQARequest parses and validates a QA request, returns the request context
@@ -63,7 +65,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
// Log request details
if requestJSON, err := json.Marshal(request); err == nil {
logger.Infof(ctx, "[%s] Request: session_id=%s, request=%s",
logPrefix, sessionID, secutils.SanitizeForLog(string(requestJSON)))
logPrefix, sessionID, secutils.SanitizeForLog(secutils.CompactImageDataURLForLog(string(requestJSON))))
}
// Get session
@@ -148,6 +150,38 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
logger.Infof(ctx, "[%s] @mention merge: request.KnowledgeBaseIDs=%v, request.MentionedItems=%d, merged kbIDs=%v, merged knowledgeIDs=%v",
logPrefix, request.KnowledgeBaseIDs, len(request.MentionedItems), kbIDs, knowledgeIDs)
// Process inline base64 images: decode and save to storage.
// VLM analysis for RAG paths is deferred to the pipeline rewrite step.
// For pure chat paths with non-vision models, VLM analysis runs here as fallback.
if len(request.Images) > 0 {
if customAgent == nil || !customAgent.Config.ImageUploadEnabled {
logger.Warnf(ctx, "[%s] Image upload is not enabled for this agent, rejecting %d images", logPrefix, len(request.Images))
return nil, nil, errors.NewBadRequestError("Image upload is not enabled for this agent")
}
tenantID := c.GetUint64(types.TenantIDContextKey.String())
agentStorageProvider := customAgent.Config.ImageStorageProvider
if err := h.saveImageAttachments(ctx, request.Images, tenantID, agentStorageProvider); err != nil {
logger.Errorf(ctx, "[%s] Failed to save images: %v", logPrefix, err)
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err))
}
// Decide whether to run VLM analysis here or defer.
// Agent mode defers to the async execution flow (so SSE stream is up and progress is visible).
// Normal mode defers when the RAG pipeline will run (rewrite step handles it).
// Only run here for Normal mode pure-chat paths (no KB, no web search).
isAgentEntry := logPrefix == "AgentQA"
hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0
agentWillResolveKBs := !hasRequestKBs &&
!customAgent.Config.RetrieveKBOnlyWhenMentioned &&
customAgent.Config.KBSelectionMode != "" &&
customAgent.Config.KBSelectionMode != "none"
willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled
if !willDeferVLM {
agentVLMModelID := customAgent.Config.VLMModelID
h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query)
}
}
// Build request context
reqCtx := &qaRequestContext{
ctx: ctx,
@@ -170,6 +204,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
enableMemory: request.EnableMemory,
mentionedItems: convertMentionedItems(request.MentionedItems),
effectiveTenantID: effectiveTenantID,
images: request.Images,
}
return reqCtx, &request, nil
@@ -379,7 +414,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
sessionID := reqCtx.sessionID
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems); err != nil {
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -443,6 +478,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
}
}()
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -455,6 +491,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
streamCtx.eventBus,
reqCtx.customAgent,
reqCtx.enableMemory,
imageURLs, imageOCRText,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -497,7 +534,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
}
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems); err != nil {
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -531,6 +568,37 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
logger.Infof(streamCtx.asyncCtx, "Agent QA service completed for session: %s", sessionID)
}()
// Run VLM analysis inside the async flow so SSE stream is already up
// and the user can see progress. Emit tool_call/tool_result events.
if len(reqCtx.images) > 0 && reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" {
toolCallID := uuid.New().String()
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolCall,
SessionID: sessionID,
Data: event.AgentToolCallData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Iteration: 0,
},
})
vlmStart := time.Now()
h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images,
reqCtx.customAgent.Config.VLMModelID, reqCtx.query)
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolResult,
SessionID: sessionID,
Data: event.AgentToolResultData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Output: "已查看图片内容",
Success: true,
Duration: time.Since(vlmStart).Milliseconds(),
Iteration: 0,
},
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.AgentQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -541,6 +609,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
reqCtx.customAgent,
reqCtx.knowledgeBaseIDs,
reqCtx.knowledgeIDs,
imageURLs, imageOCRText,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -573,3 +642,4 @@ func (h *Handler) completeAssistantMessage(ctx context.Context, assistantMessage
bgCtx := context.WithoutCancel(ctx)
go h.messageService.IndexMessageToKB(bgCtx, userQuery, assistantMessage.Content, assistantMessage.ID, assistantMessage.SessionID)
}