feat: implement image upload and multimodal support

- Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A.
- Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes.
- Updated the backend to process images alongside text queries, including support for image analysis.
- Introduced new UI components for image previews and management, improving user interaction with uploaded content.
- Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages.

These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses.
This commit is contained in:
wizardchen
2026-03-11 17:17:32 +08:00
parent 8de1cfb6b5
commit 991bd29a14
42 changed files with 1554 additions and 226 deletions
+6
View File
@@ -22,6 +22,8 @@ type Handler struct {
customAgentService interfaces.CustomAgentService // Service for managing custom agents
tenantService interfaces.TenantService // Service for loading tenant (shared agent context)
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
fileService interfaces.FileService // Service for file storage (image uploads)
modelService interfaces.ModelService // Service for model management (VLM access)
}
// NewHandler creates a new instance of Handler with all necessary dependencies
@@ -34,6 +36,8 @@ func NewHandler(
customAgentService interfaces.CustomAgentService,
tenantService interfaces.TenantService,
agentShareService interfaces.AgentShareService,
fileService interfaces.FileService,
modelService interfaces.ModelService,
) *Handler {
return &Handler{
sessionService: sessionService,
@@ -44,6 +48,8 @@ func NewHandler(
customAgentService: customAgentService,
tenantService: tenantService,
agentShareService: agentShareService,
fileService: fileService,
modelService: modelService,
}
}
+44 -1
View File
@@ -3,6 +3,7 @@ package session
import (
"context"
"fmt"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/event"
@@ -12,6 +13,47 @@ import (
"github.com/gin-gonic/gin"
)
// convertImageAttachments converts ImageAttachment slice to types.MessageImages
func convertImageAttachments(items []ImageAttachment) types.MessageImages {
if len(items) == 0 {
return nil
}
result := make(types.MessageImages, len(items))
for i, item := range items {
result[i] = types.MessageImage{
URL: item.URL,
Caption: item.Caption,
}
}
return result
}
// extractImageURLsAndOCRText extracts image references and concatenated analysis text.
// For LLM consumption it prefers the raw Data (data URI) when available so that
// image_resolve can skip the disk round-trip; falls back to the storage URL otherwise.
func extractImageURLsAndOCRText(images []ImageAttachment) (urls []string, ocrText string) {
if len(images) == 0 {
return nil, ""
}
urls = make([]string, 0, len(images))
var parts []string
for _, img := range images {
switch {
case img.Data != "":
urls = append(urls, img.Data)
case img.URL != "":
urls = append(urls, img.URL)
}
if img.Caption != "" {
parts = append(parts, img.Caption)
}
}
if len(parts) > 0 {
ocrText = strings.Join(parts, "\n")
}
return
}
// convertMentionedItems converts MentionedItemRequest slice to types.MentionedItems
func convertMentionedItems(items []MentionedItemRequest) types.MentionedItems {
if len(items) == 0 {
@@ -123,7 +165,7 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
}
// createUserMessage creates a user message
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems) error {
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages) error {
_, err := h.messageService.CreateMessage(ctx, &types.Message{
SessionID: sessionID,
Role: "user",
@@ -132,6 +174,7 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
CreatedAt: time.Now(),
IsCompleted: true,
MentionedItems: mentionedItems,
Images: images,
})
return err
}
+160
View File
@@ -0,0 +1,160 @@
package session
import (
"context"
"encoding/base64"
"fmt"
"strings"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
)
const (
maxImageSize = 10 << 20 // 10MB per image
maxImagesCount = 5
)
// saveImageAttachments decodes base64 images from the request and saves them to
// storage. The images slice is mutated in place: URL is populated.
// This is always called when images are present. VLM analysis is handled
// separately (either in the pipeline rewrite step for RAG paths, or via
// analyzeImageAttachments for pure chat paths with non-vision models).
func (h *Handler) saveImageAttachments(ctx context.Context, images []ImageAttachment, tenantID uint64, storageProvider string) error {
if len(images) == 0 {
return nil
}
if len(images) > maxImagesCount {
return fmt.Errorf("too many images, max %d", maxImagesCount)
}
fileSvc := h.resolveImageFileService(ctx, storageProvider)
for i := range images {
img := &images[i]
if img.Data == "" {
continue
}
imgBytes, ext, err := decodeDataURI(img.Data)
if err != nil {
return fmt.Errorf("decode image %d: %w", i, err)
}
if len(imgBytes) > maxImageSize {
return fmt.Errorf("image %d too large (%d bytes, max %d)", i, len(imgBytes), maxImageSize)
}
storedName := fmt.Sprintf("chat-images/%s%s", uuid.New().String(), ext)
fileURL, err := fileSvc.SaveBytes(ctx, imgBytes, tenantID, storedName, false)
if err != nil {
return fmt.Errorf("save image %d: %w", i, err)
}
img.URL = fileURL
}
return nil
}
// analyzeImageAttachments runs VLM analysis on saved images and populates Caption.
// Used as a fallback for pure chat paths where the pipeline rewrite step won't run.
// For RAG paths, image analysis is handled in the pipeline rewrite step instead.
func (h *Handler) analyzeImageAttachments(ctx context.Context, images []ImageAttachment, vlmModelID string, userQuery string) {
if len(images) == 0 || vlmModelID == "" {
return
}
vlmModel, err := h.modelService.GetVLMModel(ctx, vlmModelID)
if err != nil {
logger.Warnf(ctx, "No VLM model available for image analysis, skipping: %v", err)
return
}
for i := range images {
img := &images[i]
if img.Data == "" {
continue
}
imgBytes, _, decErr := decodeDataURI(img.Data)
if decErr != nil {
logger.Warnf(ctx, "Failed to decode image %d for VLM analysis: %v", i, decErr)
continue
}
prompt := buildImageAnalysisPrompt(userQuery)
analysis, analysisErr := vlmModel.Predict(ctx, imgBytes, prompt)
if analysisErr != nil {
logger.Warnf(ctx, "VLM analysis failed for image %d: %v", i, analysisErr)
} else {
img.Caption = analysis
}
}
}
// buildImageAnalysisPrompt generates a context-aware VLM prompt based on the
// user's question. Instead of doing generic OCR + Caption separately, we do a
// single analysis call that is tailored to the user's intent.
func buildImageAnalysisPrompt(userQuery string) string {
if strings.TrimSpace(userQuery) == "" {
return "请分析这张图片的内容。如果包含文字,请提取关键文字信息;如果是自然图片,请描述其主要内容。用简洁的中文回答。"
}
return fmt.Sprintf(
"用户的问题是:%s\n\n请分析图片中与用户问题相关的内容。"+
"如果图片包含文字/文档/表格,请提取与问题相关的关键信息。"+
"如果是自然图片/截图/图表,请描述与问题相关的视觉内容。"+
"用简洁的中文回答,只输出分析结果。",
userQuery,
)
}
func decodeDataURI(dataURI string) ([]byte, string, error) {
if !strings.HasPrefix(dataURI, "data:") {
return nil, "", fmt.Errorf("not a data URI")
}
idx := strings.Index(dataURI, ";base64,")
if idx < 0 {
return nil, "", fmt.Errorf("unsupported data URI encoding (expected base64)")
}
mimeType := dataURI[5:idx]
decoded, err := base64.StdEncoding.DecodeString(dataURI[idx+8:])
if err != nil {
return nil, "", fmt.Errorf("base64 decode: %w", err)
}
ext := mimeToExt(mimeType)
return decoded, ext, nil
}
func mimeToExt(mime string) string {
switch strings.ToLower(mime) {
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
default:
return ".png"
}
}
func (h *Handler) resolveImageFileService(ctx context.Context, storageProvider string) interfaces.FileService {
if strings.TrimSpace(storageProvider) == "" {
return h.fileService
}
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil || tenant.StorageEngineConfig == nil {
return h.fileService
}
svc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(storageProvider, tenant.StorageEngineConfig, "")
if err != nil {
logger.Warnf(ctx, "[image-storage] failed to create %s file service: %v, fallback to default", storageProvider, err)
return h.fileService
}
logger.Infof(ctx, "[image-storage] using provider=%s for image uploads", resolvedProvider)
return svc
}
+74 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
// qaRequestContext holds all the common data needed for QA requests
@@ -32,7 +33,8 @@ type qaRequestContext struct {
webSearchEnabled bool
enableMemory bool // Whether memory feature is enabled
mentionedItems types.MentionedItems
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
images []ImageAttachment // Uploaded images with analysis text
}
// parseQARequest parses and validates a QA request, returns the request context
@@ -63,7 +65,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
// Log request details
if requestJSON, err := json.Marshal(request); err == nil {
logger.Infof(ctx, "[%s] Request: session_id=%s, request=%s",
logPrefix, sessionID, secutils.SanitizeForLog(string(requestJSON)))
logPrefix, sessionID, secutils.SanitizeForLog(secutils.CompactImageDataURLForLog(string(requestJSON))))
}
// Get session
@@ -148,6 +150,38 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
logger.Infof(ctx, "[%s] @mention merge: request.KnowledgeBaseIDs=%v, request.MentionedItems=%d, merged kbIDs=%v, merged knowledgeIDs=%v",
logPrefix, request.KnowledgeBaseIDs, len(request.MentionedItems), kbIDs, knowledgeIDs)
// Process inline base64 images: decode and save to storage.
// VLM analysis for RAG paths is deferred to the pipeline rewrite step.
// For pure chat paths with non-vision models, VLM analysis runs here as fallback.
if len(request.Images) > 0 {
if customAgent == nil || !customAgent.Config.ImageUploadEnabled {
logger.Warnf(ctx, "[%s] Image upload is not enabled for this agent, rejecting %d images", logPrefix, len(request.Images))
return nil, nil, errors.NewBadRequestError("Image upload is not enabled for this agent")
}
tenantID := c.GetUint64(types.TenantIDContextKey.String())
agentStorageProvider := customAgent.Config.ImageStorageProvider
if err := h.saveImageAttachments(ctx, request.Images, tenantID, agentStorageProvider); err != nil {
logger.Errorf(ctx, "[%s] Failed to save images: %v", logPrefix, err)
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("Image save failed: %v", err))
}
// Decide whether to run VLM analysis here or defer.
// Agent mode defers to the async execution flow (so SSE stream is up and progress is visible).
// Normal mode defers when the RAG pipeline will run (rewrite step handles it).
// Only run here for Normal mode pure-chat paths (no KB, no web search).
isAgentEntry := logPrefix == "AgentQA"
hasRequestKBs := len(kbIDs) > 0 || len(knowledgeIDs) > 0
agentWillResolveKBs := !hasRequestKBs &&
!customAgent.Config.RetrieveKBOnlyWhenMentioned &&
customAgent.Config.KBSelectionMode != "" &&
customAgent.Config.KBSelectionMode != "none"
willDeferVLM := isAgentEntry || hasRequestKBs || agentWillResolveKBs || request.WebSearchEnabled
if !willDeferVLM {
agentVLMModelID := customAgent.Config.VLMModelID
h.analyzeImageAttachments(ctx, request.Images, agentVLMModelID, request.Query)
}
}
// Build request context
reqCtx := &qaRequestContext{
ctx: ctx,
@@ -170,6 +204,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
enableMemory: request.EnableMemory,
mentionedItems: convertMentionedItems(request.MentionedItems),
effectiveTenantID: effectiveTenantID,
images: request.Images,
}
return reqCtx, &request, nil
@@ -379,7 +414,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
sessionID := reqCtx.sessionID
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems); err != nil {
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -443,6 +478,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
}
}()
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -455,6 +491,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
streamCtx.eventBus,
reqCtx.customAgent,
reqCtx.enableMemory,
imageURLs, imageOCRText,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -497,7 +534,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
}
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems); err != nil {
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -531,6 +568,37 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
logger.Infof(streamCtx.asyncCtx, "Agent QA service completed for session: %s", sessionID)
}()
// Run VLM analysis inside the async flow so SSE stream is already up
// and the user can see progress. Emit tool_call/tool_result events.
if len(reqCtx.images) > 0 && reqCtx.customAgent != nil && reqCtx.customAgent.Config.VLMModelID != "" {
toolCallID := uuid.New().String()
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolCall,
SessionID: sessionID,
Data: event.AgentToolCallData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Iteration: 0,
},
})
vlmStart := time.Now()
h.analyzeImageAttachments(streamCtx.asyncCtx, reqCtx.images,
reqCtx.customAgent.Config.VLMModelID, reqCtx.query)
streamCtx.eventBus.Emit(streamCtx.asyncCtx, event.Event{
Type: event.EventAgentToolResult,
SessionID: sessionID,
Data: event.AgentToolResultData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Output: "已查看图片内容",
Success: true,
Duration: time.Since(vlmStart).Milliseconds(),
Iteration: 0,
},
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.AgentQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -541,6 +609,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
reqCtx.customAgent,
reqCtx.knowledgeBaseIDs,
reqCtx.knowledgeIDs,
imageURLs, imageOCRText,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -573,3 +642,4 @@ func (h *Handler) completeAssistantMessage(ctx context.Context, assistantMessage
bgCtx := context.WithoutCancel(ctx)
go h.messageService.IndexMessageToKB(bgCtx, userQuery, assistantMessage.Content, assistantMessage.ID, assistantMessage.SessionID)
}
+10
View File
@@ -27,6 +27,15 @@ type MentionedItemRequest struct {
KBType string `json:"kb_type"` // "document" or "faq" (only for kb type)
}
// ImageAttachment represents an image in a chat request.
// Frontend sends base64 data in the Data field; the backend saves, runs VLM analysis,
// and populates URL/Caption before proceeding with the chat pipeline.
type ImageAttachment struct {
Data string `json:"data,omitempty"` // base64 data URI from frontend (data:image/png;base64,...)
URL string `json:"url,omitempty"` // serving URL after saving to storage
Caption string `json:"caption,omitempty"` // VLM analysis result (context-aware, single call)
}
// CreateKnowledgeQARequest defines the request structure for knowledge QA
type CreateKnowledgeQARequest struct {
Query string `json:"query" binding:"required"` // Query text for knowledge base search
@@ -39,6 +48,7 @@ type CreateKnowledgeQARequest struct {
MentionedItems []MentionedItemRequest `json:"mentioned_items"` // @mentioned knowledge bases and files
DisableTitle bool `json:"disable_title"` // Whether to disable auto title generation
EnableMemory bool `json:"enable_memory"` // Whether memory feature is enabled for this request
Images []ImageAttachment `json:"images"` // Attached images for multimodal chat
}
// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization