feat: enhance multimodal image processing and session management

- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images.
- Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history.
- Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users.
- Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content.

These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
This commit is contained in:
wizardchen
2026-03-12 13:17:47 +08:00
parent bc2cb96d70
commit 4eed96f901
15 changed files with 319 additions and 65 deletions
+13 -6
View File
@@ -35,6 +35,7 @@ type qaRequestContext struct {
mentionedItems types.MentionedItems
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
images []ImageAttachment // Uploaded images with analysis text
userMessageID string // Created user message ID (populated after createUserMessage)
}
// parseQARequest parses and validates a QA request, returns the request context
@@ -403,10 +404,12 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
sessionID := reqCtx.sessionID
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
if err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
reqCtx.userMessageID = userMsg.ID
// Create assistant message
if _, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage); err != nil {
@@ -515,7 +518,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -528,7 +531,8 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
streamCtx.eventBus,
reqCtx.customAgent,
reqCtx.enableMemory,
imageURLs, imageOCRText,
imageURLs, imageDescription,
reqCtx.userMessageID,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -571,10 +575,12 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
}
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
if err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
reqCtx.userMessageID = userMsg.ID
// Create assistant message
assistantMessagePtr, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage)
@@ -635,7 +641,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.AgentQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -646,7 +652,8 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
reqCtx.customAgent,
reqCtx.knowledgeBaseIDs,
reqCtx.knowledgeIDs,
imageURLs, imageOCRText,
imageURLs, imageDescription,
reqCtx.userMessageID,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)