feat: enhance multimodal image processing and session management

- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images.
- Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history.
- Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users.
- Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content.

These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
This commit is contained in:
wizardchen
2026-03-12 13:17:47 +08:00
parent bc2cb96d70
commit 4eed96f901
15 changed files with 319 additions and 65 deletions
+9 -1
View File
@@ -260,11 +260,19 @@ func (h *Handler) UpdateSession(c *gin.Context) {
return
}
// Reload session from database to return complete timestamps and stored fields
updatedSession, err := h.sessionService.GetSession(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
// Return updated session
logger.Infof(ctx, "Session updated successfully, ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": session,
"data": updatedSession,
})
}
+3 -4
View File
@@ -164,9 +164,9 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
}
}
// createUserMessage creates a user message
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages) error {
_, err := h.messageService.CreateMessage(ctx, &types.Message{
// createUserMessage creates a user message and returns the created message.
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages) (*types.Message, error) {
return h.messageService.CreateMessage(ctx, &types.Message{
SessionID: sessionID,
Role: "user",
Content: query,
@@ -176,7 +176,6 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
MentionedItems: mentionedItems,
Images: images,
})
return err
}
// createAssistantMessage creates an assistant message
+13 -6
View File
@@ -35,6 +35,7 @@ type qaRequestContext struct {
mentionedItems types.MentionedItems
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
images []ImageAttachment // Uploaded images with analysis text
userMessageID string // Created user message ID (populated after createUserMessage)
}
// parseQARequest parses and validates a QA request, returns the request context
@@ -403,10 +404,12 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
sessionID := reqCtx.sessionID
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
if err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
reqCtx.userMessageID = userMsg.ID
// Create assistant message
if _, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage); err != nil {
@@ -515,7 +518,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.KnowledgeQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -528,7 +531,8 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
streamCtx.eventBus,
reqCtx.customAgent,
reqCtx.enableMemory,
imageURLs, imageOCRText,
imageURLs, imageDescription,
reqCtx.userMessageID,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
@@ -571,10 +575,12 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
}
// Create user message
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
if err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
}
reqCtx.userMessageID = userMsg.ID
// Create assistant message
assistantMessagePtr, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage)
@@ -635,7 +641,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
})
}
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
err := h.sessionService.AgentQA(
streamCtx.asyncCtx,
reqCtx.session,
@@ -646,7 +652,8 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
reqCtx.customAgent,
reqCtx.knowledgeBaseIDs,
reqCtx.knowledgeIDs,
imageURLs, imageOCRText,
imageURLs, imageDescription,
reqCtx.userMessageID,
)
if err != nil {
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)