feat: enhance multimodal image processing and session management
- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images. - Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history. - Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users. - Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content. These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
This commit is contained in:
@@ -260,11 +260,19 @@ func (h *Handler) UpdateSession(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Reload session from database to return complete timestamps and stored fields
|
||||
updatedSession, err := h.sessionService.GetSession(ctx, id)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// Return updated session
|
||||
logger.Infof(ctx, "Session updated successfully, ID: %s", id)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": session,
|
||||
"data": updatedSession,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -164,9 +164,9 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
|
||||
}
|
||||
}
|
||||
|
||||
// createUserMessage creates a user message
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages) error {
|
||||
_, err := h.messageService.CreateMessage(ctx, &types.Message{
|
||||
// createUserMessage creates a user message and returns the created message.
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages) (*types.Message, error) {
|
||||
return h.messageService.CreateMessage(ctx, &types.Message{
|
||||
SessionID: sessionID,
|
||||
Role: "user",
|
||||
Content: query,
|
||||
@@ -176,7 +176,6 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
|
||||
MentionedItems: mentionedItems,
|
||||
Images: images,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// createAssistantMessage creates an assistant message
|
||||
|
||||
@@ -35,6 +35,7 @@ type qaRequestContext struct {
|
||||
mentionedItems types.MentionedItems
|
||||
effectiveTenantID uint64 // when using shared agent, tenant ID for model/KB/MCP resolution; 0 = use context tenant
|
||||
images []ImageAttachment // Uploaded images with analysis text
|
||||
userMessageID string // Created user message ID (populated after createUserMessage)
|
||||
}
|
||||
|
||||
// parseQARequest parses and validates a QA request, returns the request context
|
||||
@@ -403,10 +404,12 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
|
||||
sessionID := reqCtx.sessionID
|
||||
|
||||
// Create user message
|
||||
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
|
||||
if err != nil {
|
||||
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
}
|
||||
reqCtx.userMessageID = userMsg.ID
|
||||
|
||||
// Create assistant message
|
||||
if _, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage); err != nil {
|
||||
@@ -515,7 +518,7 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
|
||||
})
|
||||
}
|
||||
|
||||
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
|
||||
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
|
||||
err := h.sessionService.KnowledgeQA(
|
||||
streamCtx.asyncCtx,
|
||||
reqCtx.session,
|
||||
@@ -528,7 +531,8 @@ func (h *Handler) executeNormalModeQA(reqCtx *qaRequestContext, generateTitle bo
|
||||
streamCtx.eventBus,
|
||||
reqCtx.customAgent,
|
||||
reqCtx.enableMemory,
|
||||
imageURLs, imageOCRText,
|
||||
imageURLs, imageDescription,
|
||||
reqCtx.userMessageID,
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
|
||||
@@ -571,10 +575,12 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
|
||||
}
|
||||
|
||||
// Create user message
|
||||
if err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images)); err != nil {
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images))
|
||||
if err != nil {
|
||||
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
}
|
||||
reqCtx.userMessageID = userMsg.ID
|
||||
|
||||
// Create assistant message
|
||||
assistantMessagePtr, err := h.createAssistantMessage(ctx, reqCtx.assistantMessage)
|
||||
@@ -635,7 +641,7 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
|
||||
})
|
||||
}
|
||||
|
||||
imageURLs, imageOCRText := extractImageURLsAndOCRText(reqCtx.images)
|
||||
imageURLs, imageDescription := extractImageURLsAndOCRText(reqCtx.images)
|
||||
err := h.sessionService.AgentQA(
|
||||
streamCtx.asyncCtx,
|
||||
reqCtx.session,
|
||||
@@ -646,7 +652,8 @@ func (h *Handler) executeAgentModeQA(reqCtx *qaRequestContext) {
|
||||
reqCtx.customAgent,
|
||||
reqCtx.knowledgeBaseIDs,
|
||||
reqCtx.knowledgeIDs,
|
||||
imageURLs, imageOCRText,
|
||||
imageURLs, imageDescription,
|
||||
reqCtx.userMessageID,
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(streamCtx.asyncCtx, err, nil)
|
||||
|
||||
Reference in New Issue
Block a user