feat: enhance multimodal image processing and session management

- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images.
- Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history.
- Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users.
- Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content.

These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
This commit is contained in:
wizardchen
2026-03-12 13:17:47 +08:00
parent bc2cb96d70
commit 4eed96f901
15 changed files with 319 additions and 65 deletions
@@ -235,6 +235,16 @@ func (r *messageRepository) GetKnowledgeIDsBySessionID(
return knowledgeIDs, nil
}
// UpdateMessageImages updates only the images JSONB column for a message.
// Uses Select to force GORM to include the column even when struct-based
// Updates would otherwise skip custom Valuer types.
func (r *messageRepository) UpdateMessageImages(ctx context.Context, sessionID, messageID string, images types.MessageImages) error {
return r.db.WithContext(ctx).
Model(&types.Message{}).
Where("id = ? AND session_id = ?", messageID, sessionID).
Update("images", images).Error
}
// UpdateMessageKnowledgeID updates the knowledge_id field for a message
func (r *messageRepository) UpdateMessageKnowledgeID(
ctx context.Context, messageID string, knowledgeID string,
+10 -3
View File
@@ -43,7 +43,7 @@ func (r *sessionRepository) Get(ctx context.Context, tenantID uint64, id string)
// GetByTenantID retrieves all sessions for a tenant
func (r *sessionRepository) GetByTenantID(ctx context.Context, tenantID uint64) ([]*types.Session, error) {
var sessions []*types.Session
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&sessions).Error
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("updated_at DESC").Find(&sessions).Error
if err != nil {
return nil, err
}
@@ -66,7 +66,7 @@ func (r *sessionRepository) GetPagedByTenantID(
// Then query the paginated data
err = r.db.WithContext(ctx).
Where("tenant_id = ?", tenantID).
Order("created_at DESC").
Order("updated_at DESC").
Offset(page.Offset()).
Limit(page.Limit()).
Find(&sessions).Error
@@ -80,7 +80,14 @@ func (r *sessionRepository) GetPagedByTenantID(
// Update updates a session
func (r *sessionRepository) Update(ctx context.Context, session *types.Session) error {
session.UpdatedAt = time.Now()
return r.db.WithContext(ctx).Where("tenant_id = ?", session.TenantID).Save(session).Error
return r.db.WithContext(ctx).
Model(&types.Session{}).
Where("tenant_id = ? AND id = ?", session.TenantID, session.ID).
Updates(map[string]interface{}{
"title": session.Title,
"description": session.Description,
"updated_at": session.UpdatedAt,
}).Error
}
// Delete deletes a session
@@ -78,6 +78,19 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
return chatMessages
}
// extractImageCaptions concatenates non-empty Caption fields from stored
// message images. Used when loading history so that previous turns' image
// descriptions are visible to the model.
func extractImageCaptions(images types.MessageImages) string {
var parts []string
for _, img := range images {
if img.Caption != "" {
parts = append(parts, img.Caption)
}
}
return strings.Join(parts, "\n")
}
// renderSystemPromptPlaceholders replaces placeholders in system prompt
// Supported placeholders:
// - {{current_time}} -> current time in RFC3339 format
@@ -90,8 +90,8 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
})
}
}
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
}
chatManage.UserContent = userContent
pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{
@@ -148,16 +148,19 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
// Append image description as text fallback only when the chat model cannot
// process images directly. Vision-capable models see images via MultiContent.
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
}
// Set formatted content back to chat management
chatManage.UserContent = userContent
pipelineInfo(ctx, "IntoChatMessage", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_content_len": len(chatManage.UserContent),
"faq_priority": chatManage.FAQPriorityEnabled,
"session_id": chatManage.SessionID,
"user_content_len": len(chatManage.UserContent),
"faq_priority": chatManage.FAQPriorityEnabled,
"skip_kb_search": chatManage.SkipKBSearch,
"image_description": chatManage.ImageDescription,
"chat_model_supports_vision": chatManage.ChatModelSupportsVision,
})
return next()
}
@@ -83,9 +83,11 @@ func (p *PluginLoadHistory) OnEvent(ctx context.Context,
h = &types.History{}
}
if message.Role == "user" {
// User message as query
h.Query = message.Content
h.CreateAt = message.CreatedAt
if desc := extractImageCaptions(message.Images); desc != "" {
h.Query += "\n\n[用户上传图片内容]\n" + desc
}
} else {
// System message as answer, while removing thinking process
h.Answer = regThink.ReplaceAllString(message.Content, "")
@@ -4,6 +4,7 @@ package chatpipline
import (
"context"
"encoding/json"
"regexp"
"slices"
"sort"
@@ -34,6 +35,12 @@ const (
noSearchPrefix = "[NO_SEARCH]"
)
type rewriteOutput struct {
RewriteQuery string
SkipKBSearch bool
ImageDescription string
}
// NewPluginRewrite creates a new query rewriting plugin instance
// Also registers the plugin with the event manager
func NewPluginRewrite(eventManager *EventManager,
@@ -183,16 +190,53 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
// --- Parse structured output ---
p.parseRewriteOutput(chatManage, response.Content)
// Persist image description back to the user message so that future turns
// can see it when loading conversation history.
if chatManage.ImageDescription != "" && chatManage.UserMessageID != "" {
p.updateUserMessageImageCaption(ctx, chatManage)
}
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"skip_kb_search": chatManage.SkipKBSearch,
"has_image_desc": chatManage.ImageOCRText != "",
"has_image_desc": chatManage.ImageDescription != "",
"original_output": response.Content,
})
return next()
}
// updateUserMessageImageCaption writes the generated ImageDescription back to
// the stored user message's Images so that subsequent turns can see it in history.
func (p *PluginRewrite) updateUserMessageImageCaption(ctx context.Context, chatManage *types.ChatManage) {
msg, err := p.messageService.GetMessage(ctx, chatManage.SessionID, chatManage.UserMessageID)
if err != nil {
pipelineWarn(ctx, "Rewrite", "get_user_message", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_message_id": chatManage.UserMessageID,
"error": err.Error(),
})
return
}
if len(msg.Images) == 0 {
return
}
msg.Images[0].Caption = chatManage.ImageDescription
// Use the targeted UpdateMessageImages to reliably persist the JSONB column.
// GORM's struct-based Updates may silently skip custom Valuer types.
if err := p.messageService.UpdateMessageImages(ctx, chatManage.SessionID, chatManage.UserMessageID, msg.Images); err != nil {
pipelineWarn(ctx, "Rewrite", "update_image_caption", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_message_id": chatManage.UserMessageID,
"error": err.Error(),
})
}
}
// loadHistory fetches and processes conversation history for rewrite context.
func (p *PluginRewrite) loadHistory(ctx context.Context, chatManage *types.ChatManage) []*types.History {
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
@@ -212,6 +256,9 @@ func (p *PluginRewrite) loadHistory(ctx context.Context, chatManage *types.ChatM
if message.Role == "user" {
h.Query = message.Content
h.CreateAt = message.CreatedAt
if desc := extractImageCaptions(message.Images); desc != "" {
h.Query += "\n\n[用户上传图片内容]\n" + desc
}
} else {
h.Answer = reg.ReplaceAllString(message.Content, "")
h.KnowledgeReferences = message.KnowledgeReferences
@@ -307,11 +354,14 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
}
// Strengthen context inheritance in multi-turn conversation:
// for follow-up questions that clearly refer to previous turns (especially
// uploaded-image understanding), prefer NO_SEARCH over KB retrieval.
// uploaded-image understanding), prefer skipping KB retrieval.
systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" +
"- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" +
"- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST set skip_kb_search=true.\n" +
"- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" +
"- In these follow-up cases, output with [NO_SEARCH] prefix."
"- You MUST output ONLY a single JSON object (no markdown/code fences, no extra text) with schema: {\"rewrite_query\":\"...\",\"skip_kb_search\":true|false,\"image_description\":\"...\"}.\n" +
"- `skip_kb_search` field is REQUIRED and must be explicit true/false.\n" +
"- For images, `image_description` MUST include a detailed visual description and complete OCR text. Keep all key details; do not overly summarize.\n" +
"- image_description should be empty string only when there are no images or truly no readable visual text/details."
conversationText := formatConversationHistory(historyList)
currentTime := time.Now().Format("2006-01-02 15:04:05")
@@ -333,27 +383,34 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
//
// Expected formats:
//
// Text only: "[NO_SEARCH] rewritten question" or "rewritten question"
// With images: "[NO_SEARCH]\nrewritten question\n---\nimage description"
// Preferred: {"rewrite_query":"...","skip_kb_search":false,"image_description":"..."}
// Legacy fallback:
// - Text only: "[NO_SEARCH] rewritten question" or "rewritten question"
// - With images: "[NO_SEARCH]\nrewritten question\n---\nimage description"
func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw string) {
content := strings.TrimSpace(raw)
if content == "" {
return
}
// 1. Parse intent marker
if output, ok := parseStructuredRewriteOutput(content); ok {
if rewrite := strings.TrimSpace(output.RewriteQuery); rewrite != "" {
chatManage.RewriteQuery = rewrite
}
chatManage.SkipKBSearch = output.SkipKBSearch
chatManage.ImageDescription = strings.TrimSpace(output.ImageDescription)
return
}
// Legacy fallback parsing for older prompts/models.
if strings.HasPrefix(content, noSearchPrefix) {
chatManage.SkipKBSearch = true
content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix))
}
// 2. Split rewritten query and image description.
// Be tolerant to model output variants like:
// - "query\n---\nimage_desc" (expected)
// - "query---\nimage_desc" (missing leading newline before separator)
if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 {
chatManage.RewriteQuery = strings.TrimSpace(m[1])
chatManage.ImageOCRText = strings.TrimSpace(m[2])
chatManage.ImageDescription = strings.TrimSpace(m[2])
return
}
if content != "" {
@@ -361,6 +418,128 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str
}
}
func parseStructuredRewriteOutput(raw string) (rewriteOutput, bool) {
content := strings.TrimSpace(raw)
if content == "" {
return rewriteOutput{}, false
}
var out rewriteOutput
if parsed, ok := parseStructuredRewriteOutputJSON(content); ok {
return parsed, true
}
// Be tolerant to occasional markdown wrappers or extra prose.
start := strings.Index(content, "{")
end := strings.LastIndex(content, "}")
if start < 0 || end <= start {
return rewriteOutput{}, false
}
candidate := content[start : end+1]
if parsed, ok := parseStructuredRewriteOutputJSON(candidate); ok {
return parsed, true
}
return out, false
}
func parseStructuredRewriteOutputJSON(content string) (rewriteOutput, bool) {
var obj map[string]json.RawMessage
if err := json.Unmarshal([]byte(content), &obj); err != nil {
return rewriteOutput{}, false
}
out := rewriteOutput{
RewriteQuery: strings.TrimSpace(firstStringField(obj,
"rewrite_query", "rewritten_query", "query", "question")),
}
// Support common variants and semantic inversion for need_search.
if v, ok := firstBoolField(obj, "skip_kb_search", "skip_search", "no_search"); ok {
out.SkipKBSearch = v
} else if v, ok := firstBoolField(obj, "need_search", "requires_search"); ok {
out.SkipKBSearch = !v
}
desc := strings.TrimSpace(firstStringField(obj,
"image_description", "image_desc", "image_text", "image_ocr_text", "description"))
ocr := strings.TrimSpace(firstStringField(obj,
"ocr_text", "ocr", "full_ocr", "image_ocr", "ocr_content"))
combined, set := mergeImageDescAndOCR(desc, ocr)
if set {
out.ImageDescription = combined
}
return out, true
}
func firstStringField(obj map[string]json.RawMessage, keys ...string) string {
for _, key := range keys {
raw, ok := obj[key]
if !ok || len(raw) == 0 {
continue
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return s
}
}
return ""
}
func firstBoolField(obj map[string]json.RawMessage, keys ...string) (bool, bool) {
for _, key := range keys {
raw, ok := obj[key]
if !ok || len(raw) == 0 {
continue
}
if v, ok := parseBoolJSON(raw); ok {
return v, true
}
}
return false, false
}
func parseBoolJSON(raw json.RawMessage) (bool, bool) {
var b bool
if err := json.Unmarshal(raw, &b); err == nil {
return b, true
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
switch strings.ToLower(strings.TrimSpace(s)) {
case "true", "1", "yes", "y":
return true, true
case "false", "0", "no", "n":
return false, true
}
}
var n float64
if err := json.Unmarshal(raw, &n); err == nil {
return n != 0, true
}
return false, false
}
func mergeImageDescAndOCR(desc, ocr string) (string, bool) {
if desc == "" && ocr == "" {
return "", false
}
if desc == "" {
return ocr, true
}
if ocr == "" {
return desc, true
}
if strings.Contains(desc, ocr) {
return desc, true
}
return desc + "\n\n[OCR]\n" + ocr, true
}
// formatConversationHistory formats conversation history for prompt template
func formatConversationHistory(historyList []*types.History) string {
if len(historyList) == 0 {
+5
View File
@@ -237,6 +237,11 @@ func (s *messageService) UpdateMessage(ctx context.Context, message *types.Messa
return nil
}
// UpdateMessageImages updates only the images JSONB column for a message.
func (s *messageService) UpdateMessageImages(ctx context.Context, sessionID, messageID string, images types.MessageImages) error {
return s.messageRepo.UpdateMessageImages(ctx, sessionID, messageID, images)
}
// DeleteMessage removes a message from a session, also cleaning up its Knowledge entry in the chat history KB.
func (s *messageService) DeleteMessage(ctx context.Context, sessionID string, messageID string) error {
logger.Info(ctx, "Start deleting message")
+25 -14
View File
@@ -523,7 +523,7 @@ func (s *sessionService) KnowledgeQA(
eventBus *event.EventBus,
customAgent *types.CustomAgent,
enableMemory bool,
imageURLs []string, imageOCRText string,
imageURLs []string, imageDescription string, userMessageID string,
) error {
logger.Infof(
ctx,
@@ -768,8 +768,9 @@ func (s *sessionService) KnowledgeQA(
FAQDirectAnswerThreshold: faqDirectAnswerThreshold,
FAQScoreBoost: faqScoreBoost,
// Image support
UserMessageID: userMessageID,
Images: imageURLs,
ImageOCRText: imageOCRText,
ImageDescription: imageDescription,
VLMModelID: vlmModelID,
ChatModelSupportsVision: chatModelSupportsVision,
}
@@ -783,8 +784,8 @@ func (s *sessionService) KnowledgeQA(
// For pure chat, UserContent is the Query (since INTO_CHAT_MESSAGE is skipped)
// Only append image text description for non-vision models; vision models see images directly
userContent := query
if imageOCRText != "" && !chatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + imageOCRText
if imageDescription != "" && !chatModelSupportsVision {
userContent += "\n\n[用户上传图片内容]\n" + imageDescription
}
chatManage.UserContent = userContent
@@ -1391,8 +1392,9 @@ func (s *sessionService) AgentQA(
customAgent *types.CustomAgent,
knowledgeBaseIDs []string,
knowledgeIDs []string,
imageURLs []string, imageOCRText string,
imageURLs []string, imageDescription string, userMessageID string,
) error {
_ = userMessageID // reserved for future use (AgentQA pipeline differs from KnowledgeQA)
sessionID := session.ID
sessionJSON, err := json.Marshal(session)
if err != nil {
@@ -1624,9 +1626,9 @@ func (s *sessionService) AgentQA(
if agentModelSupportsVision && len(imageURLs) > 0 {
agentImageURLs = imageURLs
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
} else if imageOCRText != "" {
agentQuery = query + "\n\n[用户上传图片内容]\n" + imageOCRText
logger.Infof(ctx, "Agent model does not support vision, appending image OCR text (%d chars)", len(imageOCRText))
} else if imageDescription != "" {
agentQuery = query + "\n\n[用户上传图片内容]\n" + imageDescription
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(imageDescription))
}
// Execute agent with streaming (asynchronously)
@@ -1762,9 +1764,11 @@ func (s *sessionService) handleModelFallback(ctx context.Context, chatManage *ty
}
// Start streaming response
responseChan, err := chatModel.ChatStream(ctx, []chat.Message{
{Role: "user", Content: promptContent},
}, opt)
userMsg := chat.Message{Role: "user", Content: promptContent}
if chatManage.ChatModelSupportsVision && len(chatManage.Images) > 0 {
userMsg.Images = chatManage.Images
}
responseChan, err := chatModel.ChatStream(ctx, []chat.Message{userMsg}, opt)
if err != nil {
logger.Errorf(ctx, "Failed to start streaming fallback response: %v, falling back to fixed response", err)
s.handleFixedFallback(ctx, chatManage)
@@ -1781,10 +1785,17 @@ func (s *sessionService) handleModelFallback(ctx context.Context, chatManage *ty
go s.consumeFallbackStream(ctx, chatManage, responseChan)
}
// renderFallbackPrompt renders the fallback prompt template with Query variable
// renderFallbackPrompt renders the fallback prompt template with query and image context.
func (s *sessionService) renderFallbackPrompt(ctx context.Context, chatManage *types.ChatManage) (string, error) {
// Use simple string replacement instead of Go template
result := strings.ReplaceAll(chatManage.FallbackPrompt, "{{query}}", chatManage.Query)
query := chatManage.Query
if rq := strings.TrimSpace(chatManage.RewriteQuery); rq != "" {
query = rq
}
result := strings.ReplaceAll(chatManage.FallbackPrompt, "{{query}}", query)
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
result += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
}
return result, nil
}