feat: enhance multimodal image processing and session management
- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images. - Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history. - Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users. - Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content. These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
This commit is contained in:
@@ -235,6 +235,16 @@ func (r *messageRepository) GetKnowledgeIDsBySessionID(
|
||||
return knowledgeIDs, nil
|
||||
}
|
||||
|
||||
// UpdateMessageImages updates only the images JSONB column for a message.
|
||||
// Uses Select to force GORM to include the column even when struct-based
|
||||
// Updates would otherwise skip custom Valuer types.
|
||||
func (r *messageRepository) UpdateMessageImages(ctx context.Context, sessionID, messageID string, images types.MessageImages) error {
|
||||
return r.db.WithContext(ctx).
|
||||
Model(&types.Message{}).
|
||||
Where("id = ? AND session_id = ?", messageID, sessionID).
|
||||
Update("images", images).Error
|
||||
}
|
||||
|
||||
// UpdateMessageKnowledgeID updates the knowledge_id field for a message
|
||||
func (r *messageRepository) UpdateMessageKnowledgeID(
|
||||
ctx context.Context, messageID string, knowledgeID string,
|
||||
|
||||
@@ -43,7 +43,7 @@ func (r *sessionRepository) Get(ctx context.Context, tenantID uint64, id string)
|
||||
// GetByTenantID retrieves all sessions for a tenant
|
||||
func (r *sessionRepository) GetByTenantID(ctx context.Context, tenantID uint64) ([]*types.Session, error) {
|
||||
var sessions []*types.Session
|
||||
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&sessions).Error
|
||||
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("updated_at DESC").Find(&sessions).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func (r *sessionRepository) GetPagedByTenantID(
|
||||
// Then query the paginated data
|
||||
err = r.db.WithContext(ctx).
|
||||
Where("tenant_id = ?", tenantID).
|
||||
Order("created_at DESC").
|
||||
Order("updated_at DESC").
|
||||
Offset(page.Offset()).
|
||||
Limit(page.Limit()).
|
||||
Find(&sessions).Error
|
||||
@@ -80,7 +80,14 @@ func (r *sessionRepository) GetPagedByTenantID(
|
||||
// Update updates a session
|
||||
func (r *sessionRepository) Update(ctx context.Context, session *types.Session) error {
|
||||
session.UpdatedAt = time.Now()
|
||||
return r.db.WithContext(ctx).Where("tenant_id = ?", session.TenantID).Save(session).Error
|
||||
return r.db.WithContext(ctx).
|
||||
Model(&types.Session{}).
|
||||
Where("tenant_id = ? AND id = ?", session.TenantID, session.ID).
|
||||
Updates(map[string]interface{}{
|
||||
"title": session.Title,
|
||||
"description": session.Description,
|
||||
"updated_at": session.UpdatedAt,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// Delete deletes a session
|
||||
|
||||
@@ -78,6 +78,19 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
|
||||
return chatMessages
|
||||
}
|
||||
|
||||
// extractImageCaptions concatenates non-empty Caption fields from stored
|
||||
// message images. Used when loading history so that previous turns' image
|
||||
// descriptions are visible to the model.
|
||||
func extractImageCaptions(images types.MessageImages) string {
|
||||
var parts []string
|
||||
for _, img := range images {
|
||||
if img.Caption != "" {
|
||||
parts = append(parts, img.Caption)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// renderSystemPromptPlaceholders replaces placeholders in system prompt
|
||||
// Supported placeholders:
|
||||
// - {{current_time}} -> current time in RFC3339 format
|
||||
|
||||
@@ -90,8 +90,8 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
})
|
||||
}
|
||||
}
|
||||
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
|
||||
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
|
||||
}
|
||||
chatManage.UserContent = userContent
|
||||
pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{
|
||||
@@ -148,16 +148,19 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
|
||||
// Append image description as text fallback only when the chat model cannot
|
||||
// process images directly. Vision-capable models see images via MultiContent.
|
||||
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
|
||||
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
|
||||
}
|
||||
|
||||
// Set formatted content back to chat management
|
||||
chatManage.UserContent = userContent
|
||||
pipelineInfo(ctx, "IntoChatMessage", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_content_len": len(chatManage.UserContent),
|
||||
"faq_priority": chatManage.FAQPriorityEnabled,
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_content_len": len(chatManage.UserContent),
|
||||
"faq_priority": chatManage.FAQPriorityEnabled,
|
||||
"skip_kb_search": chatManage.SkipKBSearch,
|
||||
"image_description": chatManage.ImageDescription,
|
||||
"chat_model_supports_vision": chatManage.ChatModelSupportsVision,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -83,9 +83,11 @@ func (p *PluginLoadHistory) OnEvent(ctx context.Context,
|
||||
h = &types.History{}
|
||||
}
|
||||
if message.Role == "user" {
|
||||
// User message as query
|
||||
h.Query = message.Content
|
||||
h.CreateAt = message.CreatedAt
|
||||
if desc := extractImageCaptions(message.Images); desc != "" {
|
||||
h.Query += "\n\n[用户上传图片内容]\n" + desc
|
||||
}
|
||||
} else {
|
||||
// System message as answer, while removing thinking process
|
||||
h.Answer = regThink.ReplaceAllString(message.Content, "")
|
||||
|
||||
@@ -4,6 +4,7 @@ package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -34,6 +35,12 @@ const (
|
||||
noSearchPrefix = "[NO_SEARCH]"
|
||||
)
|
||||
|
||||
type rewriteOutput struct {
|
||||
RewriteQuery string
|
||||
SkipKBSearch bool
|
||||
ImageDescription string
|
||||
}
|
||||
|
||||
// NewPluginRewrite creates a new query rewriting plugin instance
|
||||
// Also registers the plugin with the event manager
|
||||
func NewPluginRewrite(eventManager *EventManager,
|
||||
@@ -183,16 +190,53 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
|
||||
// --- Parse structured output ---
|
||||
p.parseRewriteOutput(chatManage, response.Content)
|
||||
|
||||
// Persist image description back to the user message so that future turns
|
||||
// can see it when loading conversation history.
|
||||
if chatManage.ImageDescription != "" && chatManage.UserMessageID != "" {
|
||||
p.updateUserMessageImageCaption(ctx, chatManage)
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"skip_kb_search": chatManage.SkipKBSearch,
|
||||
"has_image_desc": chatManage.ImageOCRText != "",
|
||||
"has_image_desc": chatManage.ImageDescription != "",
|
||||
"original_output": response.Content,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
// updateUserMessageImageCaption writes the generated ImageDescription back to
|
||||
// the stored user message's Images so that subsequent turns can see it in history.
|
||||
func (p *PluginRewrite) updateUserMessageImageCaption(ctx context.Context, chatManage *types.ChatManage) {
|
||||
msg, err := p.messageService.GetMessage(ctx, chatManage.SessionID, chatManage.UserMessageID)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Rewrite", "get_user_message", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_message_id": chatManage.UserMessageID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if len(msg.Images) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
msg.Images[0].Caption = chatManage.ImageDescription
|
||||
|
||||
// Use the targeted UpdateMessageImages to reliably persist the JSONB column.
|
||||
// GORM's struct-based Updates may silently skip custom Valuer types.
|
||||
if err := p.messageService.UpdateMessageImages(ctx, chatManage.SessionID, chatManage.UserMessageID, msg.Images); err != nil {
|
||||
pipelineWarn(ctx, "Rewrite", "update_image_caption", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_message_id": chatManage.UserMessageID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// loadHistory fetches and processes conversation history for rewrite context.
|
||||
func (p *PluginRewrite) loadHistory(ctx context.Context, chatManage *types.ChatManage) []*types.History {
|
||||
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
|
||||
@@ -212,6 +256,9 @@ func (p *PluginRewrite) loadHistory(ctx context.Context, chatManage *types.ChatM
|
||||
if message.Role == "user" {
|
||||
h.Query = message.Content
|
||||
h.CreateAt = message.CreatedAt
|
||||
if desc := extractImageCaptions(message.Images); desc != "" {
|
||||
h.Query += "\n\n[用户上传图片内容]\n" + desc
|
||||
}
|
||||
} else {
|
||||
h.Answer = reg.ReplaceAllString(message.Content, "")
|
||||
h.KnowledgeReferences = message.KnowledgeReferences
|
||||
@@ -307,11 +354,14 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
|
||||
}
|
||||
// Strengthen context inheritance in multi-turn conversation:
|
||||
// for follow-up questions that clearly refer to previous turns (especially
|
||||
// uploaded-image understanding), prefer NO_SEARCH over KB retrieval.
|
||||
// uploaded-image understanding), prefer skipping KB retrieval.
|
||||
systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" +
|
||||
"- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" +
|
||||
"- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST set skip_kb_search=true.\n" +
|
||||
"- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" +
|
||||
"- In these follow-up cases, output with [NO_SEARCH] prefix."
|
||||
"- You MUST output ONLY a single JSON object (no markdown/code fences, no extra text) with schema: {\"rewrite_query\":\"...\",\"skip_kb_search\":true|false,\"image_description\":\"...\"}.\n" +
|
||||
"- `skip_kb_search` field is REQUIRED and must be explicit true/false.\n" +
|
||||
"- For images, `image_description` MUST include a detailed visual description and complete OCR text. Keep all key details; do not overly summarize.\n" +
|
||||
"- image_description should be empty string only when there are no images or truly no readable visual text/details."
|
||||
|
||||
conversationText := formatConversationHistory(historyList)
|
||||
currentTime := time.Now().Format("2006-01-02 15:04:05")
|
||||
@@ -333,27 +383,34 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
|
||||
//
|
||||
// Expected formats:
|
||||
//
|
||||
// Text only: "[NO_SEARCH] rewritten question" or "rewritten question"
|
||||
// With images: "[NO_SEARCH]\nrewritten question\n---\nimage description"
|
||||
// Preferred: {"rewrite_query":"...","skip_kb_search":false,"image_description":"..."}
|
||||
// Legacy fallback:
|
||||
// - Text only: "[NO_SEARCH] rewritten question" or "rewritten question"
|
||||
// - With images: "[NO_SEARCH]\nrewritten question\n---\nimage description"
|
||||
func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw string) {
|
||||
content := strings.TrimSpace(raw)
|
||||
if content == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Parse intent marker
|
||||
if output, ok := parseStructuredRewriteOutput(content); ok {
|
||||
if rewrite := strings.TrimSpace(output.RewriteQuery); rewrite != "" {
|
||||
chatManage.RewriteQuery = rewrite
|
||||
}
|
||||
chatManage.SkipKBSearch = output.SkipKBSearch
|
||||
chatManage.ImageDescription = strings.TrimSpace(output.ImageDescription)
|
||||
return
|
||||
}
|
||||
|
||||
// Legacy fallback parsing for older prompts/models.
|
||||
if strings.HasPrefix(content, noSearchPrefix) {
|
||||
chatManage.SkipKBSearch = true
|
||||
content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix))
|
||||
}
|
||||
|
||||
// 2. Split rewritten query and image description.
|
||||
// Be tolerant to model output variants like:
|
||||
// - "query\n---\nimage_desc" (expected)
|
||||
// - "query---\nimage_desc" (missing leading newline before separator)
|
||||
if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 {
|
||||
chatManage.RewriteQuery = strings.TrimSpace(m[1])
|
||||
chatManage.ImageOCRText = strings.TrimSpace(m[2])
|
||||
chatManage.ImageDescription = strings.TrimSpace(m[2])
|
||||
return
|
||||
}
|
||||
if content != "" {
|
||||
@@ -361,6 +418,128 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str
|
||||
}
|
||||
}
|
||||
|
||||
func parseStructuredRewriteOutput(raw string) (rewriteOutput, bool) {
|
||||
content := strings.TrimSpace(raw)
|
||||
if content == "" {
|
||||
return rewriteOutput{}, false
|
||||
}
|
||||
|
||||
var out rewriteOutput
|
||||
if parsed, ok := parseStructuredRewriteOutputJSON(content); ok {
|
||||
return parsed, true
|
||||
}
|
||||
|
||||
// Be tolerant to occasional markdown wrappers or extra prose.
|
||||
start := strings.Index(content, "{")
|
||||
end := strings.LastIndex(content, "}")
|
||||
if start < 0 || end <= start {
|
||||
return rewriteOutput{}, false
|
||||
}
|
||||
candidate := content[start : end+1]
|
||||
if parsed, ok := parseStructuredRewriteOutputJSON(candidate); ok {
|
||||
return parsed, true
|
||||
}
|
||||
return out, false
|
||||
}
|
||||
|
||||
func parseStructuredRewriteOutputJSON(content string) (rewriteOutput, bool) {
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(content), &obj); err != nil {
|
||||
return rewriteOutput{}, false
|
||||
}
|
||||
|
||||
out := rewriteOutput{
|
||||
RewriteQuery: strings.TrimSpace(firstStringField(obj,
|
||||
"rewrite_query", "rewritten_query", "query", "question")),
|
||||
}
|
||||
|
||||
// Support common variants and semantic inversion for need_search.
|
||||
if v, ok := firstBoolField(obj, "skip_kb_search", "skip_search", "no_search"); ok {
|
||||
out.SkipKBSearch = v
|
||||
} else if v, ok := firstBoolField(obj, "need_search", "requires_search"); ok {
|
||||
out.SkipKBSearch = !v
|
||||
}
|
||||
|
||||
desc := strings.TrimSpace(firstStringField(obj,
|
||||
"image_description", "image_desc", "image_text", "image_ocr_text", "description"))
|
||||
ocr := strings.TrimSpace(firstStringField(obj,
|
||||
"ocr_text", "ocr", "full_ocr", "image_ocr", "ocr_content"))
|
||||
combined, set := mergeImageDescAndOCR(desc, ocr)
|
||||
if set {
|
||||
out.ImageDescription = combined
|
||||
}
|
||||
|
||||
return out, true
|
||||
}
|
||||
|
||||
func firstStringField(obj map[string]json.RawMessage, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
raw, ok := obj[key]
|
||||
if !ok || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err == nil {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstBoolField(obj map[string]json.RawMessage, keys ...string) (bool, bool) {
|
||||
for _, key := range keys {
|
||||
raw, ok := obj[key]
|
||||
if !ok || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
if v, ok := parseBoolJSON(raw); ok {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func parseBoolJSON(raw json.RawMessage) (bool, bool) {
|
||||
var b bool
|
||||
if err := json.Unmarshal(raw, &b); err == nil {
|
||||
return b, true
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err == nil {
|
||||
switch strings.ToLower(strings.TrimSpace(s)) {
|
||||
case "true", "1", "yes", "y":
|
||||
return true, true
|
||||
case "false", "0", "no", "n":
|
||||
return false, true
|
||||
}
|
||||
}
|
||||
|
||||
var n float64
|
||||
if err := json.Unmarshal(raw, &n); err == nil {
|
||||
return n != 0, true
|
||||
}
|
||||
|
||||
return false, false
|
||||
}
|
||||
|
||||
func mergeImageDescAndOCR(desc, ocr string) (string, bool) {
|
||||
if desc == "" && ocr == "" {
|
||||
return "", false
|
||||
}
|
||||
if desc == "" {
|
||||
return ocr, true
|
||||
}
|
||||
if ocr == "" {
|
||||
return desc, true
|
||||
}
|
||||
if strings.Contains(desc, ocr) {
|
||||
return desc, true
|
||||
}
|
||||
return desc + "\n\n[OCR]\n" + ocr, true
|
||||
}
|
||||
|
||||
// formatConversationHistory formats conversation history for prompt template
|
||||
func formatConversationHistory(historyList []*types.History) string {
|
||||
if len(historyList) == 0 {
|
||||
|
||||
@@ -237,6 +237,11 @@ func (s *messageService) UpdateMessage(ctx context.Context, message *types.Messa
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMessageImages updates only the images JSONB column for a message.
|
||||
func (s *messageService) UpdateMessageImages(ctx context.Context, sessionID, messageID string, images types.MessageImages) error {
|
||||
return s.messageRepo.UpdateMessageImages(ctx, sessionID, messageID, images)
|
||||
}
|
||||
|
||||
// DeleteMessage removes a message from a session, also cleaning up its Knowledge entry in the chat history KB.
|
||||
func (s *messageService) DeleteMessage(ctx context.Context, sessionID string, messageID string) error {
|
||||
logger.Info(ctx, "Start deleting message")
|
||||
|
||||
@@ -523,7 +523,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
eventBus *event.EventBus,
|
||||
customAgent *types.CustomAgent,
|
||||
enableMemory bool,
|
||||
imageURLs []string, imageOCRText string,
|
||||
imageURLs []string, imageDescription string, userMessageID string,
|
||||
) error {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
@@ -768,8 +768,9 @@ func (s *sessionService) KnowledgeQA(
|
||||
FAQDirectAnswerThreshold: faqDirectAnswerThreshold,
|
||||
FAQScoreBoost: faqScoreBoost,
|
||||
// Image support
|
||||
UserMessageID: userMessageID,
|
||||
Images: imageURLs,
|
||||
ImageOCRText: imageOCRText,
|
||||
ImageDescription: imageDescription,
|
||||
VLMModelID: vlmModelID,
|
||||
ChatModelSupportsVision: chatModelSupportsVision,
|
||||
}
|
||||
@@ -783,8 +784,8 @@ func (s *sessionService) KnowledgeQA(
|
||||
// For pure chat, UserContent is the Query (since INTO_CHAT_MESSAGE is skipped)
|
||||
// Only append image text description for non-vision models; vision models see images directly
|
||||
userContent := query
|
||||
if imageOCRText != "" && !chatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + imageOCRText
|
||||
if imageDescription != "" && !chatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + imageDescription
|
||||
}
|
||||
chatManage.UserContent = userContent
|
||||
|
||||
@@ -1391,8 +1392,9 @@ func (s *sessionService) AgentQA(
|
||||
customAgent *types.CustomAgent,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
imageURLs []string, imageOCRText string,
|
||||
imageURLs []string, imageDescription string, userMessageID string,
|
||||
) error {
|
||||
_ = userMessageID // reserved for future use (AgentQA pipeline differs from KnowledgeQA)
|
||||
sessionID := session.ID
|
||||
sessionJSON, err := json.Marshal(session)
|
||||
if err != nil {
|
||||
@@ -1624,9 +1626,9 @@ func (s *sessionService) AgentQA(
|
||||
if agentModelSupportsVision && len(imageURLs) > 0 {
|
||||
agentImageURLs = imageURLs
|
||||
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
|
||||
} else if imageOCRText != "" {
|
||||
agentQuery = query + "\n\n[用户上传图片内容]\n" + imageOCRText
|
||||
logger.Infof(ctx, "Agent model does not support vision, appending image OCR text (%d chars)", len(imageOCRText))
|
||||
} else if imageDescription != "" {
|
||||
agentQuery = query + "\n\n[用户上传图片内容]\n" + imageDescription
|
||||
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(imageDescription))
|
||||
}
|
||||
|
||||
// Execute agent with streaming (asynchronously)
|
||||
@@ -1762,9 +1764,11 @@ func (s *sessionService) handleModelFallback(ctx context.Context, chatManage *ty
|
||||
}
|
||||
|
||||
// Start streaming response
|
||||
responseChan, err := chatModel.ChatStream(ctx, []chat.Message{
|
||||
{Role: "user", Content: promptContent},
|
||||
}, opt)
|
||||
userMsg := chat.Message{Role: "user", Content: promptContent}
|
||||
if chatManage.ChatModelSupportsVision && len(chatManage.Images) > 0 {
|
||||
userMsg.Images = chatManage.Images
|
||||
}
|
||||
responseChan, err := chatModel.ChatStream(ctx, []chat.Message{userMsg}, opt)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to start streaming fallback response: %v, falling back to fixed response", err)
|
||||
s.handleFixedFallback(ctx, chatManage)
|
||||
@@ -1781,10 +1785,17 @@ func (s *sessionService) handleModelFallback(ctx context.Context, chatManage *ty
|
||||
go s.consumeFallbackStream(ctx, chatManage, responseChan)
|
||||
}
|
||||
|
||||
// renderFallbackPrompt renders the fallback prompt template with Query variable
|
||||
// renderFallbackPrompt renders the fallback prompt template with query and image context.
|
||||
func (s *sessionService) renderFallbackPrompt(ctx context.Context, chatManage *types.ChatManage) (string, error) {
|
||||
// Use simple string replacement instead of Go template
|
||||
result := strings.ReplaceAll(chatManage.FallbackPrompt, "{{query}}", chatManage.Query)
|
||||
query := chatManage.Query
|
||||
if rq := strings.TrimSpace(chatManage.RewriteQuery); rq != "" {
|
||||
query = rq
|
||||
}
|
||||
result := strings.ReplaceAll(chatManage.FallbackPrompt, "{{query}}", query)
|
||||
|
||||
if chatManage.ImageDescription != "" && !chatManage.ChatModelSupportsVision {
|
||||
result += "\n\n[用户上传图片内容]\n" + chatManage.ImageDescription
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user