feat: implement image upload and multimodal support
- Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A. - Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes. - Updated the backend to process images alongside text queries, including support for image analysis. - Introduced new UI components for image previews and management, improving user interaction with uploaded content. - Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages. These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses.
This commit is contained in:
@@ -67,8 +67,13 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
|
||||
chatMessages = append(chatMessages, chat.Message{Role: "assistant", Content: history.Answer})
|
||||
}
|
||||
|
||||
// Add current user message
|
||||
chatMessages = append(chatMessages, chat.Message{Role: "user", Content: chatManage.UserContent})
|
||||
// Add current user message. Only include images when the chat model supports
|
||||
// vision; non-vision models rely on the text description in UserContent.
|
||||
userMsg := chat.Message{Role: "user", Content: chatManage.UserContent}
|
||||
if chatManage.ChatModelSupportsVision && len(chatManage.Images) > 0 {
|
||||
userMsg.Images = chatManage.Images
|
||||
}
|
||||
chatMessages = append(chatMessages, userMsg)
|
||||
|
||||
return chatMessages
|
||||
}
|
||||
|
||||
@@ -122,6 +122,12 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
userContent = strings.ReplaceAll(userContent, "{{current_time}}", time.Now().Format("2006-01-02 15:04:05"))
|
||||
userContent = strings.ReplaceAll(userContent, "{{current_week}}", weekdayName[time.Now().Weekday()])
|
||||
|
||||
// Append image description as text fallback only when the chat model cannot
|
||||
// process images directly. Vision-capable models see images via MultiContent.
|
||||
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
|
||||
}
|
||||
|
||||
// Set formatted content back to chat management
|
||||
chatManage.UserContent = userContent
|
||||
pipelineInfo(ctx, "IntoChatMessage", "output", map[string]interface{}{
|
||||
|
||||
@@ -27,6 +27,11 @@ type PluginRewrite struct {
|
||||
// reg is a regular expression used to match and remove content between <think></think> tags
|
||||
var reg = regexp.MustCompile(`(?s)<think>.*?</think>`)
|
||||
|
||||
const (
|
||||
noSearchPrefix = "[NO_SEARCH]"
|
||||
imageDescSeparator = "\n---\n"
|
||||
)
|
||||
|
||||
// NewPluginRewrite creates a new query rewriting plugin instance
|
||||
// Also registers the plugin with the event manager
|
||||
func NewPluginRewrite(eventManager *EventManager,
|
||||
@@ -48,18 +53,24 @@ func (p *PluginRewrite) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.REWRITE_QUERY}
|
||||
}
|
||||
|
||||
// OnEvent processes triggered events
|
||||
// When receiving a REWRITE_QUERY event, it rewrites the user query using conversation history and the language model
|
||||
// OnEvent processes triggered events.
|
||||
// Handles three input combinations:
|
||||
// - Text only: standard rewrite + intent classification (uses chat model)
|
||||
// - Text + images: multimodal rewrite + intent + image description (uses VLM/vision model)
|
||||
// - Images only: multimodal analysis + intent + image description (uses VLM/vision model)
|
||||
func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
// Initialize rewritten query as original query
|
||||
chatManage.RewriteQuery = chatManage.Query
|
||||
|
||||
if !chatManage.EnableRewrite {
|
||||
hasImages := len(chatManage.Images) > 0
|
||||
needRewrite := chatManage.EnableRewrite
|
||||
// When images are present we always run the step for image analysis + intent,
|
||||
// even without history or rewrite enabled.
|
||||
if !needRewrite && !hasImages {
|
||||
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"reason": "rewrite_disabled",
|
||||
"reason": "rewrite_disabled_no_images",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
@@ -68,126 +79,53 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
"session_id": chatManage.SessionID,
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"user_query": chatManage.Query,
|
||||
"has_images": hasImages,
|
||||
"enable_rewrite": chatManage.EnableRewrite,
|
||||
})
|
||||
|
||||
// Get conversation history
|
||||
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Rewrite", "history_fetch", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
// --- Load and prepare conversation history ---
|
||||
historyList := p.loadHistory(ctx, chatManage)
|
||||
|
||||
// Convert historical messages to conversation history structure
|
||||
historyMap := make(map[string]*types.History)
|
||||
|
||||
// Process historical messages, grouped by requestID
|
||||
for _, message := range history {
|
||||
history, ok := historyMap[message.RequestID]
|
||||
if !ok {
|
||||
history = &types.History{}
|
||||
}
|
||||
if message.Role == "user" {
|
||||
// User message as query
|
||||
history.Query = message.Content
|
||||
history.CreateAt = message.CreatedAt
|
||||
} else {
|
||||
// System message as answer, while removing thinking process
|
||||
history.Answer = reg.ReplaceAllString(message.Content, "")
|
||||
history.KnowledgeReferences = message.KnowledgeReferences
|
||||
}
|
||||
historyMap[message.RequestID] = history
|
||||
}
|
||||
|
||||
// Convert to list and filter incomplete conversations
|
||||
historyList := make([]*types.History, 0)
|
||||
for _, history := range historyMap {
|
||||
if history.Answer != "" && history.Query != "" {
|
||||
historyList = append(historyList, history)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by time, keep the most recent conversations
|
||||
sort.Slice(historyList, func(i, j int) bool {
|
||||
return historyList[i].CreateAt.After(historyList[j].CreateAt)
|
||||
})
|
||||
|
||||
// Limit the number of historical records
|
||||
maxRounds := p.config.Conversation.MaxRounds
|
||||
if chatManage.MaxRounds > 0 {
|
||||
maxRounds = chatManage.MaxRounds
|
||||
}
|
||||
if len(historyList) > maxRounds {
|
||||
historyList = historyList[:maxRounds]
|
||||
}
|
||||
|
||||
// Reverse to chronological order
|
||||
slices.Reverse(historyList)
|
||||
chatManage.History = historyList
|
||||
if len(historyList) == 0 {
|
||||
// Skip if there's nothing to do: no history to rewrite AND no images to analyse
|
||||
if len(historyList) == 0 && !hasImages {
|
||||
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"reason": "empty_history",
|
||||
"reason": "empty_history_no_images",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
pipelineInfo(ctx, "Rewrite", "history_ready", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"history_rounds": len(historyList),
|
||||
"max_rounds": maxRounds,
|
||||
})
|
||||
|
||||
userPrompt := p.config.Conversation.RewritePromptUser
|
||||
if chatManage.RewritePromptUser != "" {
|
||||
userPrompt = chatManage.RewritePromptUser
|
||||
}
|
||||
systemPrompt := p.config.Conversation.RewritePromptSystem
|
||||
if chatManage.RewritePromptSystem != "" {
|
||||
systemPrompt = chatManage.RewritePromptSystem
|
||||
}
|
||||
|
||||
// Format conversation history for template
|
||||
conversationText := formatConversationHistory(historyList)
|
||||
currentTime := time.Now().Format("2006-01-02 15:04:05")
|
||||
yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
|
||||
|
||||
// Replace placeholders in prompts
|
||||
userContent := strings.ReplaceAll(userPrompt, "{{conversation}}", conversationText)
|
||||
userContent = strings.ReplaceAll(userContent, "{{query}}", chatManage.Query)
|
||||
userContent = strings.ReplaceAll(userContent, "{{current_time}}", currentTime)
|
||||
userContent = strings.ReplaceAll(userContent, "{{yesterday}}", yesterday)
|
||||
|
||||
systemContent := strings.ReplaceAll(systemPrompt, "{{conversation}}", conversationText)
|
||||
systemContent = strings.ReplaceAll(systemContent, "{{query}}", chatManage.Query)
|
||||
systemContent = strings.ReplaceAll(systemContent, "{{current_time}}", currentTime)
|
||||
systemContent = strings.ReplaceAll(systemContent, "{{yesterday}}", yesterday)
|
||||
|
||||
rewriteModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
// --- Select the appropriate model ---
|
||||
rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages)
|
||||
if rewriteModel == nil {
|
||||
pipelineError(ctx, "Rewrite", "get_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"chat_model_id": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
// Call model to rewrite query
|
||||
// --- Build prompts ---
|
||||
systemContent, userContent := p.buildPrompts(chatManage, historyList)
|
||||
|
||||
// Build user message (with images when using a vision-capable model)
|
||||
userMsg := chat.Message{Role: "user", Content: userContent}
|
||||
if useImages {
|
||||
userMsg.Images = chatManage.Images
|
||||
}
|
||||
|
||||
maxTokens := 60
|
||||
if useImages {
|
||||
maxTokens = 500
|
||||
}
|
||||
|
||||
// --- Call model ---
|
||||
thinking := false
|
||||
response, err := rewriteModel.Chat(ctx, []chat.Message{
|
||||
{
|
||||
Role: "system",
|
||||
Content: systemContent,
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: userContent,
|
||||
},
|
||||
{Role: "system", Content: systemContent},
|
||||
userMsg,
|
||||
}, &chat.ChatOptions{
|
||||
Temperature: 0.3,
|
||||
MaxCompletionTokens: 50,
|
||||
MaxCompletionTokens: maxTokens,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -198,17 +136,175 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
if response.Content != "" {
|
||||
// Update rewritten query
|
||||
chatManage.RewriteQuery = response.Content
|
||||
}
|
||||
// --- Parse structured output ---
|
||||
p.parseRewriteOutput(chatManage, response.Content)
|
||||
|
||||
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"skip_kb_search": chatManage.SkipKBSearch,
|
||||
"has_image_desc": chatManage.ImageOCRText != "",
|
||||
"original_output": response.Content,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
// loadHistory fetches and processes conversation history for rewrite context.
|
||||
func (p *PluginRewrite) loadHistory(ctx context.Context, chatManage *types.ChatManage) []*types.History {
|
||||
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Rewrite", "history_fetch", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
historyMap := make(map[string]*types.History)
|
||||
for _, message := range history {
|
||||
h, ok := historyMap[message.RequestID]
|
||||
if !ok {
|
||||
h = &types.History{}
|
||||
}
|
||||
if message.Role == "user" {
|
||||
h.Query = message.Content
|
||||
h.CreateAt = message.CreatedAt
|
||||
} else {
|
||||
h.Answer = reg.ReplaceAllString(message.Content, "")
|
||||
h.KnowledgeReferences = message.KnowledgeReferences
|
||||
}
|
||||
historyMap[message.RequestID] = h
|
||||
}
|
||||
|
||||
historyList := make([]*types.History, 0)
|
||||
for _, h := range historyMap {
|
||||
if h.Answer != "" && h.Query != "" {
|
||||
historyList = append(historyList, h)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(historyList, func(i, j int) bool {
|
||||
return historyList[i].CreateAt.After(historyList[j].CreateAt)
|
||||
})
|
||||
|
||||
maxRounds := p.config.Conversation.MaxRounds
|
||||
if chatManage.MaxRounds > 0 {
|
||||
maxRounds = chatManage.MaxRounds
|
||||
}
|
||||
if len(historyList) > maxRounds {
|
||||
historyList = historyList[:maxRounds]
|
||||
}
|
||||
|
||||
slices.Reverse(historyList)
|
||||
chatManage.History = historyList
|
||||
|
||||
if len(historyList) > 0 {
|
||||
pipelineInfo(ctx, "Rewrite", "history_ready", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"history_rounds": len(historyList),
|
||||
})
|
||||
}
|
||||
|
||||
return historyList
|
||||
}
|
||||
|
||||
// selectModel picks the model for rewrite. When images are present it prefers
|
||||
// a vision-capable model (either the chat model itself, or the agent's VLM).
|
||||
// Returns (model, useImages).
|
||||
func (p *PluginRewrite) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) {
|
||||
if hasImages {
|
||||
if chatManage.ChatModelSupportsVision {
|
||||
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err == nil {
|
||||
return m, true
|
||||
}
|
||||
pipelineWarn(ctx, "Rewrite", "vision_model_fallback", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
if chatManage.VLMModelID != "" {
|
||||
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
|
||||
if err == nil {
|
||||
return m, true
|
||||
}
|
||||
pipelineWarn(ctx, "Rewrite", "vlm_model_fallback", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"vlm_model_id": chatManage.VLMModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
pipelineWarn(ctx, "Rewrite", "no_vision_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
}
|
||||
|
||||
// Fallback: text-only rewrite with chat model
|
||||
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
pipelineError(ctx, "Rewrite", "get_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"chat_model_id": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, false
|
||||
}
|
||||
return m, false
|
||||
}
|
||||
|
||||
// buildPrompts constructs system and user prompts with placeholder replacement.
|
||||
func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList []*types.History) (string, string) {
|
||||
userPrompt := p.config.Conversation.RewritePromptUser
|
||||
if chatManage.RewritePromptUser != "" {
|
||||
userPrompt = chatManage.RewritePromptUser
|
||||
}
|
||||
systemPrompt := p.config.Conversation.RewritePromptSystem
|
||||
if chatManage.RewritePromptSystem != "" {
|
||||
systemPrompt = chatManage.RewritePromptSystem
|
||||
}
|
||||
|
||||
conversationText := formatConversationHistory(historyList)
|
||||
currentTime := time.Now().Format("2006-01-02 15:04:05")
|
||||
yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
|
||||
|
||||
replacePlaceholders := func(s string) string {
|
||||
s = strings.ReplaceAll(s, "{{conversation}}", conversationText)
|
||||
s = strings.ReplaceAll(s, "{{query}}", chatManage.Query)
|
||||
s = strings.ReplaceAll(s, "{{current_time}}", currentTime)
|
||||
s = strings.ReplaceAll(s, "{{yesterday}}", yesterday)
|
||||
return s
|
||||
}
|
||||
|
||||
return replacePlaceholders(systemPrompt), replacePlaceholders(userPrompt)
|
||||
}
|
||||
|
||||
// parseRewriteOutput extracts intent classification, rewritten query, and
|
||||
// optional image description from the model's structured output.
|
||||
//
|
||||
// Expected formats:
|
||||
//
|
||||
// Text only: "[NO_SEARCH] rewritten question" or "rewritten question"
|
||||
// With images: "[NO_SEARCH]\nrewritten question\n---\nimage description"
|
||||
func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw string) {
|
||||
content := strings.TrimSpace(raw)
|
||||
if content == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Parse intent marker
|
||||
if strings.HasPrefix(content, noSearchPrefix) {
|
||||
chatManage.SkipKBSearch = true
|
||||
content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix))
|
||||
}
|
||||
|
||||
// 2. Split rewritten query and image description
|
||||
if idx := strings.Index(content, imageDescSeparator); idx >= 0 {
|
||||
chatManage.RewriteQuery = strings.TrimSpace(content[:idx])
|
||||
chatManage.ImageOCRText = strings.TrimSpace(content[idx+len(imageDescSeparator):])
|
||||
} else if content != "" {
|
||||
chatManage.RewriteQuery = content
|
||||
}
|
||||
}
|
||||
|
||||
// formatConversationHistory formats conversation history for prompt template
|
||||
func formatConversationHistory(historyList []*types.History) string {
|
||||
if len(historyList) == 0 {
|
||||
|
||||
@@ -89,6 +89,15 @@ func (p *PluginSearchParallel) ActivationEvents() []types.EventType {
|
||||
func (p *PluginSearchParallel) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
// Intent-based skip: rewrite step determined KB retrieval is unnecessary
|
||||
if chatManage.SkipKBSearch {
|
||||
pipelineInfo(ctx, "SearchParallel", "skip", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"reason": "intent_no_search",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "SearchParallel", "start", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"has_entities": len(chatManage.Entity) > 0,
|
||||
|
||||
@@ -523,6 +523,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
eventBus *event.EventBus,
|
||||
customAgent *types.CustomAgent,
|
||||
enableMemory bool,
|
||||
imageURLs []string, imageOCRText string,
|
||||
) error {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
@@ -690,6 +691,18 @@ func (s *sessionService) KnowledgeQA(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve chat model vision capability and VLM model ID for image routing
|
||||
var chatModelSupportsVision bool
|
||||
var vlmModelID string
|
||||
if chatModelID != "" {
|
||||
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
|
||||
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
|
||||
}
|
||||
}
|
||||
if customAgent != nil {
|
||||
vlmModelID = customAgent.Config.VLMModelID
|
||||
}
|
||||
|
||||
// Retrieval scope: when agent is set, use agent's tenant (own or shared); otherwise session tenant or context
|
||||
retrievalTenantID := session.TenantID
|
||||
if customAgent != nil && customAgent.TenantID != 0 {
|
||||
@@ -754,6 +767,11 @@ func (s *sessionService) KnowledgeQA(
|
||||
FAQPriorityEnabled: faqPriorityEnabled,
|
||||
FAQDirectAnswerThreshold: faqDirectAnswerThreshold,
|
||||
FAQScoreBoost: faqScoreBoost,
|
||||
// Image support
|
||||
Images: imageURLs,
|
||||
ImageOCRText: imageOCRText,
|
||||
VLMModelID: vlmModelID,
|
||||
ChatModelSupportsVision: chatModelSupportsVision,
|
||||
}
|
||||
|
||||
// Determine pipeline based on knowledge bases availability and web search setting
|
||||
@@ -763,7 +781,12 @@ func (s *sessionService) KnowledgeQA(
|
||||
if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) == 0 && !webSearchEnabled {
|
||||
logger.Info(ctx, "No knowledge bases selected and web search disabled, using chat pipeline")
|
||||
// For pure chat, UserContent is the Query (since INTO_CHAT_MESSAGE is skipped)
|
||||
chatManage.UserContent = query
|
||||
// Only append image text description for non-vision models; vision models see images directly
|
||||
userContent := query
|
||||
if imageOCRText != "" && !chatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + imageOCRText
|
||||
}
|
||||
chatManage.UserContent = userContent
|
||||
|
||||
// Use chat_history_stream if multi-turn is enabled, otherwise use chat_stream
|
||||
if maxRounds > 0 {
|
||||
@@ -1368,6 +1391,7 @@ func (s *sessionService) AgentQA(
|
||||
customAgent *types.CustomAgent,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
imageURLs []string, imageOCRText string,
|
||||
) error {
|
||||
sessionID := session.ID
|
||||
sessionJSON, err := json.Marshal(session)
|
||||
@@ -1587,10 +1611,28 @@ func (s *sessionService) AgentQA(
|
||||
return err
|
||||
}
|
||||
|
||||
// Route image data based on agent model's vision capability
|
||||
var agentModelSupportsVision bool
|
||||
if effectiveModelID != "" {
|
||||
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
|
||||
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
|
||||
}
|
||||
}
|
||||
|
||||
agentQuery := query
|
||||
var agentImageURLs []string
|
||||
if agentModelSupportsVision && len(imageURLs) > 0 {
|
||||
agentImageURLs = imageURLs
|
||||
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
|
||||
} else if imageOCRText != "" {
|
||||
agentQuery = query + "\n\n[用户上传图片内容]\n" + imageOCRText
|
||||
logger.Infof(ctx, "Agent model does not support vision, appending image OCR text (%d chars)", len(imageOCRText))
|
||||
}
|
||||
|
||||
// Execute agent with streaming (asynchronously)
|
||||
// Events will be emitted to EventBus and handled by the Handler layer
|
||||
logger.Info(ctx, "Executing agent with streaming")
|
||||
if _, err := engine.Execute(ctx, sessionID, assistantMessageID, query, llmContext); err != nil {
|
||||
if _, err := engine.Execute(ctx, sessionID, assistantMessageID, agentQuery, llmContext, agentImageURLs); err != nil {
|
||||
logger.Errorf(ctx, "Agent execution failed: %v", err)
|
||||
// Emit error event to the EventBus used by this agent
|
||||
eventBus.Emit(ctx, event.Event{
|
||||
|
||||
Reference in New Issue
Block a user