feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -79,8 +79,10 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
})
|
||||
responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt)
|
||||
streamCtx, streamCancel := context.WithCancel(ctx)
|
||||
responseChan, err := chatModel.ChatStream(streamCtx, chatMessages, opt)
|
||||
if err != nil {
|
||||
streamCancel()
|
||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
@@ -88,6 +90,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
return ErrModelCall.WithError(err)
|
||||
}
|
||||
if responseChan == nil {
|
||||
streamCancel()
|
||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
"error": "nil_channel",
|
||||
@@ -105,6 +108,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
// The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled
|
||||
// and the upstream channel is not closed promptly.
|
||||
go func() {
|
||||
defer streamCancel()
|
||||
thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8])
|
||||
answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8])
|
||||
thinkingOpen := false
|
||||
@@ -145,19 +149,20 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
if response.ResponseType == types.ResponseTypeError {
|
||||
pipelineError(ctx, "Stream", "stream_error", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": response.Content,
|
||||
"error": "chat completion stream failed",
|
||||
})
|
||||
eventBus.Emit(ctx, types.Event{
|
||||
ID: fmt.Sprintf("%s-error", uuid.New().String()[:8]),
|
||||
Type: types.EventType(event.EventError),
|
||||
SessionID: chatManage.SessionID,
|
||||
Data: event.ErrorData{
|
||||
Error: response.Content,
|
||||
Error: "chat completion stream failed",
|
||||
Stage: "chat_completion_stream",
|
||||
SessionID: chatManage.SessionID,
|
||||
},
|
||||
})
|
||||
continue
|
||||
closeThinking()
|
||||
return
|
||||
}
|
||||
|
||||
if response.ResponseType == types.ResponseTypeThinking {
|
||||
@@ -190,6 +195,9 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
Done: response.Done,
|
||||
},
|
||||
})
|
||||
if response.Done {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,14 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
|
||||
}
|
||||
|
||||
// --- Select the appropriate model ---
|
||||
rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages)
|
||||
rewriteModel, useImages, selectErr := p.selectModel(ctx, chatManage, hasImages)
|
||||
if selectErr != nil {
|
||||
pipelineError(ctx, "QueryUnderstand", "get_visual_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": selectErr.Error(),
|
||||
})
|
||||
return &PluginError{Err: selectErr, Description: "Visual model is unavailable", ErrorType: "image_understanding_failed"}
|
||||
}
|
||||
if rewriteModel == nil {
|
||||
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
@@ -127,11 +134,18 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
if hasImages {
|
||||
return &PluginError{Err: err, Description: "Visual model request failed", ErrorType: "image_understanding_failed"}
|
||||
}
|
||||
return next()
|
||||
}
|
||||
|
||||
// --- Parse structured output ---
|
||||
p.parseOutput(chatManage, response.Content)
|
||||
if hasImages && !chatManage.ChatModelSupportsVision && strings.TrimSpace(chatManage.ImageDescription) == "" {
|
||||
err := fmt.Errorf("visual model returned no image description")
|
||||
return &PluginError{Err: err, Description: "Visual model returned no usable image description", ErrorType: "image_understanding_failed"}
|
||||
}
|
||||
|
||||
// Persist image description asynchronously — this DB write does not affect
|
||||
// the current pipeline result, so it can run in the background.
|
||||
@@ -222,33 +236,43 @@ func (p *PluginQueryUnderstand) loadHistory(ctx context.Context, chatManage *typ
|
||||
}
|
||||
|
||||
// selectModel picks the model for query understanding. When images are present
|
||||
// it prefers a vision-capable model. Returns (model, useImages).
|
||||
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) {
|
||||
// it requires a vision-capable model and never falls back to text-only processing.
|
||||
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool, error) {
|
||||
if hasImages {
|
||||
var lastErr error
|
||||
if chatManage.ChatModelSupportsVision {
|
||||
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err == nil {
|
||||
return m, true
|
||||
if err == nil && chat.SupportsImageInput(m) {
|
||||
return m, true, nil
|
||||
}
|
||||
pipelineWarn(ctx, "QueryUnderstand", "vision_model_fallback", map[string]interface{}{
|
||||
if err == nil {
|
||||
err = fmt.Errorf("configured Chat transport does not support image input")
|
||||
}
|
||||
lastErr = err
|
||||
pipelineWarn(ctx, "QueryUnderstand", "vision_model_unavailable", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
if chatManage.VLMModelID != "" {
|
||||
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
|
||||
if err == nil {
|
||||
return m, true
|
||||
if err == nil && chat.SupportsImageInput(m) {
|
||||
return m, true, nil
|
||||
}
|
||||
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_fallback", map[string]interface{}{
|
||||
if err == nil {
|
||||
err = fmt.Errorf("configured VLM transport does not support image input")
|
||||
}
|
||||
lastErr = err
|
||||
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_unavailable", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"vlm_model_id": chatManage.VLMModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
pipelineWarn(ctx, "QueryUnderstand", "no_vision_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
if lastErr != nil {
|
||||
return nil, false, fmt.Errorf("no usable visual model: %w", lastErr)
|
||||
}
|
||||
return nil, false, fmt.Errorf("no visual model is configured")
|
||||
}
|
||||
|
||||
textModelID := chatManage.ChatModelID
|
||||
@@ -266,7 +290,7 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
|
||||
"error": err.Error(),
|
||||
})
|
||||
if fallback, fbErr := p.modelService.GetChatModel(ctx, chatManage.ChatModelID); fbErr == nil {
|
||||
return fallback, false
|
||||
return fallback, false, nil
|
||||
}
|
||||
}
|
||||
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
||||
@@ -274,9 +298,9 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
|
||||
"chat_model_id": textModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, false
|
||||
return nil, false, nil
|
||||
}
|
||||
return m, false
|
||||
return m, false, nil
|
||||
}
|
||||
|
||||
// buildPrompts constructs system and user prompts with placeholder replacement.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package chatpipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
modelchat "github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
type visionModelStub struct{}
|
||||
|
||||
func (visionModelStub) Chat(context.Context, []modelchat.Message, *modelchat.ChatOptions) (*types.ChatResponse, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (visionModelStub) ChatStream(context.Context, []modelchat.Message, *modelchat.ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (visionModelStub) GetModelName() string { return "test-vlm" }
|
||||
func (visionModelStub) GetModelID() string { return "test-vlm" }
|
||||
func (visionModelStub) SupportsImageInput() bool { return true }
|
||||
|
||||
type modelServiceStub struct {
|
||||
interfaces.ModelService
|
||||
model modelchat.Chat
|
||||
}
|
||||
|
||||
func (s modelServiceStub) GetChatModel(context.Context, string) (modelchat.Chat, error) {
|
||||
return s.model, nil
|
||||
}
|
||||
|
||||
func TestSelectModelFailsClosedWhenImageHasNoVisualModel(t *testing.T) {
|
||||
plugin := &PluginQueryUnderstand{}
|
||||
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{ChatModelSupportsVision: false}}
|
||||
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
|
||||
if err == nil || model != nil || useImages {
|
||||
t.Fatalf("selectModel = model %v useImages %v err %v; want fail closed", model, useImages, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectModelUsesVLMWhenChatTransportIsNotVisual(t *testing.T) {
|
||||
vlm := visionModelStub{}
|
||||
plugin := &PluginQueryUnderstand{modelService: modelServiceStub{model: vlm}}
|
||||
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{
|
||||
ChatModelSupportsVision: false,
|
||||
VLMModelID: "document-parser-vlm",
|
||||
}}
|
||||
|
||||
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
|
||||
if err != nil || model == nil || !useImages {
|
||||
t.Fatalf("selectModel = model %v useImages %v err %v; want visual VLM fallback", model, useImages, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
const maxAgentFallbackImageBytes = 10 << 20
|
||||
|
||||
// prepareAgentImageInput routes image input according to the resolved primary
|
||||
// transport capability. A configured VLM is required only when the actual
|
||||
// primary Chat transport cannot carry images directly.
|
||||
func prepareAgentImageInput(
|
||||
ctx context.Context,
|
||||
req *types.QARequest,
|
||||
primarySupportsVision bool,
|
||||
modelService interfaces.ModelService,
|
||||
) (string, []string, error) {
|
||||
if req == nil {
|
||||
return "", nil, errors.New("QA request is required")
|
||||
}
|
||||
if len(req.ImageURLs) == 0 || primarySupportsVision || strings.TrimSpace(req.ImageDescription) != "" {
|
||||
return routeAgentImageInput(req, primarySupportsVision)
|
||||
}
|
||||
if req.CustomAgent == nil || strings.TrimSpace(req.CustomAgent.Config.VLMModelID) == "" {
|
||||
return "", nil, errors.New("agent model does not support visual input and no VLM is configured")
|
||||
}
|
||||
if modelService == nil {
|
||||
return "", nil, errors.New("agent VLM service is unavailable")
|
||||
}
|
||||
|
||||
images := make([][]byte, 0, len(req.ImageURLs))
|
||||
for _, raw := range req.ImageURLs {
|
||||
comma := strings.IndexByte(raw, ',')
|
||||
if comma <= 0 || !strings.HasPrefix(strings.ToLower(raw[:comma]), "data:image/") ||
|
||||
!strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
|
||||
return "", nil, errors.New("agent image payload is invalid")
|
||||
}
|
||||
decoded, err := base64.StdEncoding.Strict().DecodeString(raw[comma+1:])
|
||||
if err != nil || len(decoded) == 0 || len(decoded) > maxAgentFallbackImageBytes {
|
||||
return "", nil, errors.New("agent image payload is invalid")
|
||||
}
|
||||
images = append(images, decoded)
|
||||
}
|
||||
|
||||
model, err := modelService.GetVLMModel(ctx, req.CustomAgent.Config.VLMModelID)
|
||||
if err != nil || model == nil {
|
||||
return "", nil, errors.New("agent VLM is unavailable")
|
||||
}
|
||||
prompt := fmt.Sprintf("Analyze the uploaded image(s) for the user's question. Return a concise factual description only.\nQuestion: %s", req.Query)
|
||||
description, err := model.Predict(ctx, images, prompt)
|
||||
if err != nil || strings.TrimSpace(description) == "" {
|
||||
return "", nil, errors.New("agent VLM analysis failed")
|
||||
}
|
||||
req.ImageDescription = strings.TrimSpace(description)
|
||||
return routeAgentImageInput(req, false)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestRouteAgentImageInputFailsClosedWithoutVision(t *testing.T) {
|
||||
req := &types.QARequest{Query: "what is this", ImageURLs: []string{"data:image/png;base64,abc"}}
|
||||
if _, images, err := routeAgentImageInput(req, false); err == nil || len(images) != 0 {
|
||||
t.Fatalf("routeAgentImageInput should reject unsupported visual input: images=%v err=%v", images, err)
|
||||
}
|
||||
|
||||
query, images, err := routeAgentImageInput(req, true)
|
||||
if err != nil || len(images) != 1 || query != req.Query {
|
||||
t.Fatalf("vision route = query %q images %d err %v", query, len(images), err)
|
||||
}
|
||||
|
||||
req.ImageDescription = "a blue robot"
|
||||
query, images, err = routeAgentImageInput(req, false)
|
||||
if err != nil || len(images) != 0 || !strings.Contains(query, req.ImageDescription) {
|
||||
t.Fatalf("description fallback = query %q images %d err %v", query, len(images), err)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent/tools"
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
@@ -156,21 +157,18 @@ func (s *sessionService) AgentQA(
|
||||
return err
|
||||
}
|
||||
|
||||
// Route image data based on agent model's vision capability
|
||||
var agentModelSupportsVision bool
|
||||
if effectiveModelID != "" {
|
||||
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
|
||||
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
|
||||
}
|
||||
}
|
||||
// The resolved concrete transport is authoritative. Stored model metadata can
|
||||
// be stale (notably older WeKnora Cloud records), while unknown transports
|
||||
// still fail closed through SupportsImageInput.
|
||||
agentModelSupportsVision := modelSupportsDirectImages(summaryModel)
|
||||
|
||||
agentQuery := req.Query
|
||||
var agentImageURLs []string
|
||||
if agentModelSupportsVision && len(req.ImageURLs) > 0 {
|
||||
agentImageURLs = req.ImageURLs
|
||||
agentQuery, agentImageURLs, err := prepareAgentImageInput(ctx, req, agentModelSupportsVision, s.modelService)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(agentImageURLs) > 0 {
|
||||
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
|
||||
} else if req.ImageDescription != "" {
|
||||
agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
|
||||
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
|
||||
}
|
||||
if req.QuotedContext != "" {
|
||||
@@ -208,6 +206,22 @@ func (s *sessionService) AgentQA(
|
||||
return nil
|
||||
}
|
||||
|
||||
func routeAgentImageInput(req *types.QARequest, modelSupportsVision bool) (string, []string, error) {
|
||||
if req == nil {
|
||||
return "", nil, errors.New("QA request is required")
|
||||
}
|
||||
if len(req.ImageURLs) == 0 {
|
||||
return req.Query, nil, nil
|
||||
}
|
||||
if modelSupportsVision {
|
||||
return req.Query, req.ImageURLs, nil
|
||||
}
|
||||
if strings.TrimSpace(req.ImageDescription) != "" {
|
||||
return req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription, nil, nil
|
||||
}
|
||||
return "", nil, errors.New("agent model does not support visual input and no image description is available")
|
||||
}
|
||||
|
||||
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
|
||||
// tenant info, and resolved knowledge bases / search targets.
|
||||
func (s *sessionService) buildAgentConfig(
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
type agentImageVLMStub struct {
|
||||
calls int
|
||||
images [][]byte
|
||||
result string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *agentImageVLMStub) Predict(_ context.Context, images [][]byte, _ string) (string, error) {
|
||||
s.calls++
|
||||
s.images = images
|
||||
return s.result, s.err
|
||||
}
|
||||
func (*agentImageVLMStub) GetModelName() string { return "vlm-stub" }
|
||||
func (*agentImageVLMStub) GetModelID() string { return "vlm-1" }
|
||||
|
||||
type agentImageModelServiceStub struct {
|
||||
interfaces.ModelService
|
||||
model vlm.VLM
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *agentImageModelServiceStub) GetVLMModel(context.Context, string) (vlm.VLM, error) {
|
||||
s.calls++
|
||||
return s.model, s.err
|
||||
}
|
||||
|
||||
func TestPrepareAgentImageInputUsesVLMOnlyForNonVisualPrimary(t *testing.T) {
|
||||
visual := &agentImageVLMStub{result: "机械臂报警E01"}
|
||||
models := &agentImageModelServiceStub{model: visual}
|
||||
req := &types.QARequest{
|
||||
Query: "什么故障?",
|
||||
ImageURLs: []string{"data:image/png;base64,YWJj"},
|
||||
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "vlm-1"}},
|
||||
}
|
||||
query, images, err := prepareAgentImageInput(context.Background(), req, false, models)
|
||||
if err != nil {
|
||||
t.Fatalf("prepareAgentImageInput: %v", err)
|
||||
}
|
||||
if len(images) != 0 || query == req.Query || req.ImageDescription != visual.result {
|
||||
t.Fatalf("query=%q images=%v description=%q", query, images, req.ImageDescription)
|
||||
}
|
||||
if models.calls != 1 || visual.calls != 1 || len(visual.images) != 1 {
|
||||
t.Fatalf("model calls=%d VLM calls=%d images=%d", models.calls, visual.calls, len(visual.images))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareAgentImageInputSkipsBrokenVLMForVisualPrimary(t *testing.T) {
|
||||
models := &agentImageModelServiceStub{err: errors.New("stale VLM")}
|
||||
req := &types.QARequest{
|
||||
Query: "describe",
|
||||
ImageURLs: []string{"data:image/png;base64,YWJj"},
|
||||
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "broken"}},
|
||||
}
|
||||
primary := &captureChatModel{supportsImage: true}
|
||||
query, images, err := prepareAgentImageInput(context.Background(), req, modelSupportsDirectImages(primary), models)
|
||||
if err != nil {
|
||||
t.Fatalf("visual primary should bypass VLM: %v", err)
|
||||
}
|
||||
if query != req.Query || len(images) != 1 || models.calls != 0 {
|
||||
t.Fatalf("query=%q images=%v model_calls=%d", query, images, models.calls)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,12 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// modelSupportsDirectImages asks the resolved concrete Chat transport. Unknown
|
||||
// transports fail closed because chat.SupportsImageInput returns false.
|
||||
func modelSupportsDirectImages(model chat.Chat) bool {
|
||||
return chat.SupportsImageInput(model)
|
||||
}
|
||||
|
||||
// KnowledgeQA performs knowledge base question answering with LLM summarization
|
||||
// Events are emitted through eventBus (references, answer chunks, completion)
|
||||
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
|
||||
@@ -68,12 +74,17 @@ func (s *sessionService) KnowledgeQA(
|
||||
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
|
||||
}
|
||||
|
||||
// Resolve chat model vision capability and VLM model ID for image routing
|
||||
// Resolve the concrete Chat transport for image routing. Runtime transport
|
||||
// capability is authoritative; stored metadata may be stale for existing
|
||||
// WeKnora Cloud records.
|
||||
var chatModelSupportsVision bool
|
||||
var vlmModelID string
|
||||
if chatModelID != "" {
|
||||
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
|
||||
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
|
||||
if chatModelID != "" && len(req.ImageURLs) > 0 {
|
||||
chatModel, modelErr := s.modelService.GetChatModel(ctx, chatModelID)
|
||||
if modelErr != nil {
|
||||
logger.Warnf(ctx, "Failed to initialize Chat model %s for image routing: %v", chatModelID, modelErr)
|
||||
} else {
|
||||
chatModelSupportsVision = modelSupportsDirectImages(chatModel)
|
||||
}
|
||||
}
|
||||
if req.CustomAgent != nil {
|
||||
@@ -136,7 +147,8 @@ func (s *sessionService) KnowledgeQA(
|
||||
VLMModelID: vlmModelID,
|
||||
ChatModelSupportsVision: chatModelSupportsVision,
|
||||
Attachments: req.Attachments,
|
||||
Language: types.LanguageNameFromContext(ctx),
|
||||
|
||||
Language: types.LanguageNameFromContext(ctx),
|
||||
},
|
||||
PipelineState: types.PipelineState{
|
||||
RewriteQuery: req.Query,
|
||||
@@ -158,6 +170,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
|
||||
needsRAG := hasKB || req.WebSearchEnabled
|
||||
hasHistory := chatManage.MaxRounds > 0
|
||||
needsVisualDescription := len(req.ImageURLs) > 0 && !chatModelSupportsVision
|
||||
|
||||
var pipeline []types.EventType
|
||||
if !needsRAG {
|
||||
@@ -178,6 +191,8 @@ func (s *sessionService) KnowledgeQA(
|
||||
pipeline = types.NewPipelineBuilder().
|
||||
AddIf(hasHistory, types.LOAD_HISTORY).
|
||||
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
|
||||
AddIf(needsVisualDescription, types.QUERY_UNDERSTAND).
|
||||
AddIf(needsVisualDescription, types.INTO_CHAT_MESSAGE).
|
||||
Add(types.CHAT_COMPLETION_STREAM).
|
||||
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
|
||||
Build()
|
||||
|
||||
@@ -16,15 +16,19 @@ import (
|
||||
)
|
||||
|
||||
type captureChatModel struct {
|
||||
lastMessages []chat.Message
|
||||
lastMessages []chat.Message
|
||||
supportsImage bool
|
||||
chatResponse *types.ChatResponse
|
||||
chatErr error
|
||||
}
|
||||
|
||||
func (m *captureChatModel) Chat(
|
||||
context.Context,
|
||||
[]chat.Message,
|
||||
*chat.ChatOptions,
|
||||
_ context.Context,
|
||||
messages []chat.Message,
|
||||
_ *chat.ChatOptions,
|
||||
) (*types.ChatResponse, error) {
|
||||
return nil, nil
|
||||
m.lastMessages = append([]chat.Message(nil), messages...)
|
||||
return m.chatResponse, m.chatErr
|
||||
}
|
||||
|
||||
func (m *captureChatModel) ChatStream(
|
||||
@@ -44,8 +48,14 @@ func (m *captureChatModel) ChatStream(
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (m *captureChatModel) GetModelName() string { return "capture" }
|
||||
func (m *captureChatModel) GetModelID() string { return "capture" }
|
||||
func (m *captureChatModel) GetModelName() string { return "capture" }
|
||||
func (m *captureChatModel) GetModelID() string { return "capture" }
|
||||
func (m *captureChatModel) SupportsImageInput() bool { return m.supportsImage }
|
||||
|
||||
func TestModelSupportsDirectImagesUsesResolvedTransportAuthority(t *testing.T) {
|
||||
assert.False(t, modelSupportsDirectImages(&captureChatModel{supportsImage: false}))
|
||||
assert.True(t, modelSupportsDirectImages(&captureChatModel{supportsImage: true}))
|
||||
}
|
||||
|
||||
type stubModelService struct {
|
||||
chatModel chat.Chat
|
||||
|
||||
Reference in New Issue
Block a user