feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
@@ -79,8 +79,10 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
})
responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt)
streamCtx, streamCancel := context.WithCancel(ctx)
responseChan, err := chatModel.ChatStream(streamCtx, chatMessages, opt)
if err != nil {
streamCancel()
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
"error": err.Error(),
@@ -88,6 +90,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
return ErrModelCall.WithError(err)
}
if responseChan == nil {
streamCancel()
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
"error": "nil_channel",
@@ -105,6 +108,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
// The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled
// and the upstream channel is not closed promptly.
go func() {
defer streamCancel()
thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8])
answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8])
thinkingOpen := false
@@ -145,19 +149,20 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
if response.ResponseType == types.ResponseTypeError {
pipelineError(ctx, "Stream", "stream_error", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": response.Content,
"error": "chat completion stream failed",
})
eventBus.Emit(ctx, types.Event{
ID: fmt.Sprintf("%s-error", uuid.New().String()[:8]),
Type: types.EventType(event.EventError),
SessionID: chatManage.SessionID,
Data: event.ErrorData{
Error: response.Content,
Error: "chat completion stream failed",
Stage: "chat_completion_stream",
SessionID: chatManage.SessionID,
},
})
continue
closeThinking()
return
}
if response.ResponseType == types.ResponseTypeThinking {
@@ -190,6 +195,9 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
Done: response.Done,
},
})
if response.Done {
return
}
}
}
}
@@ -91,7 +91,14 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
}
// --- Select the appropriate model ---
rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages)
rewriteModel, useImages, selectErr := p.selectModel(ctx, chatManage, hasImages)
if selectErr != nil {
pipelineError(ctx, "QueryUnderstand", "get_visual_model", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": selectErr.Error(),
})
return &PluginError{Err: selectErr, Description: "Visual model is unavailable", ErrorType: "image_understanding_failed"}
}
if rewriteModel == nil {
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
"session_id": chatManage.SessionID,
@@ -127,11 +134,18 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
"session_id": chatManage.SessionID,
"error": err.Error(),
})
if hasImages {
return &PluginError{Err: err, Description: "Visual model request failed", ErrorType: "image_understanding_failed"}
}
return next()
}
// --- Parse structured output ---
p.parseOutput(chatManage, response.Content)
if hasImages && !chatManage.ChatModelSupportsVision && strings.TrimSpace(chatManage.ImageDescription) == "" {
err := fmt.Errorf("visual model returned no image description")
return &PluginError{Err: err, Description: "Visual model returned no usable image description", ErrorType: "image_understanding_failed"}
}
// Persist image description asynchronously — this DB write does not affect
// the current pipeline result, so it can run in the background.
@@ -222,33 +236,43 @@ func (p *PluginQueryUnderstand) loadHistory(ctx context.Context, chatManage *typ
}
// selectModel picks the model for query understanding. When images are present
// it prefers a vision-capable model. Returns (model, useImages).
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) {
// it requires a vision-capable model and never falls back to text-only processing.
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool, error) {
if hasImages {
var lastErr error
if chatManage.ChatModelSupportsVision {
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err == nil {
return m, true
if err == nil && chat.SupportsImageInput(m) {
return m, true, nil
}
pipelineWarn(ctx, "QueryUnderstand", "vision_model_fallback", map[string]interface{}{
if err == nil {
err = fmt.Errorf("configured Chat transport does not support image input")
}
lastErr = err
pipelineWarn(ctx, "QueryUnderstand", "vision_model_unavailable", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
}
if chatManage.VLMModelID != "" {
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
if err == nil {
return m, true
if err == nil && chat.SupportsImageInput(m) {
return m, true, nil
}
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_fallback", map[string]interface{}{
if err == nil {
err = fmt.Errorf("configured VLM transport does not support image input")
}
lastErr = err
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_unavailable", map[string]interface{}{
"session_id": chatManage.SessionID,
"vlm_model_id": chatManage.VLMModelID,
"error": err.Error(),
})
}
pipelineWarn(ctx, "QueryUnderstand", "no_vision_model", map[string]interface{}{
"session_id": chatManage.SessionID,
})
if lastErr != nil {
return nil, false, fmt.Errorf("no usable visual model: %w", lastErr)
}
return nil, false, fmt.Errorf("no visual model is configured")
}
textModelID := chatManage.ChatModelID
@@ -266,7 +290,7 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
"error": err.Error(),
})
if fallback, fbErr := p.modelService.GetChatModel(ctx, chatManage.ChatModelID); fbErr == nil {
return fallback, false
return fallback, false, nil
}
}
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
@@ -274,9 +298,9 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
"chat_model_id": textModelID,
"error": err.Error(),
})
return nil, false
return nil, false, nil
}
return m, false
return m, false, nil
}
// buildPrompts constructs system and user prompts with placeholder replacement.
@@ -0,0 +1,54 @@
package chatpipeline
import (
"context"
"testing"
modelchat "github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
type visionModelStub struct{}
func (visionModelStub) Chat(context.Context, []modelchat.Message, *modelchat.ChatOptions) (*types.ChatResponse, error) {
return nil, nil
}
func (visionModelStub) ChatStream(context.Context, []modelchat.Message, *modelchat.ChatOptions) (<-chan types.StreamResponse, error) {
return nil, nil
}
func (visionModelStub) GetModelName() string { return "test-vlm" }
func (visionModelStub) GetModelID() string { return "test-vlm" }
func (visionModelStub) SupportsImageInput() bool { return true }
type modelServiceStub struct {
interfaces.ModelService
model modelchat.Chat
}
func (s modelServiceStub) GetChatModel(context.Context, string) (modelchat.Chat, error) {
return s.model, nil
}
func TestSelectModelFailsClosedWhenImageHasNoVisualModel(t *testing.T) {
plugin := &PluginQueryUnderstand{}
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{ChatModelSupportsVision: false}}
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
if err == nil || model != nil || useImages {
t.Fatalf("selectModel = model %v useImages %v err %v; want fail closed", model, useImages, err)
}
}
func TestSelectModelUsesVLMWhenChatTransportIsNotVisual(t *testing.T) {
vlm := visionModelStub{}
plugin := &PluginQueryUnderstand{modelService: modelServiceStub{model: vlm}}
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{
ChatModelSupportsVision: false,
VLMModelID: "document-parser-vlm",
}}
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
if err != nil || model == nil || !useImages {
t.Fatalf("selectModel = model %v useImages %v err %v; want visual VLM fallback", model, useImages, err)
}
}
@@ -0,0 +1,63 @@
package service
import (
"context"
"encoding/base64"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
const maxAgentFallbackImageBytes = 10 << 20
// prepareAgentImageInput routes image input according to the resolved primary
// transport capability. A configured VLM is required only when the actual
// primary Chat transport cannot carry images directly.
func prepareAgentImageInput(
ctx context.Context,
req *types.QARequest,
primarySupportsVision bool,
modelService interfaces.ModelService,
) (string, []string, error) {
if req == nil {
return "", nil, errors.New("QA request is required")
}
if len(req.ImageURLs) == 0 || primarySupportsVision || strings.TrimSpace(req.ImageDescription) != "" {
return routeAgentImageInput(req, primarySupportsVision)
}
if req.CustomAgent == nil || strings.TrimSpace(req.CustomAgent.Config.VLMModelID) == "" {
return "", nil, errors.New("agent model does not support visual input and no VLM is configured")
}
if modelService == nil {
return "", nil, errors.New("agent VLM service is unavailable")
}
images := make([][]byte, 0, len(req.ImageURLs))
for _, raw := range req.ImageURLs {
comma := strings.IndexByte(raw, ',')
if comma <= 0 || !strings.HasPrefix(strings.ToLower(raw[:comma]), "data:image/") ||
!strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
return "", nil, errors.New("agent image payload is invalid")
}
decoded, err := base64.StdEncoding.Strict().DecodeString(raw[comma+1:])
if err != nil || len(decoded) == 0 || len(decoded) > maxAgentFallbackImageBytes {
return "", nil, errors.New("agent image payload is invalid")
}
images = append(images, decoded)
}
model, err := modelService.GetVLMModel(ctx, req.CustomAgent.Config.VLMModelID)
if err != nil || model == nil {
return "", nil, errors.New("agent VLM is unavailable")
}
prompt := fmt.Sprintf("Analyze the uploaded image(s) for the user's question. Return a concise factual description only.\nQuestion: %s", req.Query)
description, err := model.Predict(ctx, images, prompt)
if err != nil || strings.TrimSpace(description) == "" {
return "", nil, errors.New("agent VLM analysis failed")
}
req.ImageDescription = strings.TrimSpace(description)
return routeAgentImageInput(req, false)
}
@@ -0,0 +1,26 @@
package service
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestRouteAgentImageInputFailsClosedWithoutVision(t *testing.T) {
req := &types.QARequest{Query: "what is this", ImageURLs: []string{"data:image/png;base64,abc"}}
if _, images, err := routeAgentImageInput(req, false); err == nil || len(images) != 0 {
t.Fatalf("routeAgentImageInput should reject unsupported visual input: images=%v err=%v", images, err)
}
query, images, err := routeAgentImageInput(req, true)
if err != nil || len(images) != 1 || query != req.Query {
t.Fatalf("vision route = query %q images %d err %v", query, len(images), err)
}
req.ImageDescription = "a blue robot"
query, images, err = routeAgentImageInput(req, false)
if err != nil || len(images) != 0 || !strings.Contains(query, req.ImageDescription) {
t.Fatalf("description fallback = query %q images %d err %v", query, len(images), err)
}
}
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"os"
"strings"
"github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/event"
@@ -156,21 +157,18 @@ func (s *sessionService) AgentQA(
return err
}
// Route image data based on agent model's vision capability
var agentModelSupportsVision bool
if effectiveModelID != "" {
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
}
}
// The resolved concrete transport is authoritative. Stored model metadata can
// be stale (notably older WeKnora Cloud records), while unknown transports
// still fail closed through SupportsImageInput.
agentModelSupportsVision := modelSupportsDirectImages(summaryModel)
agentQuery := req.Query
var agentImageURLs []string
if agentModelSupportsVision && len(req.ImageURLs) > 0 {
agentImageURLs = req.ImageURLs
agentQuery, agentImageURLs, err := prepareAgentImageInput(ctx, req, agentModelSupportsVision, s.modelService)
if err != nil {
return err
}
if len(agentImageURLs) > 0 {
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
} else if req.ImageDescription != "" {
agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
}
if req.QuotedContext != "" {
@@ -208,6 +206,22 @@ func (s *sessionService) AgentQA(
return nil
}
func routeAgentImageInput(req *types.QARequest, modelSupportsVision bool) (string, []string, error) {
if req == nil {
return "", nil, errors.New("QA request is required")
}
if len(req.ImageURLs) == 0 {
return req.Query, nil, nil
}
if modelSupportsVision {
return req.Query, req.ImageURLs, nil
}
if strings.TrimSpace(req.ImageDescription) != "" {
return req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription, nil, nil
}
return "", nil, errors.New("agent model does not support visual input and no image description is available")
}
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
// tenant info, and resolved knowledge bases / search targets.
func (s *sessionService) buildAgentConfig(
@@ -0,0 +1,75 @@
package service
import (
"context"
"errors"
"testing"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
type agentImageVLMStub struct {
calls int
images [][]byte
result string
err error
}
func (s *agentImageVLMStub) Predict(_ context.Context, images [][]byte, _ string) (string, error) {
s.calls++
s.images = images
return s.result, s.err
}
func (*agentImageVLMStub) GetModelName() string { return "vlm-stub" }
func (*agentImageVLMStub) GetModelID() string { return "vlm-1" }
type agentImageModelServiceStub struct {
interfaces.ModelService
model vlm.VLM
err error
calls int
}
func (s *agentImageModelServiceStub) GetVLMModel(context.Context, string) (vlm.VLM, error) {
s.calls++
return s.model, s.err
}
func TestPrepareAgentImageInputUsesVLMOnlyForNonVisualPrimary(t *testing.T) {
visual := &agentImageVLMStub{result: "机械臂报警E01"}
models := &agentImageModelServiceStub{model: visual}
req := &types.QARequest{
Query: "什么故障?",
ImageURLs: []string{"data:image/png;base64,YWJj"},
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "vlm-1"}},
}
query, images, err := prepareAgentImageInput(context.Background(), req, false, models)
if err != nil {
t.Fatalf("prepareAgentImageInput: %v", err)
}
if len(images) != 0 || query == req.Query || req.ImageDescription != visual.result {
t.Fatalf("query=%q images=%v description=%q", query, images, req.ImageDescription)
}
if models.calls != 1 || visual.calls != 1 || len(visual.images) != 1 {
t.Fatalf("model calls=%d VLM calls=%d images=%d", models.calls, visual.calls, len(visual.images))
}
}
func TestPrepareAgentImageInputSkipsBrokenVLMForVisualPrimary(t *testing.T) {
models := &agentImageModelServiceStub{err: errors.New("stale VLM")}
req := &types.QARequest{
Query: "describe",
ImageURLs: []string{"data:image/png;base64,YWJj"},
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "broken"}},
}
primary := &captureChatModel{supportsImage: true}
query, images, err := prepareAgentImageInput(context.Background(), req, modelSupportsDirectImages(primary), models)
if err != nil {
t.Fatalf("visual primary should bypass VLM: %v", err)
}
if query != req.Query || len(images) != 1 || models.calls != 0 {
t.Fatalf("query=%q images=%v model_calls=%d", query, images, models.calls)
}
}
@@ -16,6 +16,12 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// modelSupportsDirectImages asks the resolved concrete Chat transport. Unknown
// transports fail closed because chat.SupportsImageInput returns false.
func modelSupportsDirectImages(model chat.Chat) bool {
return chat.SupportsImageInput(model)
}
// KnowledgeQA performs knowledge base question answering with LLM summarization
// Events are emitted through eventBus (references, answer chunks, completion)
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
@@ -68,12 +74,17 @@ func (s *sessionService) KnowledgeQA(
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
}
// Resolve chat model vision capability and VLM model ID for image routing
// Resolve the concrete Chat transport for image routing. Runtime transport
// capability is authoritative; stored metadata may be stale for existing
// WeKnora Cloud records.
var chatModelSupportsVision bool
var vlmModelID string
if chatModelID != "" {
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
if chatModelID != "" && len(req.ImageURLs) > 0 {
chatModel, modelErr := s.modelService.GetChatModel(ctx, chatModelID)
if modelErr != nil {
logger.Warnf(ctx, "Failed to initialize Chat model %s for image routing: %v", chatModelID, modelErr)
} else {
chatModelSupportsVision = modelSupportsDirectImages(chatModel)
}
}
if req.CustomAgent != nil {
@@ -136,7 +147,8 @@ func (s *sessionService) KnowledgeQA(
VLMModelID: vlmModelID,
ChatModelSupportsVision: chatModelSupportsVision,
Attachments: req.Attachments,
Language: types.LanguageNameFromContext(ctx),
Language: types.LanguageNameFromContext(ctx),
},
PipelineState: types.PipelineState{
RewriteQuery: req.Query,
@@ -158,6 +170,7 @@ func (s *sessionService) KnowledgeQA(
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
needsRAG := hasKB || req.WebSearchEnabled
hasHistory := chatManage.MaxRounds > 0
needsVisualDescription := len(req.ImageURLs) > 0 && !chatModelSupportsVision
var pipeline []types.EventType
if !needsRAG {
@@ -178,6 +191,8 @@ func (s *sessionService) KnowledgeQA(
pipeline = types.NewPipelineBuilder().
AddIf(hasHistory, types.LOAD_HISTORY).
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
AddIf(needsVisualDescription, types.QUERY_UNDERSTAND).
AddIf(needsVisualDescription, types.INTO_CHAT_MESSAGE).
Add(types.CHAT_COMPLETION_STREAM).
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
Build()
@@ -16,15 +16,19 @@ import (
)
type captureChatModel struct {
lastMessages []chat.Message
lastMessages []chat.Message
supportsImage bool
chatResponse *types.ChatResponse
chatErr error
}
func (m *captureChatModel) Chat(
context.Context,
[]chat.Message,
*chat.ChatOptions,
_ context.Context,
messages []chat.Message,
_ *chat.ChatOptions,
) (*types.ChatResponse, error) {
return nil, nil
m.lastMessages = append([]chat.Message(nil), messages...)
return m.chatResponse, m.chatErr
}
func (m *captureChatModel) ChatStream(
@@ -44,8 +48,14 @@ func (m *captureChatModel) ChatStream(
return ch, nil
}
func (m *captureChatModel) GetModelName() string { return "capture" }
func (m *captureChatModel) GetModelID() string { return "capture" }
func (m *captureChatModel) GetModelName() string { return "capture" }
func (m *captureChatModel) GetModelID() string { return "capture" }
func (m *captureChatModel) SupportsImageInput() bool { return m.supportsImage }
func TestModelSupportsDirectImagesUsesResolvedTransportAuthority(t *testing.T) {
assert.False(t, modelSupportsDirectImages(&captureChatModel{supportsImage: false}))
assert.True(t, modelSupportsDirectImages(&captureChatModel{supportsImage: true}))
}
type stubModelService struct {
chatModel chat.Chat