feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
@@ -16,6 +16,12 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// modelSupportsDirectImages asks the resolved concrete Chat transport. Unknown
// transports fail closed because chat.SupportsImageInput returns false.
func modelSupportsDirectImages(model chat.Chat) bool {
return chat.SupportsImageInput(model)
}
// KnowledgeQA performs knowledge base question answering with LLM summarization
// Events are emitted through eventBus (references, answer chunks, completion)
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
@@ -68,12 +74,17 @@ func (s *sessionService) KnowledgeQA(
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
}
// Resolve chat model vision capability and VLM model ID for image routing
// Resolve the concrete Chat transport for image routing. Runtime transport
// capability is authoritative; stored metadata may be stale for existing
// WeKnora Cloud records.
var chatModelSupportsVision bool
var vlmModelID string
if chatModelID != "" {
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
if chatModelID != "" && len(req.ImageURLs) > 0 {
chatModel, modelErr := s.modelService.GetChatModel(ctx, chatModelID)
if modelErr != nil {
logger.Warnf(ctx, "Failed to initialize Chat model %s for image routing: %v", chatModelID, modelErr)
} else {
chatModelSupportsVision = modelSupportsDirectImages(chatModel)
}
}
if req.CustomAgent != nil {
@@ -136,7 +147,8 @@ func (s *sessionService) KnowledgeQA(
VLMModelID: vlmModelID,
ChatModelSupportsVision: chatModelSupportsVision,
Attachments: req.Attachments,
Language: types.LanguageNameFromContext(ctx),
Language: types.LanguageNameFromContext(ctx),
},
PipelineState: types.PipelineState{
RewriteQuery: req.Query,
@@ -158,6 +170,7 @@ func (s *sessionService) KnowledgeQA(
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
needsRAG := hasKB || req.WebSearchEnabled
hasHistory := chatManage.MaxRounds > 0
needsVisualDescription := len(req.ImageURLs) > 0 && !chatModelSupportsVision
var pipeline []types.EventType
if !needsRAG {
@@ -178,6 +191,8 @@ func (s *sessionService) KnowledgeQA(
pipeline = types.NewPipelineBuilder().
AddIf(hasHistory, types.LOAD_HISTORY).
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
AddIf(needsVisualDescription, types.QUERY_UNDERSTAND).
AddIf(needsVisualDescription, types.INTO_CHAT_MESSAGE).
Add(types.CHAT_COMPLETION_STREAM).
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
Build()