feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -16,6 +16,12 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// modelSupportsDirectImages asks the resolved concrete Chat transport. Unknown
|
||||
// transports fail closed because chat.SupportsImageInput returns false.
|
||||
func modelSupportsDirectImages(model chat.Chat) bool {
|
||||
return chat.SupportsImageInput(model)
|
||||
}
|
||||
|
||||
// KnowledgeQA performs knowledge base question answering with LLM summarization
|
||||
// Events are emitted through eventBus (references, answer chunks, completion)
|
||||
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
|
||||
@@ -68,12 +74,17 @@ func (s *sessionService) KnowledgeQA(
|
||||
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
|
||||
}
|
||||
|
||||
// Resolve chat model vision capability and VLM model ID for image routing
|
||||
// Resolve the concrete Chat transport for image routing. Runtime transport
|
||||
// capability is authoritative; stored metadata may be stale for existing
|
||||
// WeKnora Cloud records.
|
||||
var chatModelSupportsVision bool
|
||||
var vlmModelID string
|
||||
if chatModelID != "" {
|
||||
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
|
||||
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
|
||||
if chatModelID != "" && len(req.ImageURLs) > 0 {
|
||||
chatModel, modelErr := s.modelService.GetChatModel(ctx, chatModelID)
|
||||
if modelErr != nil {
|
||||
logger.Warnf(ctx, "Failed to initialize Chat model %s for image routing: %v", chatModelID, modelErr)
|
||||
} else {
|
||||
chatModelSupportsVision = modelSupportsDirectImages(chatModel)
|
||||
}
|
||||
}
|
||||
if req.CustomAgent != nil {
|
||||
@@ -136,7 +147,8 @@ func (s *sessionService) KnowledgeQA(
|
||||
VLMModelID: vlmModelID,
|
||||
ChatModelSupportsVision: chatModelSupportsVision,
|
||||
Attachments: req.Attachments,
|
||||
Language: types.LanguageNameFromContext(ctx),
|
||||
|
||||
Language: types.LanguageNameFromContext(ctx),
|
||||
},
|
||||
PipelineState: types.PipelineState{
|
||||
RewriteQuery: req.Query,
|
||||
@@ -158,6 +170,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
|
||||
needsRAG := hasKB || req.WebSearchEnabled
|
||||
hasHistory := chatManage.MaxRounds > 0
|
||||
needsVisualDescription := len(req.ImageURLs) > 0 && !chatModelSupportsVision
|
||||
|
||||
var pipeline []types.EventType
|
||||
if !needsRAG {
|
||||
@@ -178,6 +191,8 @@ func (s *sessionService) KnowledgeQA(
|
||||
pipeline = types.NewPipelineBuilder().
|
||||
AddIf(hasHistory, types.LOAD_HISTORY).
|
||||
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
|
||||
AddIf(needsVisualDescription, types.QUERY_UNDERSTAND).
|
||||
AddIf(needsVisualDescription, types.INTO_CHAT_MESSAGE).
|
||||
Add(types.CHAT_COMPLETION_STREAM).
|
||||
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
|
||||
Build()
|
||||
|
||||
Reference in New Issue
Block a user