Files
Enterprise-Knowledge-Base/internal/application/service/session_agent_qa.go
T
ochan.kwon b291381cc2 feat: auto-describe MCP tool result images via VLM
When an MCP tool returns image content items, the agent engine now
automatically generates text descriptions using the configured VLM
model and appends them to the tool result message. This makes image
content accessible to LLMs that receive tool results as text.

Images are not passed directly to the LLM because Chat Completions API
does not reliably support images in tool role messages across providers
(tested: gpt-5.4 silently ignores them, Qwen3.5 processes them).

Changes:
- Add Images field to ToolResult for preserving base64 data URIs
- Add extractContentAndImages() with MIME whitelist, size/count limits
- Add ImageDescriberFunc injection to AgentEngine (no vlm package dep)
- Add describeImages() with graceful error handling and ctx cancellation
- Add VLM model injection in CreateAgentEngine (same pattern as
  SetAppConfig/SetSkillsManager)
- Redact image base64 from Data map to prevent log/SSE exposure
2026-03-25 21:38:15 +08:00

374 lines
15 KiB
Go

package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"github.com/Tencent/WeKnora/internal/agent/tools"
llmcontext "github.com/Tencent/WeKnora/internal/application/service/llmcontext"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// AgentQA performs agent-based question answering with conversation history and streaming support
// customAgent is optional - if provided, uses custom agent configuration instead of tenant defaults
// summaryModelID is optional - if provided, overrides the model from customAgent config
func (s *sessionService) AgentQA(
ctx context.Context,
req *types.QARequest,
eventBus *event.EventBus,
) error {
sessionID := req.Session.ID
sessionJSON, err := json.Marshal(req.Session)
if err != nil {
logger.Errorf(ctx, "Failed to marshal session, session ID: %s, error: %v", sessionID, err)
return fmt.Errorf("failed to marshal session: %w", err)
}
// customAgent is required for AgentQA (handler has already done permission check for shared agent)
if req.CustomAgent == nil {
logger.Warnf(ctx, "Custom agent not provided for session: %s", sessionID)
return errors.New("custom agent configuration is required for agent QA")
}
// Resolve retrieval tenant using shared helper
agentTenantID := s.resolveRetrievalTenantID(ctx, req)
logger.Infof(ctx, "Start agent-based question answering, session ID: %s, agent tenant ID: %d, query: %s, session: %s",
sessionID, agentTenantID, req.Query, string(sessionJSON))
var tenantInfo *types.Tenant
if v := ctx.Value(types.TenantInfoContextKey); v != nil {
tenantInfo, _ = v.(*types.Tenant)
}
// When agent belongs to another tenant (shared agent), use agent's tenant for KB/model scope; load tenantInfo if needed
if tenantInfo == nil || tenantInfo.ID != agentTenantID {
if s.tenantService != nil {
if agentTenant, err := s.tenantService.GetTenantByID(ctx, agentTenantID); err == nil && agentTenant != nil {
tenantInfo = agentTenant
logger.Infof(ctx, "Using agent tenant info for retrieval scope, tenant ID: %d", agentTenantID)
}
}
}
if tenantInfo == nil {
logger.Warnf(ctx, "Tenant info not available for agent tenant %d, proceeding with defaults", agentTenantID)
tenantInfo = &types.Tenant{ID: agentTenantID}
}
// Ensure defaults are set
req.CustomAgent.EnsureDefaults()
// Build AgentConfig from custom agent and tenant info
agentConfig, err := s.buildAgentConfig(ctx, req, tenantInfo, agentTenantID)
if err != nil {
return err
}
// Set VLM model ID for tool result image analysis (runtime-only field)
if req.CustomAgent != nil && req.CustomAgent.Config.VLMModelID != "" {
agentConfig.VLMModelID = req.CustomAgent.Config.VLMModelID
}
// Resolve model ID using shared helper (AgentQA requires a model, so error if not found)
effectiveModelID, err := s.resolveChatModelID(ctx, req, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs)
if err != nil {
return err
}
if effectiveModelID == "" {
logger.Warnf(ctx, "No summary model configured for custom agent %s", req.CustomAgent.ID)
return errors.New("summary model (model_id) is not configured in custom agent settings")
}
summaryModel, err := s.modelService.GetChatModel(ctx, effectiveModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
// Get rerank model from custom agent config (only required when knowledge bases are configured)
var rerankModel rerank.Reranker
hasKnowledge := len(agentConfig.KnowledgeBases) > 0 || len(agentConfig.KnowledgeIDs) > 0
if hasKnowledge {
rerankModelID := req.CustomAgent.Config.RerankModelID
if rerankModelID == "" {
logger.Warnf(ctx, "No rerank model configured for custom agent %s, but knowledge bases are specified", req.CustomAgent.ID)
return errors.New("rerank model (rerank_model_id) is not configured in custom agent settings")
}
rerankModel, err = s.modelService.GetRerankModel(ctx, rerankModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get rerank model: %v", err)
return fmt.Errorf("failed to get rerank model: %w", err)
}
} else {
logger.Infof(ctx, "No knowledge bases configured, skipping rerank model initialization")
}
// Get or create contextManager for this session
contextManager := s.getContextManagerForSession(ctx, req.Session, summaryModel)
// Set system prompt for the current agent in context manager
// This ensures the context uses the correct system prompt when switching agents
systemPrompt := agentConfig.ResolveSystemPrompt(agentConfig.WebSearchEnabled)
if systemPrompt != "" {
if err := contextManager.SetSystemPrompt(ctx, sessionID, systemPrompt); err != nil {
logger.Warnf(ctx, "Failed to set system prompt in context manager: %v", err)
} else {
logger.Infof(ctx, "System prompt updated in context manager for agent")
}
}
// Get LLM context from context manager
llmContext, err := s.getContextForSession(ctx, contextManager, sessionID)
if err != nil {
logger.Warnf(ctx, "Failed to get LLM context: %v, continuing without history", err)
llmContext = []chat.Message{}
}
logger.Infof(ctx, "Loaded %d messages from LLM context manager", len(llmContext))
// Apply multi-turn configuration for Agent mode
// Note: In Agent mode, context is managed by contextManager with compression strategies,
// so we don't apply HistoryTurns limit here. HistoryTurns is used in normal (KnowledgeQA) mode.
if !agentConfig.MultiTurnEnabled {
// Multi-turn disabled, clear history
logger.Infof(ctx, "Multi-turn disabled for this agent, clearing history context")
llmContext = []chat.Message{}
}
// Create agent engine with EventBus and ContextManager
logger.Info(ctx, "Creating agent engine")
engine, err := s.agentService.CreateAgentEngine(
ctx,
agentConfig,
summaryModel,
rerankModel,
eventBus,
contextManager,
sessionID,
)
if err != nil {
logger.Errorf(ctx, "Failed to create agent engine: %v", err)
return err
}
// Route image data based on agent model's vision capability
var agentModelSupportsVision bool
if effectiveModelID != "" {
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
}
}
agentQuery := req.Query
var agentImageURLs []string
if agentModelSupportsVision && len(req.ImageURLs) > 0 {
agentImageURLs = req.ImageURLs
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
} else if req.ImageDescription != "" {
agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
}
// Execute agent with streaming (asynchronously)
// Events will be emitted to EventBus and handled by the Handler layer
logger.Info(ctx, "Executing agent with streaming")
if _, err := engine.Execute(ctx, sessionID, req.AssistantMessageID, agentQuery, llmContext, agentImageURLs); err != nil {
logger.Errorf(ctx, "Agent execution failed: %v", err)
// Emit error event to the EventBus used by this agent
eventBus.Emit(ctx, event.Event{
Type: event.EventError,
SessionID: sessionID,
Data: event.ErrorData{
Error: err.Error(),
Stage: "agent_execution",
SessionID: sessionID,
},
})
}
// Return empty - events will be handled by Handler via EventBus subscription
return nil
}
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
// tenant info, and resolved knowledge bases / search targets.
func (s *sessionService) buildAgentConfig(
ctx context.Context,
req *types.QARequest,
tenantInfo *types.Tenant,
agentTenantID uint64,
) (*types.AgentConfig, error) {
customAgent := req.CustomAgent
agentConfig := &types.AgentConfig{
MaxIterations: customAgent.Config.MaxIterations,
ReflectionEnabled: customAgent.Config.ReflectionEnabled,
Temperature: customAgent.Config.Temperature,
WebSearchEnabled: customAgent.Config.WebSearchEnabled && req.WebSearchEnabled,
WebSearchMaxResults: customAgent.Config.WebSearchMaxResults,
MultiTurnEnabled: customAgent.Config.MultiTurnEnabled,
HistoryTurns: customAgent.Config.HistoryTurns,
MCPSelectionMode: customAgent.Config.MCPSelectionMode,
MCPServices: customAgent.Config.MCPServices,
Thinking: customAgent.Config.Thinking,
RetrieveKBOnlyWhenMentioned: customAgent.Config.RetrieveKBOnlyWhenMentioned,
}
// Configure skills based on CustomAgentConfig
s.configureSkillsFromAgent(ctx, agentConfig, customAgent)
// Resolve knowledge bases using shared helper
agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs = s.resolveKnowledgeBases(ctx, req)
// Use custom agent's allowed tools if specified, otherwise use defaults
if len(customAgent.Config.AllowedTools) > 0 {
agentConfig.AllowedTools = customAgent.Config.AllowedTools
} else {
agentConfig.AllowedTools = tools.DefaultAllowedTools()
}
// Use custom agent's system prompt if specified
if customAgent.Config.SystemPrompt != "" {
agentConfig.UseCustomSystemPrompt = true
agentConfig.SystemPrompt = customAgent.Config.SystemPrompt
}
logger.Infof(ctx, "Custom agent config applied: MaxIterations=%d, Temperature=%.2f, AllowedTools=%v, WebSearchEnabled=%v",
agentConfig.MaxIterations, agentConfig.Temperature, agentConfig.AllowedTools, agentConfig.WebSearchEnabled)
// Set web search max results from tenant config if not set (default: 5)
if agentConfig.WebSearchMaxResults == 0 {
agentConfig.WebSearchMaxResults = 5
if tenantInfo.WebSearchConfig != nil && tenantInfo.WebSearchConfig.MaxResults > 0 {
agentConfig.WebSearchMaxResults = tenantInfo.WebSearchConfig.MaxResults
}
}
logger.Infof(ctx, "Merged agent config from tenant %d and session %s", tenantInfo.ID, req.Session.ID)
// Log knowledge bases if present
if len(agentConfig.KnowledgeBases) > 0 {
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
} else {
logger.Infof(ctx, "No knowledge bases specified for agent, running in pure agent mode")
}
// Build search targets using agent's tenant (handler has validated access for shared agent)
searchTargets, err := s.buildSearchTargets(ctx, agentTenantID, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to build search targets for agent: %v", err)
}
agentConfig.SearchTargets = searchTargets
logger.Infof(ctx, "Agent search targets built: %d targets", len(searchTargets))
return agentConfig, nil
}
// configureSkillsFromAgent configures skills settings in AgentConfig based on CustomAgentConfig
// Returns the skill directories and allowed skills based on the selection mode:
// - "all": uses all preloaded skills
// - "selected": uses the explicitly selected skills
// - "none" or "": skills are disabled
func (s *sessionService) configureSkillsFromAgent(
ctx context.Context,
agentConfig *types.AgentConfig,
customAgent *types.CustomAgent,
) {
if customAgent == nil {
return
}
// When sandbox is disabled, skills cannot be enabled (no script execution environment)
sandboxMode := os.Getenv("WEKNORA_SANDBOX_MODE")
if sandboxMode == "" || sandboxMode == "disabled" {
agentConfig.SkillsEnabled = false
agentConfig.SkillDirs = nil
agentConfig.AllowedSkills = nil
logger.Infof(ctx, "Sandbox is disabled: skills are not available")
return
}
switch customAgent.Config.SkillsSelectionMode {
case "all":
// Enable all preloaded skills
agentConfig.SkillsEnabled = true
agentConfig.SkillDirs = []string{DefaultPreloadedSkillsDir}
agentConfig.AllowedSkills = nil // Empty means all skills allowed
logger.Infof(ctx, "SkillsSelectionMode=all: enabled all preloaded skills")
case "selected":
// Enable only selected skills
if len(customAgent.Config.SelectedSkills) > 0 {
agentConfig.SkillsEnabled = true
agentConfig.SkillDirs = []string{DefaultPreloadedSkillsDir}
agentConfig.AllowedSkills = customAgent.Config.SelectedSkills
logger.Infof(ctx, "SkillsSelectionMode=selected: enabled %d selected skills: %v",
len(customAgent.Config.SelectedSkills), customAgent.Config.SelectedSkills)
} else {
agentConfig.SkillsEnabled = false
logger.Infof(ctx, "SkillsSelectionMode=selected but no skills selected: skills disabled")
}
case "none", "":
// Skills disabled
agentConfig.SkillsEnabled = false
logger.Infof(ctx, "SkillsSelectionMode=%s: skills disabled", customAgent.Config.SkillsSelectionMode)
default:
// Unknown mode, disable skills
agentConfig.SkillsEnabled = false
logger.Warnf(ctx, "Unknown SkillsSelectionMode=%s: skills disabled", customAgent.Config.SkillsSelectionMode)
}
}
// getContextManagerForSession creates a context manager for the session based on configuration
// Returns the configured context manager (tenant-level or session-level) or default
func (s *sessionService) getContextManagerForSession(
ctx context.Context,
session *types.Session,
chatModel chat.Chat,
) interfaces.ContextManager {
// Get tenant to access global context configuration
tenant, _ := types.TenantInfoFromContext(ctx)
// Determine which context config to use: tenant-level or default
var contextConfig *types.ContextConfig
if tenant != nil && tenant.ContextConfig != nil {
// Use tenant-level configuration
contextConfig = tenant.ContextConfig
logger.Infof(ctx, "Using tenant-level context config for session %s", session.ID)
} else {
// Use service's default context manager
logger.Debugf(ctx, "Using default context manager for session %s", session.ID)
contextConfig = &types.ContextConfig{
MaxTokens: llmcontext.DefaultMaxTokens,
CompressionStrategy: llmcontext.DefaultCompressionStrategy,
RecentMessageCount: llmcontext.DefaultRecentMessageCount,
SummarizeThreshold: llmcontext.DefaultSummarizeThreshold,
}
}
return llmcontext.NewContextManagerFromConfig(contextConfig, s.sessionStorage, chatModel)
}
// getContextForSession retrieves LLM context for a session
func (s *sessionService) getContextForSession(
ctx context.Context,
contextManager interfaces.ContextManager,
sessionID string,
) ([]chat.Message, error) {
history, err := contextManager.GetContext(ctx, sessionID)
if err != nil {
return nil, fmt.Errorf("failed to get context: %w", err)
}
// Log context statistics
stats, _ := contextManager.GetContextStats(ctx, sessionID)
if stats != nil {
logger.Infof(ctx, "LLM context stats for session %s: messages=%d, tokens=~%d, compressed=%v",
sessionID, stats.MessageCount, stats.TokenCount, stats.IsCompressed)
}
return history, nil
}