feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -19,6 +19,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
query string,
|
||||
state *types.AgentState,
|
||||
sessionID string,
|
||||
imageURLs []string,
|
||||
) error {
|
||||
totalToolCalls := countTotalToolCalls(state.RoundSteps)
|
||||
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
|
||||
@@ -36,7 +37,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: userTurn},
|
||||
{Role: "user", Content: userTurn, Images: append([]string(nil), imageURLs...)},
|
||||
}
|
||||
|
||||
// Add all tool call results as context
|
||||
@@ -138,10 +139,23 @@ Now generate the final answer:`, query)
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectAgentImageURLs(messages []chat.Message) []string {
|
||||
var images []string
|
||||
for _, message := range messages {
|
||||
images = append(images, message.Images...)
|
||||
for _, part := range message.MultiContent {
|
||||
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
|
||||
images = append(images, part.ImageURL.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
return images
|
||||
}
|
||||
|
||||
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
|
||||
// without the LLM producing a natural stop. It marks state.IsComplete = true.
|
||||
func (e *AgentEngine) handleMaxIterations(
|
||||
ctx context.Context, query string, state *types.AgentState, sessionID string,
|
||||
ctx context.Context, query string, state *types.AgentState, sessionID string, imageURLs []string,
|
||||
) {
|
||||
logger.Info(ctx, "Reached max iterations, generating final answer")
|
||||
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
|
||||
@@ -150,7 +164,7 @@ func (e *AgentEngine) handleMaxIterations(
|
||||
})
|
||||
|
||||
// Stream final answer generation through EventBus
|
||||
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); err != nil {
|
||||
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs); err != nil {
|
||||
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
|
||||
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
|
||||
Reference in New Issue
Block a user