feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
+17 -3
View File
@@ -19,6 +19,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
query string,
state *types.AgentState,
sessionID string,
imageURLs []string,
) error {
totalToolCalls := countTotalToolCalls(state.RoundSteps)
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
@@ -36,7 +37,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
messages := []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userTurn},
{Role: "user", Content: userTurn, Images: append([]string(nil), imageURLs...)},
}
// Add all tool call results as context
@@ -138,10 +139,23 @@ Now generate the final answer:`, query)
return nil
}
func collectAgentImageURLs(messages []chat.Message) []string {
var images []string
for _, message := range messages {
images = append(images, message.Images...)
for _, part := range message.MultiContent {
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
images = append(images, part.ImageURL.URL)
}
}
}
return images
}
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
// without the LLM producing a natural stop. It marks state.IsComplete = true.
func (e *AgentEngine) handleMaxIterations(
ctx context.Context, query string, state *types.AgentState, sessionID string,
ctx context.Context, query string, state *types.AgentState, sessionID string, imageURLs []string,
) {
logger.Info(ctx, "Reached max iterations, generating final answer")
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
@@ -150,7 +164,7 @@ func (e *AgentEngine) handleMaxIterations(
})
// Stream final answer generation through EventBus
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); err != nil {
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs); err != nil {
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
"error": err.Error(),