feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
+4 -3
View File
@@ -269,7 +269,7 @@ func (e *AgentEngine) Execute(
"tools": toolListStr,
})
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID)
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID, imgs)
if err != nil {
logger.Errorf(ctx, "[Agent] Execution failed: %v", err)
e.eventBus.Emit(ctx, event.Event{
@@ -349,6 +349,7 @@ func (e *AgentEngine) executeLoop(
tools []chat.Tool,
sessionID string,
messageID string,
imageURLs []string,
) (*types.AgentState, error) {
startTime := time.Now()
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
@@ -386,7 +387,7 @@ loop:
if totalTC := countTotalToolCalls(state.RoundSteps); totalTC > 0 {
logger.Infof(ctx, "[Agent] Synthesizing final answer from %d existing tool results",
totalTC)
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID)
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs)
state.IsComplete = true
}
return state, ctx.Err()
@@ -419,7 +420,7 @@ loop:
// complete answer." message, which then leaks to the UI as the final
// answer for a conversation the user deliberately stopped.
if !state.IsComplete && ctx.Err() == nil {
e.handleMaxIterations(ctx, query, state, sessionID)
e.handleMaxIterations(ctx, query, state, sessionID, imageURLs)
}
return state, nil
+5 -5
View File
@@ -120,7 +120,7 @@ func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
assert.NoError(t, err)
assert.True(t, state.IsComplete)
@@ -146,7 +146,7 @@ func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) {
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
assert.NoError(t, err)
assert.True(t, state.IsComplete)
@@ -173,7 +173,7 @@ func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) {
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
assert.NoError(t, err)
assert.True(t, state.IsComplete)
@@ -328,7 +328,7 @@ func TestExecuteLoop_NaturalStop_DoesNotDuplicateAnswer(t *testing.T) {
state := &types.AgentState{}
_, err := engine.executeLoop(context.Background(), state, "test query",
emptyMessages(), emptyTools(), "sess-1", "msg-1")
emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
require.NoError(t, err)
assert.True(t, state.IsComplete)
@@ -358,7 +358,7 @@ func TestStreamFinalAnswerToEventBus_EmitsDoneWhenProviderEndsWithEmptyChunk(t *
})
state := &types.AgentState{}
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1")
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1", nil)
require.NoError(t, err)
require.Len(t, finalAnswerEvents, 2)
+17 -3
View File
@@ -19,6 +19,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
query string,
state *types.AgentState,
sessionID string,
imageURLs []string,
) error {
totalToolCalls := countTotalToolCalls(state.RoundSteps)
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
@@ -36,7 +37,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
messages := []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userTurn},
{Role: "user", Content: userTurn, Images: append([]string(nil), imageURLs...)},
}
// Add all tool call results as context
@@ -138,10 +139,23 @@ Now generate the final answer:`, query)
return nil
}
func collectAgentImageURLs(messages []chat.Message) []string {
var images []string
for _, message := range messages {
images = append(images, message.Images...)
for _, part := range message.MultiContent {
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
images = append(images, part.ImageURL.URL)
}
}
}
return images
}
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
// without the LLM producing a natural stop. It marks state.IsComplete = true.
func (e *AgentEngine) handleMaxIterations(
ctx context.Context, query string, state *types.AgentState, sessionID string,
ctx context.Context, query string, state *types.AgentState, sessionID string, imageURLs []string,
) {
logger.Info(ctx, "Reached max iterations, generating final answer")
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
@@ -150,7 +164,7 @@ func (e *AgentEngine) handleMaxIterations(
})
// Stream final answer generation through EventBus
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); err != nil {
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs); err != nil {
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
"error": err.Error(),
+18
View File
@@ -0,0 +1,18 @@
package agent
import (
"testing"
"github.com/Tencent/WeKnora/internal/models/chat"
)
func TestCollectAgentImageURLsForFinalSynthesis(t *testing.T) {
messages := []chat.Message{
{Role: "system", Content: "system"},
{Role: "user", Content: "question", Images: []string{"data:image/png;base64,YWJj"}},
}
got := collectAgentImageURLs(messages)
if len(got) != 1 || got[0] != "data:image/png;base64,YWJj" {
t.Fatalf("collectAgentImageURLs() = %#v", got)
}
}
+10 -6
View File
@@ -2,6 +2,7 @@ package agent
import (
"context"
"errors"
"fmt"
"strings"
"time"
@@ -60,7 +61,7 @@ func (e *AgentEngine) streamLLMToEventBus(
// as if they were part of the LLM answer.
if chunk.ResponseType == types.ResponseTypeError {
result.StreamError = chunk.Content
continue
break
}
if chunk.Content != "" {
@@ -89,6 +90,9 @@ func (e *AgentEngine) streamLLMToEventBus(
if emitFunc != nil {
emitFunc(&chunk, result.Content)
}
if chunk.Done && chunk.ResponseType == types.ResponseTypeAnswer {
break
}
}
// Stream diagnostic summary: helps identify non-streaming patterns
@@ -101,10 +105,10 @@ func (e *AgentEngine) streamLLMToEventBus(
chunkCount, len(result.Content), len(result.ToolCalls),
streamDuration.Milliseconds(), responseTypeCounts)
// If the stream produced an error and no usable content/tool calls,
// surface it as a Go error so the caller can retry or degrade gracefully.
if result.StreamError != "" && result.Content == "" && len(result.ToolCalls) == 0 {
return result, fmt.Errorf("LLM stream error: %s", result.StreamError)
// A provider error invalidates the entire stream, even if partial content or
// tool calls arrived first. The caller must replace/discard that partial data.
if result.StreamError != "" {
return result, errors.New("LLM stream failed")
}
return result, nil
@@ -406,7 +410,7 @@ func (e *AgentEngine) callLLMWithRetry(
"steps": len(state.RoundSteps),
"tool_calls": totalTC,
})
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); synthErr != nil {
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, collectAgentImageURLs(messages)); synthErr != nil {
logger.Errorf(ctx, "[Agent] Final answer synthesis also failed: %v", synthErr)
return nil, fmt.Errorf("LLM call failed: %w (synthesis also failed: %v)", err, synthErr)
}
+26
View File
@@ -0,0 +1,26 @@
package agent
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
func TestStreamLLMToEventBusRejectsPartialContentAfterStreamError(t *testing.T) {
model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{
{ResponseType: types.ResponseTypeAnswer, Content: "unverified partial"},
{ResponseType: types.ResponseTypeError, Content: "provider failed", Done: true},
{ResponseType: types.ResponseTypeAnswer, Content: "must be ignored", Done: true},
}}}}
engine := &AgentEngine{config: &types.AgentConfig{}, chatModel: model}
result, err := engine.streamLLMToEventBus(context.Background(), []chat.Message{{Role: "user", Content: "inspect image"}}, nil, nil)
if err == nil {
t.Fatal("expected stream error after partial content")
}
if result == nil || result.Content != "unverified partial" {
t.Fatalf("result = %#v; caller needs partial content identified for replacement", result)
}
}