feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -269,7 +269,7 @@ func (e *AgentEngine) Execute(
|
||||
"tools": toolListStr,
|
||||
})
|
||||
|
||||
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID)
|
||||
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID, imgs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[Agent] Execution failed: %v", err)
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
@@ -349,6 +349,7 @@ func (e *AgentEngine) executeLoop(
|
||||
tools []chat.Tool,
|
||||
sessionID string,
|
||||
messageID string,
|
||||
imageURLs []string,
|
||||
) (*types.AgentState, error) {
|
||||
startTime := time.Now()
|
||||
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
|
||||
@@ -386,7 +387,7 @@ loop:
|
||||
if totalTC := countTotalToolCalls(state.RoundSteps); totalTC > 0 {
|
||||
logger.Infof(ctx, "[Agent] Synthesizing final answer from %d existing tool results",
|
||||
totalTC)
|
||||
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID)
|
||||
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs)
|
||||
state.IsComplete = true
|
||||
}
|
||||
return state, ctx.Err()
|
||||
@@ -419,7 +420,7 @@ loop:
|
||||
// complete answer." message, which then leaks to the UI as the final
|
||||
// answer for a conversation the user deliberately stopped.
|
||||
if !state.IsComplete && ctx.Err() == nil {
|
||||
e.handleMaxIterations(ctx, query, state, sessionID)
|
||||
e.handleMaxIterations(ctx, query, state, sessionID, imageURLs)
|
||||
}
|
||||
|
||||
return state, nil
|
||||
|
||||
@@ -120,7 +120,7 @@ func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
@@ -146,7 +146,7 @@ func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) {
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
@@ -173,7 +173,7 @@ func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) {
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
@@ -328,7 +328,7 @@ func TestExecuteLoop_NaturalStop_DoesNotDuplicateAnswer(t *testing.T) {
|
||||
|
||||
state := &types.AgentState{}
|
||||
_, err := engine.executeLoop(context.Background(), state, "test query",
|
||||
emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, state.IsComplete)
|
||||
@@ -358,7 +358,7 @@ func TestStreamFinalAnswerToEventBus_EmitsDoneWhenProviderEndsWithEmptyChunk(t *
|
||||
})
|
||||
|
||||
state := &types.AgentState{}
|
||||
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1")
|
||||
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, finalAnswerEvents, 2)
|
||||
|
||||
@@ -19,6 +19,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
query string,
|
||||
state *types.AgentState,
|
||||
sessionID string,
|
||||
imageURLs []string,
|
||||
) error {
|
||||
totalToolCalls := countTotalToolCalls(state.RoundSteps)
|
||||
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
|
||||
@@ -36,7 +37,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: userTurn},
|
||||
{Role: "user", Content: userTurn, Images: append([]string(nil), imageURLs...)},
|
||||
}
|
||||
|
||||
// Add all tool call results as context
|
||||
@@ -138,10 +139,23 @@ Now generate the final answer:`, query)
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectAgentImageURLs(messages []chat.Message) []string {
|
||||
var images []string
|
||||
for _, message := range messages {
|
||||
images = append(images, message.Images...)
|
||||
for _, part := range message.MultiContent {
|
||||
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
|
||||
images = append(images, part.ImageURL.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
return images
|
||||
}
|
||||
|
||||
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
|
||||
// without the LLM producing a natural stop. It marks state.IsComplete = true.
|
||||
func (e *AgentEngine) handleMaxIterations(
|
||||
ctx context.Context, query string, state *types.AgentState, sessionID string,
|
||||
ctx context.Context, query string, state *types.AgentState, sessionID string, imageURLs []string,
|
||||
) {
|
||||
logger.Info(ctx, "Reached max iterations, generating final answer")
|
||||
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
|
||||
@@ -150,7 +164,7 @@ func (e *AgentEngine) handleMaxIterations(
|
||||
})
|
||||
|
||||
// Stream final answer generation through EventBus
|
||||
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); err != nil {
|
||||
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs); err != nil {
|
||||
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
|
||||
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
)
|
||||
|
||||
func TestCollectAgentImageURLsForFinalSynthesis(t *testing.T) {
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "question", Images: []string{"data:image/png;base64,YWJj"}},
|
||||
}
|
||||
got := collectAgentImageURLs(messages)
|
||||
if len(got) != 1 || got[0] != "data:image/png;base64,YWJj" {
|
||||
t.Fatalf("collectAgentImageURLs() = %#v", got)
|
||||
}
|
||||
}
|
||||
+10
-6
@@ -2,6 +2,7 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -60,7 +61,7 @@ func (e *AgentEngine) streamLLMToEventBus(
|
||||
// as if they were part of the LLM answer.
|
||||
if chunk.ResponseType == types.ResponseTypeError {
|
||||
result.StreamError = chunk.Content
|
||||
continue
|
||||
break
|
||||
}
|
||||
|
||||
if chunk.Content != "" {
|
||||
@@ -89,6 +90,9 @@ func (e *AgentEngine) streamLLMToEventBus(
|
||||
if emitFunc != nil {
|
||||
emitFunc(&chunk, result.Content)
|
||||
}
|
||||
if chunk.Done && chunk.ResponseType == types.ResponseTypeAnswer {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Stream diagnostic summary: helps identify non-streaming patterns
|
||||
@@ -101,10 +105,10 @@ func (e *AgentEngine) streamLLMToEventBus(
|
||||
chunkCount, len(result.Content), len(result.ToolCalls),
|
||||
streamDuration.Milliseconds(), responseTypeCounts)
|
||||
|
||||
// If the stream produced an error and no usable content/tool calls,
|
||||
// surface it as a Go error so the caller can retry or degrade gracefully.
|
||||
if result.StreamError != "" && result.Content == "" && len(result.ToolCalls) == 0 {
|
||||
return result, fmt.Errorf("LLM stream error: %s", result.StreamError)
|
||||
// A provider error invalidates the entire stream, even if partial content or
|
||||
// tool calls arrived first. The caller must replace/discard that partial data.
|
||||
if result.StreamError != "" {
|
||||
return result, errors.New("LLM stream failed")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
@@ -406,7 +410,7 @@ func (e *AgentEngine) callLLMWithRetry(
|
||||
"steps": len(state.RoundSteps),
|
||||
"tool_calls": totalTC,
|
||||
})
|
||||
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); synthErr != nil {
|
||||
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, collectAgentImageURLs(messages)); synthErr != nil {
|
||||
logger.Errorf(ctx, "[Agent] Final answer synthesis also failed: %v", synthErr)
|
||||
return nil, fmt.Errorf("LLM call failed: %w (synthesis also failed: %v)", err, synthErr)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestStreamLLMToEventBusRejectsPartialContentAfterStreamError(t *testing.T) {
|
||||
model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{
|
||||
{ResponseType: types.ResponseTypeAnswer, Content: "unverified partial"},
|
||||
{ResponseType: types.ResponseTypeError, Content: "provider failed", Done: true},
|
||||
{ResponseType: types.ResponseTypeAnswer, Content: "must be ignored", Done: true},
|
||||
}}}}
|
||||
engine := &AgentEngine{config: &types.AgentConfig{}, chatModel: model}
|
||||
|
||||
result, err := engine.streamLLMToEventBus(context.Background(), []chat.Message{{Role: "user", Content: "inspect image"}}, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected stream error after partial content")
|
||||
}
|
||||
if result == nil || result.Content != "unverified partial" {
|
||||
t.Fatalf("result = %#v; caller needs partial content identified for replacement", result)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user