feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
+45 -11
View File
@@ -2,10 +2,12 @@ package chat
import (
"context"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
@@ -16,8 +18,9 @@ type langfuseChat struct {
inner Chat
}
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
func (l *langfuseChat) SupportsImageInput() bool { return SupportsImageInput(l.inner) }
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
mgr := langfuse.GetManager()
@@ -47,7 +50,7 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
)
}
gen.Finish(output, usage, err)
gen.Finish(output, usage, sanitizeLangfuseError(err))
return resp, err
}
@@ -71,7 +74,7 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
ch, err := l.inner.ChatStream(genCtx, messages, opts)
if err != nil {
gen.Finish(nil, nil, err)
gen.Finish(nil, nil, sanitizeLangfuseError(err))
return ch, err
}
if ch == nil {
@@ -131,10 +134,21 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
"role": m.Role,
}
if m.Content != "" {
entry["content"] = m.Content
entry["content"] = secutils.RedactDataURLsForLog(m.Content)
}
if len(m.MultiContent) > 0 {
entry["content"] = m.MultiContent
parts := make([]MessageContentPart, len(m.MultiContent))
copy(parts, m.MultiContent)
for i := range parts {
parts[i].Text = secutils.RedactDataURLsForLog(parts[i].Text)
if parts[i].ImageURL == nil {
continue
}
imageURL := *parts[i].ImageURL
imageURL.URL = "[image-url-redacted]"
parts[i].ImageURL = &imageURL
}
entry["content"] = parts
}
if m.Name != "" {
entry["name"] = m.Name
@@ -143,10 +157,15 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
entry["tool_call_id"] = m.ToolCallID
}
if len(m.ToolCalls) > 0 {
entry["tool_calls"] = m.ToolCalls
calls := make([]ToolCall, len(m.ToolCalls))
copy(calls, m.ToolCalls)
for i := range calls {
calls[i].Function.Arguments = secutils.RedactDataURLsForLog(calls[i].Function.Arguments)
}
entry["tool_calls"] = calls
}
if m.ReasoningContent != "" {
entry["reasoning_content"] = m.ReasoningContent
entry["reasoning_content"] = secutils.RedactDataURLsForLog(m.ReasoningContent)
}
out = append(out, entry)
}
@@ -157,17 +176,32 @@ func buildLangfuseGenerationOutput(
content, reasoningContent, finishReason string,
toolCalls []types.LLMToolCall,
) map[string]interface{} {
var safeToolCalls []types.LLMToolCall
if toolCalls != nil {
safeToolCalls = make([]types.LLMToolCall, len(toolCalls))
copy(safeToolCalls, toolCalls)
}
for i := range safeToolCalls {
safeToolCalls[i].Function.Arguments = secutils.RedactDataURLsForLog(safeToolCalls[i].Function.Arguments)
}
output := map[string]interface{}{
"content": content,
"tool_calls": toolCalls,
"content": secutils.RedactDataURLsForLog(content),
"tool_calls": safeToolCalls,
"finish_reason": finishReason,
}
if reasoningContent != "" {
output["reasoning_content"] = reasoningContent
output["reasoning_content"] = secutils.RedactDataURLsForLog(reasoningContent)
}
return output
}
func sanitizeLangfuseError(err error) error {
if err == nil {
return nil
}
return errors.New(secutils.RedactDataURLsForLog(err.Error()))
}
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
if opts == nil {
return nil