feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -2,10 +2,12 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
|
||||
@@ -16,8 +18,9 @@ type langfuseChat struct {
|
||||
inner Chat
|
||||
}
|
||||
|
||||
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
||||
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
||||
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
||||
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
||||
func (l *langfuseChat) SupportsImageInput() bool { return SupportsImageInput(l.inner) }
|
||||
|
||||
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
mgr := langfuse.GetManager()
|
||||
@@ -47,7 +50,7 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
|
||||
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
|
||||
)
|
||||
}
|
||||
gen.Finish(output, usage, err)
|
||||
gen.Finish(output, usage, sanitizeLangfuseError(err))
|
||||
return resp, err
|
||||
}
|
||||
|
||||
@@ -71,7 +74,7 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
|
||||
ch, err := l.inner.ChatStream(genCtx, messages, opts)
|
||||
if err != nil {
|
||||
gen.Finish(nil, nil, err)
|
||||
gen.Finish(nil, nil, sanitizeLangfuseError(err))
|
||||
return ch, err
|
||||
}
|
||||
if ch == nil {
|
||||
@@ -131,10 +134,21 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
||||
"role": m.Role,
|
||||
}
|
||||
if m.Content != "" {
|
||||
entry["content"] = m.Content
|
||||
entry["content"] = secutils.RedactDataURLsForLog(m.Content)
|
||||
}
|
||||
if len(m.MultiContent) > 0 {
|
||||
entry["content"] = m.MultiContent
|
||||
parts := make([]MessageContentPart, len(m.MultiContent))
|
||||
copy(parts, m.MultiContent)
|
||||
for i := range parts {
|
||||
parts[i].Text = secutils.RedactDataURLsForLog(parts[i].Text)
|
||||
if parts[i].ImageURL == nil {
|
||||
continue
|
||||
}
|
||||
imageURL := *parts[i].ImageURL
|
||||
imageURL.URL = "[image-url-redacted]"
|
||||
parts[i].ImageURL = &imageURL
|
||||
}
|
||||
entry["content"] = parts
|
||||
}
|
||||
if m.Name != "" {
|
||||
entry["name"] = m.Name
|
||||
@@ -143,10 +157,15 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
||||
entry["tool_call_id"] = m.ToolCallID
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
entry["tool_calls"] = m.ToolCalls
|
||||
calls := make([]ToolCall, len(m.ToolCalls))
|
||||
copy(calls, m.ToolCalls)
|
||||
for i := range calls {
|
||||
calls[i].Function.Arguments = secutils.RedactDataURLsForLog(calls[i].Function.Arguments)
|
||||
}
|
||||
entry["tool_calls"] = calls
|
||||
}
|
||||
if m.ReasoningContent != "" {
|
||||
entry["reasoning_content"] = m.ReasoningContent
|
||||
entry["reasoning_content"] = secutils.RedactDataURLsForLog(m.ReasoningContent)
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
@@ -157,17 +176,32 @@ func buildLangfuseGenerationOutput(
|
||||
content, reasoningContent, finishReason string,
|
||||
toolCalls []types.LLMToolCall,
|
||||
) map[string]interface{} {
|
||||
var safeToolCalls []types.LLMToolCall
|
||||
if toolCalls != nil {
|
||||
safeToolCalls = make([]types.LLMToolCall, len(toolCalls))
|
||||
copy(safeToolCalls, toolCalls)
|
||||
}
|
||||
for i := range safeToolCalls {
|
||||
safeToolCalls[i].Function.Arguments = secutils.RedactDataURLsForLog(safeToolCalls[i].Function.Arguments)
|
||||
}
|
||||
output := map[string]interface{}{
|
||||
"content": content,
|
||||
"tool_calls": toolCalls,
|
||||
"content": secutils.RedactDataURLsForLog(content),
|
||||
"tool_calls": safeToolCalls,
|
||||
"finish_reason": finishReason,
|
||||
}
|
||||
if reasoningContent != "" {
|
||||
output["reasoning_content"] = reasoningContent
|
||||
output["reasoning_content"] = secutils.RedactDataURLsForLog(reasoningContent)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func sanitizeLangfuseError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return errors.New(secutils.RedactDataURLsForLog(err.Error()))
|
||||
}
|
||||
|
||||
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
|
||||
if opts == nil {
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user