feat(dingtalk): enable secure multimodal image QA

This commit is contained in:
姚章浩
2026-07-16 00:17:16 +08:00
parent 4022040e98
commit c9c4f0aaa5
59 changed files with 3279 additions and 429 deletions
+271 -63
View File
@@ -3,7 +3,9 @@ package chat
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -15,19 +17,37 @@ import (
secutils "github.com/Tencent/WeKnora/internal/utils"
)
const anthropicVersion = "2023-06-01"
const (
anthropicVersion = "2023-06-01"
anthropicResponseBodyLimit = 1 << 20
)
type AnthropicChat struct {
modelName string
modelID string
baseURL string
apiKey string
customHeaders map[string]string
modelName string
modelID string
baseURL string
apiKey string
supportsVision bool
customHeaders map[string]string
}
func (c *AnthropicChat) SupportsImageInput() bool { return c.supportsVision }
type anthropicMessage struct {
Role string `json:"role"`
Content string `json:"content"`
Content any `json:"content"`
}
type anthropicContentBlock struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
Source *anthropicImageSource `json:"source,omitempty"`
}
type anthropicImageSource struct {
Type string `json:"type"`
MediaType string `json:"media_type"`
Data string `json:"data"`
}
type anthropicRequest struct {
@@ -61,6 +81,7 @@ type anthropicResponse struct {
type anthropicStreamEvent struct {
Type string `json:"type"`
Index *int `json:"index,omitempty"`
Message *struct {
Usage struct {
InputTokens int `json:"input_tokens"`
@@ -69,6 +90,9 @@ type anthropicStreamEvent struct {
CacheReadInputTokens int `json:"cache_read_input_tokens"`
} `json:"usage"`
} `json:"message,omitempty"`
ContentBlock *struct {
Type string `json:"type"`
} `json:"content_block,omitempty"`
Delta *struct {
Type string `json:"type"`
Text string `json:"text"`
@@ -86,6 +110,117 @@ type anthropicStreamEvent struct {
} `json:"error,omitempty"`
}
type anthropicStreamProtocolState struct {
messageStarted bool
finishReason string
messageStopped bool
activeBlocks map[int]string
completedBlocks map[int]struct{}
nextBlockIndex int
}
func validAnthropicStopReason(reason string) bool {
switch reason {
case "end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded":
return true
default:
return false
}
}
func anthropicDeltaMatchesBlock(blockType, deltaType string) bool {
switch deltaType {
case "text_delta", "citations_delta":
return blockType == "text"
case "thinking_delta", "signature_delta":
return blockType == "thinking"
case "input_json_delta":
return blockType == "tool_use" || blockType == "server_tool_use" || blockType == "mcp_tool_use"
default:
return false
}
}
func (s *anthropicStreamProtocolState) observe(event *anthropicStreamEvent) error {
if event == nil {
return errors.New("anthropic stream event is missing")
}
if event.Type == "error" || event.Error != nil {
return errors.New("anthropic stream failed")
}
if s.messageStopped {
if event.Type == "ping" {
return nil
}
return errors.New("anthropic event followed message_stop")
}
if event.Delta != nil && event.Delta.StopReason != "" && event.Type != "message_delta" {
return errors.New("anthropic stop reason appeared on an invalid event")
}
switch event.Type {
case "message_start":
if s.messageStarted || s.finishReason != "" {
return errors.New("anthropic message_start is out of order")
}
s.messageStarted = true
s.activeBlocks = make(map[int]string)
s.completedBlocks = make(map[int]struct{})
case "message_delta":
if !s.messageStarted || s.finishReason != "" || len(s.activeBlocks) != 0 || event.Delta == nil || !validAnthropicStopReason(event.Delta.StopReason) {
return errors.New("anthropic message_delta has an invalid stop reason")
}
s.finishReason = event.Delta.StopReason
case "message_stop":
if !s.messageStarted || s.finishReason == "" || len(s.activeBlocks) != 0 {
return errors.New("anthropic message_stop preceded valid completion")
}
s.messageStopped = true
case "content_block_start":
if !s.messageStarted || s.finishReason != "" || event.Index == nil || *event.Index < 0 || event.ContentBlock == nil || strings.TrimSpace(event.ContentBlock.Type) == "" {
return errors.New("anthropic content_block_start is invalid")
}
index := *event.Index
if len(s.activeBlocks) != 0 || index != s.nextBlockIndex {
return errors.New("anthropic content block index is out of order")
}
if _, active := s.activeBlocks[index]; active {
return errors.New("anthropic content block started twice")
}
if _, completed := s.completedBlocks[index]; completed {
return errors.New("anthropic completed content block restarted")
}
s.activeBlocks[index] = event.ContentBlock.Type
case "content_block_delta":
if !s.messageStarted || s.finishReason != "" || event.Index == nil || event.Delta == nil {
return errors.New("anthropic content_block_delta is invalid")
}
blockType, active := s.activeBlocks[*event.Index]
if !active || !anthropicDeltaMatchesBlock(blockType, event.Delta.Type) {
return errors.New("anthropic content block delta is out of order")
}
case "content_block_stop":
if !s.messageStarted || s.finishReason != "" || event.Index == nil {
return errors.New("anthropic content_block_stop is invalid")
}
index := *event.Index
if _, active := s.activeBlocks[index]; !active {
return errors.New("anthropic content block stop is out of order")
}
delete(s.activeBlocks, index)
s.completedBlocks[index] = struct{}{}
s.nextBlockIndex++
case "ping":
default:
return errors.New("anthropic stream event type is invalid")
}
return nil
}
func (s *anthropicStreamProtocolState) complete() bool {
return s.messageStarted && s.messageStopped && len(s.activeBlocks) == 0 && validAnthropicStopReason(s.finishReason)
}
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
if config.BaseURL != "" {
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
@@ -102,16 +237,31 @@ func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
}
return &AnthropicChat{
modelName: config.ModelName,
modelID: config.ModelID,
baseURL: baseURL,
apiKey: config.APIKey,
customHeaders: config.CustomHeaders,
modelName: config.ModelName,
modelID: config.ModelID,
baseURL: baseURL,
apiKey: config.APIKey,
supportsVision: config.SupportsVision,
customHeaders: config.CustomHeaders,
}, nil
}
func readAnthropicResponseBody(body io.ReadCloser) ([]byte, error) {
data, err := io.ReadAll(io.LimitReader(body, anthropicResponseBodyLimit+1))
if err != nil {
return nil, errors.New("read Anthropic response failed")
}
if len(data) > anthropicResponseBodyLimit {
return nil, errors.New("Anthropic response body too large")
}
return data, nil
}
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
reqBody := c.buildRequest(messages, opts)
reqBody, err := c.buildRequest(messages, opts)
if err != nil {
return nil, err
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
@@ -136,13 +286,13 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
return nil, errors.New("send Anthropic request failed")
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
body, err := readAnthropicResponseBody(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
return nil, err
}
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
@@ -151,7 +301,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
return nil, err
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content)
return nil, remoteAPIStatusError(resp.StatusCode, body)
}
logUsage(ctx, c.modelName, &chatResp.Usage)
return chatResp, nil
@@ -159,13 +309,10 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
var chatResp anthropicResponse
if err := json.Unmarshal(body, &chatResp); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
return nil, errors.New("decode Anthropic response failed")
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
if chatResp.Error != nil && chatResp.Error.Message != "" {
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Error.Message)
}
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
return nil, remoteAPIStatusError(resp.StatusCode, body)
}
result := c.parseResponse(&chatResp)
@@ -174,7 +321,10 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
}
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
reqBody := c.buildRequest(messages, opts)
reqBody, err := c.buildRequest(messages, opts)
if err != nil {
return nil, err
}
reqBody.Stream = true
jsonData, err := json.Marshal(reqBody)
if err != nil {
@@ -198,12 +348,15 @@ func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
return nil, errors.New("send Anthropic stream request failed")
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(resp.Body)
body, readErr := readAnthropicResponseBody(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
if readErr != nil {
return nil, readErr
}
return nil, remoteAPIStatusError(resp.StatusCode, body)
}
streamChan := make(chan types.StreamResponse)
@@ -248,7 +401,7 @@ func isAnthropicVersionedBaseURL(baseURL string) bool {
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
}
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) (anthropicRequest, error) {
req := anthropicRequest{
Model: c.modelName,
MaxTokens: 1024,
@@ -276,22 +429,75 @@ func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anth
if content == "" {
content = textFromMultiContent(msg.MultiContent)
}
if content == "" {
images := append([]string(nil), msg.Images...)
for _, part := range msg.MultiContent {
if part.Type == "image_url" && part.ImageURL != nil && strings.TrimSpace(part.ImageURL.URL) != "" {
images = append(images, part.ImageURL.URL)
}
}
if len(images) == 0 && content == "" {
continue
}
switch msg.Role {
case "system":
if len(images) > 0 {
return anthropicRequest{}, fmt.Errorf("anthropic system messages cannot contain images")
}
systemParts = append(systemParts, content)
case "assistant":
if len(images) > 0 {
return anthropicRequest{}, fmt.Errorf("anthropic assistant messages cannot contain images")
}
req.Messages = append(req.Messages, anthropicMessage{Role: "assistant", Content: content})
case "user":
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
default:
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
if len(images) == 0 {
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
continue
}
blocks := make([]anthropicContentBlock, 0, len(images)+1)
if content != "" {
blocks = append(blocks, anthropicContentBlock{Type: "text", Text: content})
}
for _, imageURL := range images {
block, err := anthropicImageBlock(imageURL)
if err != nil {
return anthropicRequest{}, err
}
blocks = append(blocks, block)
}
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: blocks})
}
}
req.System = strings.Join(systemParts, "\n\n")
return req
return req, nil
}
func anthropicImageBlock(raw string) (anthropicContentBlock, error) {
metadata, payload, ok := strings.Cut(strings.TrimSpace(raw), ",")
if !ok {
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
}
metadata = strings.ToLower(metadata)
var mediaType string
switch metadata {
case "data:image/jpeg;base64", "data:image/jpg;base64":
mediaType = "image/jpeg"
case "data:image/png;base64":
mediaType = "image/png"
default:
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
}
if _, err := base64.StdEncoding.DecodeString(payload); err != nil {
return anthropicContentBlock{}, fmt.Errorf("anthropic image data is invalid")
}
return anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: "base64",
MediaType: mediaType,
Data: payload,
},
}, nil
}
func textFromMultiContent(parts []MessageContentPart) string {
@@ -330,7 +536,7 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
sseReader := NewSSEReader(reader)
var contentParts []string
var finishReason string
var protocol anthropicStreamProtocolState
var inputTokens int
var outputTokens int
@@ -353,8 +559,8 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
return nil, fmt.Errorf("decode SSE response: %w", err)
}
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message)
if err := protocol.observe(&streamEvent); err != nil {
return nil, errors.New("anthropic stream protocol failed")
}
if streamEvent.Message != nil {
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
@@ -364,19 +570,19 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
contentParts = append(contentParts, streamEvent.Delta.Text)
}
if streamEvent.Delta.StopReason != "" {
finishReason = streamEvent.Delta.StopReason
}
}
if streamEvent.Usage != nil {
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
}
}
if !protocol.complete() {
return nil, errors.New("anthropic stream ended before completion")
}
return &types.ChatResponse{
Content: strings.Join(contentParts, ""),
FinishReason: finishReason,
FinishReason: protocol.finishReason,
Usage: types.TokenUsage{
PromptTokens: inputTokens,
CompletionTokens: outputTokens,
@@ -391,38 +597,43 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
sseReader := NewSSEReader(resp.Body)
var usage *types.TokenUsage
var finishReason string
var protocol anthropicStreamProtocolState
emitTerminal := func() {
if !protocol.complete() {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: "anthropic stream ended before completion",
Done: true,
}
return
}
logUsage(ctx, model, usage)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
Usage: usage,
FinishReason: protocol.finishReason,
}
}
for {
event, err := sseReader.ReadEvent()
if err != nil {
if err == io.EOF {
logUsage(ctx, model, usage)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
Usage: usage,
FinishReason: finishReason,
}
emitTerminal()
} else {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: err.Error(),
Content: "anthropic stream read failed",
Done: true,
}
}
return
}
if event.Done {
logUsage(ctx, model, usage)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
Usage: usage,
FinishReason: finishReason,
}
emitTerminal()
return
}
if len(event.Data) == 0 {
@@ -433,15 +644,15 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: fmt.Sprintf("decode SSE response: %v", err),
Content: "anthropic stream response was invalid",
Done: true,
}
return
}
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
if err := protocol.observe(&streamEvent); err != nil {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: streamEvent.Error.Message,
Content: "anthropic stream protocol failed",
Done: true,
}
return
@@ -450,9 +661,6 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
}
if streamEvent.Delta != nil {
if streamEvent.Delta.StopReason != "" {
finishReason = streamEvent.Delta.StopReason
}
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
@@ -0,0 +1,38 @@
package chat
import (
"context"
"io"
"net/http"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestAnthropicProviderErrorDoesNotExposeImagePayload(t *testing.T) {
secret := "TOPSECRETIMAGEPAYLOAD"
payload := "data: {\"type\":\"error\",\"error\":{\"message\":\"data:image/png;base64," + secret + "\"}}\n\n"
_, err := parseAnthropicSSE(strings.NewReader(payload))
if err == nil {
t.Fatal("expected Anthropic SSE error")
}
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
t.Fatalf("non-stream error leaked provider payload: %v", err)
}
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse)
go processAnthropicStream(context.Background(), "vision-test", resp, stream)
var terminal types.StreamResponse
for chunk := range stream {
terminal = chunk
}
if terminal.ResponseType != types.ResponseTypeError {
t.Fatalf("terminal response = %#v; want error", terminal)
}
if strings.Contains(terminal.Content, secret) || strings.Contains(strings.ToLower(terminal.Content), "data:image") {
t.Fatalf("stream error leaked provider payload: %q", terminal.Content)
}
}
@@ -0,0 +1,223 @@
package chat
import (
"context"
"io"
"net/http"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
const anthropicTestMessageStart = "data: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":1,\"output_tokens\":0}}}\n\n"
const anthropicTestBlockStart0 = "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"
const anthropicTestBlockDelta0 = "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n"
const anthropicTestBlockStop0 = "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n"
func invalidAnthropicBlockPayloads() map[string]string {
return map[string]string{
"stop before start": anthropicTestMessageStart + anthropicTestBlockStop0,
"delta before start": anthropicTestMessageStart + anthropicTestBlockDelta0,
"delta after stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockDelta0,
"duplicate start": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStart0,
"duplicate stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockStop0,
"mismatched index": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n",
"index gap": anthropicTestMessageStart + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
"concurrent block start": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
"mismatched delta type": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\"}}\n\n",
}
}
func TestAnthropicContentBlockLifecycleRejectedByBothParsers(t *testing.T) {
for name, payload := range invalidAnthropicBlockPayloads() {
t.Run(name+"/aggregate", func(t *testing.T) {
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
t.Fatal("aggregate parser accepted invalid content-block lifecycle")
}
})
t.Run(name+"/stream", func(t *testing.T) {
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 8)
go processAnthropicStream(context.Background(), "test", resp, stream)
sawError := false
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
t.Fatalf("stream accepted invalid content-block lifecycle: %#v", item)
}
if item.Done && item.ResponseType == types.ResponseTypeError {
sawError = true
}
}
if !sawError {
t.Fatal("stream did not emit an error terminal")
}
})
}
}
func TestAnthropicContentBlockLifecycleAcceptsSequentialBlocks(t *testing.T) {
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 + anthropicTestBlockStop0 +
"data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\n" +
"data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"thinking_delta\"}}\n\n" +
"data: {\"type\":\"content_block_stop\",\"index\":1}\n\n" +
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"data: {\"type\":\"message_stop\"}\n\n"
response, err := parseAnthropicSSE(strings.NewReader(payload))
if err != nil || response.FinishReason != "end_turn" || response.Content != "x" {
t.Fatalf("valid sequential blocks rejected: response=%#v err=%v", response, err)
}
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 8)
go processAnthropicStream(context.Background(), "test", resp, stream)
successTerminals := 0
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeError {
t.Fatalf("valid stream rejected: %#v", item)
}
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
successTerminals++
}
}
if successTerminals != 1 {
t.Fatalf("success terminals=%d, want 1", successTerminals)
}
}
func TestAnthropicProtocolAcceptsOfficialStopReasons(t *testing.T) {
reasons := []string{"end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded"}
for _, reason := range reasons {
t.Run(reason, func(t *testing.T) {
var state anthropicStreamProtocolState
start := anthropicStreamEvent{Type: "message_start"}
if err := state.observe(&start); err != nil {
t.Fatalf("message_start: %v", err)
}
delta := anthropicStreamEvent{Type: "message_delta"}
delta.Delta = &struct {
Type string `json:"type"`
Text string `json:"text"`
StopReason string `json:"stop_reason"`
}{StopReason: reason}
if err := state.observe(&delta); err != nil {
t.Fatalf("message_delta: %v", err)
}
stop := anthropicStreamEvent{Type: "message_stop"}
if err := state.observe(&stop); err != nil || !state.complete() {
t.Fatalf("message_stop err=%v complete=%v", err, state.complete())
}
})
}
}
func TestParseAnthropicSSERejectsPrematureEOF(t *testing.T) {
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
t.Fatal("expected premature Anthropic EOF to fail closed")
}
}
func TestParseAnthropicSSERejectsMalformedPacketAfterFinishReason(t *testing.T) {
payload := anthropicTestMessageStart + "event: message_delta\n" +
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"event: broken\n" +
"data: {not-json}\n\n"
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
t.Fatal("expected trailing malformed packet to override finish reason")
}
}
func TestParseAnthropicSSERejectsInvalidOrOutOfOrderCompletion(t *testing.T) {
cases := map[string]string{
"invalid stop reason": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"made_up\"}}\n\n" +
"data: {\"type\":\"message_stop\"}\n\n",
"message stop first": "data: {\"type\":\"message_stop\"}\n\n" +
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n",
"stop reason wrong event": anthropicTestMessageStart + "data: {\"type\":\"content_block_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"data: {\"type\":\"message_stop\"}\n\n",
"empty provider error after completion": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"data: {\"type\":\"message_stop\"}\n\n" +
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n",
}
for name, payload := range cases {
t.Run(name, func(t *testing.T) {
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
t.Fatal("expected invalid Anthropic completion protocol to fail")
}
})
}
}
func TestProcessAnthropicStreamRejectsPrematureDone(t *testing.T) {
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 +
"data: [DONE]\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 4)
go processAnthropicStream(context.Background(), "test", resp, stream)
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
t.Fatalf("premature [DONE] produced success: %#v", item)
}
}
}
func TestProcessAnthropicStreamRejectsEmptyProviderErrorAfterCompletion(t *testing.T) {
payload := anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"data: {\"type\":\"message_stop\"}\n\n" +
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 4)
go processAnthropicStream(context.Background(), "test", resp, stream)
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
t.Fatalf("empty-message provider error was ignored: %#v", item)
}
if item.Done && item.ResponseType == types.ResponseTypeError {
return
}
}
t.Fatal("expected provider error terminal")
}
func TestProcessAnthropicStreamRejectsPrematureEOF(t *testing.T) {
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 4)
go processAnthropicStream(context.Background(), "test", resp, stream)
successTerminals, errorTerminals := 0, 0
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
successTerminals++
}
if item.Done && item.ResponseType == types.ResponseTypeError {
errorTerminals++
}
}
if successTerminals != 0 || errorTerminals != 1 {
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
}
}
func TestProcessAnthropicStreamRejectsMalformedPacketAfterFinishReason(t *testing.T) {
payload := anthropicTestMessageStart + "event: message_delta\n" +
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
"event: broken\n" +
"data: {not-json}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
stream := make(chan types.StreamResponse, 4)
go processAnthropicStream(context.Background(), "test", resp, stream)
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
t.Fatalf("unexpected success terminal before trailing malformed packet: %#v", item)
}
if item.ResponseType == types.ResponseTypeError {
return
}
}
t.Fatal("expected terminal stream error")
}
+41 -2
View File
@@ -13,6 +13,33 @@ import (
"github.com/stretchr/testify/require"
)
func TestAnthropicBuildRequestPreservesImageDataURI(t *testing.T) {
chat := &AnthropicChat{modelName: "claude-test"}
req, err := chat.buildRequest([]Message{{
Role: "user",
Content: "describe this image",
Images: []string{"data:image/png;base64,YWJj"},
}}, nil)
require.NoError(t, err)
require.Len(t, req.Messages, 1)
blocks, ok := req.Messages[0].Content.([]anthropicContentBlock)
require.True(t, ok, "content type = %T", req.Messages[0].Content)
require.Len(t, blocks, 2)
assert.Equal(t, "text", blocks[0].Type)
assert.Equal(t, "describe this image", blocks[0].Text)
assert.Equal(t, "image", blocks[1].Type)
require.NotNil(t, blocks[1].Source)
assert.Equal(t, "base64", blocks[1].Source.Type)
assert.Equal(t, "image/png", blocks[1].Source.MediaType)
assert.Equal(t, "YWJj", blocks[1].Source.Data)
}
func TestAnthropicBuildRequestRejectsMalformedImage(t *testing.T) {
chat := &AnthropicChat{modelName: "claude-test"}
_, err := chat.buildRequest([]Message{{Role: "user", Content: "describe", Images: []string{"not-an-image"}}}, nil)
require.Error(t, err)
}
func TestAnthropicChat(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
@@ -147,8 +174,14 @@ func TestAnthropicChat_SSEResponse(t *testing.T) {
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
event: content_block_start
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
event: content_block_stop
data: {"type":"content_block_stop","index":0}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
@@ -191,8 +224,14 @@ func TestAnthropicChat_ChatStream(t *testing.T) {
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
event: content_block_start
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
event: content_block_stop
data: {"type":"content_block_stop","index":0}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
+36 -17
View File
@@ -98,14 +98,27 @@ type Chat interface {
GetModelID() string
}
type imageInputCapable interface {
SupportsImageInput() bool
}
// SupportsImageInput reports whether the concrete Chat transport can preserve
// image input. Unknown implementations fail closed instead of trusting model
// metadata alone.
func SupportsImageInput(model Chat) bool {
capable, ok := model.(imageInputCapable)
return ok && capable.SupportsImageInput()
}
type ChatConfig struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
Provider string
ExtraConfig map[string]string
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
Provider string
SupportsVision bool
ExtraConfig map[string]string
// CustomHeaders 允许在调用远程 OpenAI 兼容 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
CustomHeaders map[string]string
AppID string
@@ -120,17 +133,23 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) *ChatConfig {
if m == nil {
return nil
}
supportsVision := m.Parameters.SupportsVision
if provider.ProviderName(m.Parameters.Provider) == provider.ProviderWeKnoraCloud &&
m.Type == types.ModelTypeKnowledgeQA && strings.EqualFold(strings.TrimSpace(m.Name), "chat") {
supportsVision = true
}
return &ChatConfig{
ModelID: m.ID,
APIKey: m.Parameters.APIKey,
BaseURL: m.Parameters.BaseURL,
ModelName: m.Name,
Source: m.Source,
Provider: m.Parameters.Provider,
ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
AppID: appID,
AppSecret: appSecret,
ModelID: m.ID,
APIKey: m.Parameters.APIKey,
BaseURL: m.Parameters.BaseURL,
ModelName: m.Name,
Source: m.Source,
Provider: m.Parameters.Provider,
SupportsVision: supportsVision,
ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
AppID: appID,
AppSecret: appSecret,
}
}
-21
View File
@@ -74,24 +74,3 @@ func readLocalStorageBytes(storagePath string) []byte {
}
return data
}
// isMultimodalNotSupportedError checks if an error indicates the model does not
// support multimodal/image input.
func isMultimodalNotSupportedError(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return (strings.Contains(msg, "multimodal") || strings.Contains(msg, "image") || strings.Contains(msg, "vision")) &&
(strings.Contains(msg, "not support") || strings.Contains(msg, "unsupported") || strings.Contains(msg, "400"))
}
// stripImagesFromMessages returns a copy of messages with all image data removed.
func stripImagesFromMessages(messages []Message) []Message {
cleaned := make([]Message, len(messages))
for i, msg := range messages {
cleaned[i] = msg
cleaned[i].Images = nil
}
return cleaned
}
@@ -0,0 +1,26 @@
package chat
import (
"encoding/json"
"strings"
"testing"
)
func TestBuildLangfuseMessagesRedactsMultiContentImage(t *testing.T) {
secret := "TOPSECRETBASE64"
msgs := buildLangfuseMessages([]Message{{
Role: "user",
MultiContent: []MessageContentPart{
{Type: "image_url", ImageURL: &ImageURL{URL: "data:image/png;base64," + secret}},
{Type: "text", Text: "describe"},
},
}})
encoded, err := json.Marshal(msgs)
if err != nil {
t.Fatal(err)
}
serialized := string(encoded)
if strings.Contains(serialized, secret) || strings.Contains(strings.ToLower(serialized), "data:image") {
t.Fatalf("Langfuse input leaked image payload: %s", serialized)
}
}
+45 -11
View File
@@ -2,10 +2,12 @@ package chat
import (
"context"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
@@ -16,8 +18,9 @@ type langfuseChat struct {
inner Chat
}
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
func (l *langfuseChat) SupportsImageInput() bool { return SupportsImageInput(l.inner) }
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
mgr := langfuse.GetManager()
@@ -47,7 +50,7 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
)
}
gen.Finish(output, usage, err)
gen.Finish(output, usage, sanitizeLangfuseError(err))
return resp, err
}
@@ -71,7 +74,7 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
ch, err := l.inner.ChatStream(genCtx, messages, opts)
if err != nil {
gen.Finish(nil, nil, err)
gen.Finish(nil, nil, sanitizeLangfuseError(err))
return ch, err
}
if ch == nil {
@@ -131,10 +134,21 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
"role": m.Role,
}
if m.Content != "" {
entry["content"] = m.Content
entry["content"] = secutils.RedactDataURLsForLog(m.Content)
}
if len(m.MultiContent) > 0 {
entry["content"] = m.MultiContent
parts := make([]MessageContentPart, len(m.MultiContent))
copy(parts, m.MultiContent)
for i := range parts {
parts[i].Text = secutils.RedactDataURLsForLog(parts[i].Text)
if parts[i].ImageURL == nil {
continue
}
imageURL := *parts[i].ImageURL
imageURL.URL = "[image-url-redacted]"
parts[i].ImageURL = &imageURL
}
entry["content"] = parts
}
if m.Name != "" {
entry["name"] = m.Name
@@ -143,10 +157,15 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
entry["tool_call_id"] = m.ToolCallID
}
if len(m.ToolCalls) > 0 {
entry["tool_calls"] = m.ToolCalls
calls := make([]ToolCall, len(m.ToolCalls))
copy(calls, m.ToolCalls)
for i := range calls {
calls[i].Function.Arguments = secutils.RedactDataURLsForLog(calls[i].Function.Arguments)
}
entry["tool_calls"] = calls
}
if m.ReasoningContent != "" {
entry["reasoning_content"] = m.ReasoningContent
entry["reasoning_content"] = secutils.RedactDataURLsForLog(m.ReasoningContent)
}
out = append(out, entry)
}
@@ -157,17 +176,32 @@ func buildLangfuseGenerationOutput(
content, reasoningContent, finishReason string,
toolCalls []types.LLMToolCall,
) map[string]interface{} {
var safeToolCalls []types.LLMToolCall
if toolCalls != nil {
safeToolCalls = make([]types.LLMToolCall, len(toolCalls))
copy(safeToolCalls, toolCalls)
}
for i := range safeToolCalls {
safeToolCalls[i].Function.Arguments = secutils.RedactDataURLsForLog(safeToolCalls[i].Function.Arguments)
}
output := map[string]interface{}{
"content": content,
"tool_calls": toolCalls,
"content": secutils.RedactDataURLsForLog(content),
"tool_calls": safeToolCalls,
"finish_reason": finishReason,
}
if reasoningContent != "" {
output["reasoning_content"] = reasoningContent
output["reasoning_content"] = secutils.RedactDataURLsForLog(reasoningContent)
}
return output
}
func sanitizeLangfuseError(err error) error {
if err == nil {
return nil
}
return errors.New(secutils.RedactDataURLsForLog(err.Error()))
}
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
if opts == nil {
return nil
+1 -1
View File
@@ -29,7 +29,7 @@ func buildLLMMessages(messages []Message) []logger.LLMMessage {
parts = append(parts, mc.Text)
case "image_url":
if mc.ImageURL != nil {
parts = append(parts, fmt.Sprintf("[image_url: %s]", truncateForDebug(mc.ImageURL.URL, 120)))
parts = append(parts, fmt.Sprintf("[image_url: redacted (%d bytes)]", len(mc.ImageURL.URL)))
}
}
}
@@ -0,0 +1,27 @@
package chat
import (
"strings"
"testing"
)
func TestBuildLLMMessagesRedactsMultiContentImageURL(t *testing.T) {
secret := "TOPSECRETBASE64"
messages := buildLLMMessages([]Message{{
Role: "user",
MultiContent: []MessageContentPart{{
Type: "image_url",
ImageURL: &ImageURL{URL: "data:image/png;base64," + secret},
}},
}})
if len(messages) != 1 {
t.Fatalf("messages length = %d", len(messages))
}
got := messages[0].Content
if strings.Contains(got, secret) || strings.Contains(strings.ToLower(got), "data:image") {
t.Fatalf("buildLLMMessages leaked image URL: %q", got)
}
if !strings.Contains(got, "redacted") {
t.Fatalf("buildLLMMessages missing redaction marker: %q", got)
}
}
+3 -2
View File
@@ -16,8 +16,9 @@ type debugChat struct {
inner Chat
}
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
func (d *debugChat) SupportsImageInput() bool { return SupportsImageInput(d.inner) }
func (d *debugChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
callStart := time.Now()
@@ -0,0 +1,29 @@
package chat
import (
"io"
"strings"
"testing"
)
func TestOllamaBuildChatRequestRejectsAnyUnresolvableImage(t *testing.T) {
c := &OllamaChat{supportsVision: true}
_, err := c.buildChatRequest([]Message{{
Role: "user",
Content: "describe",
Images: []string{"data:image/png;base64,%%%INVALID%%%"},
}}, nil, false)
if err == nil {
t.Fatal("expected malformed image to fail closed")
}
if strings.Contains(err.Error(), "%%%INVALID%%%") {
t.Fatalf("image value leaked in error: %v", err)
}
}
func TestReadAnthropicResponseBodyRejectsOversize(t *testing.T) {
_, err := readAnthropicResponseBody(io.NopCloser(strings.NewReader(strings.Repeat("x", anthropicResponseBodyLimit+1))))
if err == nil {
t.Fatal("expected oversized Anthropic response rejection")
}
}
+77 -41
View File
@@ -3,7 +3,7 @@ package chat
import (
"context"
"encoding/json"
"fmt"
"errors"
"io"
"net/http"
"strconv"
@@ -13,27 +13,33 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
ollamaapi "github.com/ollama/ollama/api"
)
// OllamaChat 实现了基于 Ollama 的聊天
type OllamaChat struct {
modelName string
modelID string
ollamaService *ollama.OllamaService
modelName string
modelID string
supportsVision bool
ollamaService *ollama.OllamaService
}
func (c *OllamaChat) SupportsImageInput() bool { return c.supportsVision }
// NewOllamaChat 创建 Ollama 聊天实例
func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*OllamaChat, error) {
return &OllamaChat{
modelName: config.ModelName,
modelID: config.ModelID,
ollamaService: ollamaService,
modelName: config.ModelName,
modelID: config.ModelID,
supportsVision: config.SupportsVision,
ollamaService: ollamaService,
}, nil
}
// convertMessages 转换消息格式为Ollama API格式
func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message {
// convertMessages converts messages and fails closed if any declared image
// cannot be resolved. Silently dropping one image would change the user's input.
func (c *OllamaChat) convertMessages(messages []Message) ([]ollamaapi.Message, error) {
ollamaMessages := make([]ollamaapi.Message, 0, len(messages))
for _, msg := range messages {
msgOllama := ollamaapi.Message{
@@ -46,47 +52,69 @@ func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message {
}
if len(msg.Images) > 0 && msg.Role == "user" {
for _, imgURL := range msg.Images {
if imgData := resolveImageForOllama(imgURL); imgData != nil {
msgOllama.Images = append(msgOllama.Images, imgData)
imgData, err := resolveImageForOllama(imgURL)
if err != nil {
return nil, err
}
msgOllama.Images = append(msgOllama.Images, imgData)
}
}
ollamaMessages = append(ollamaMessages, msgOllama)
}
return ollamaMessages
return ollamaMessages, nil
}
// resolveImageForOllama resolves an image URL into raw bytes for Ollama.
// Handles local serving paths (/files/...), data URIs, and remote HTTP URLs.
func resolveImageForOllama(imageURL string) ollamaapi.ImageData {
func resolveImageForOllama(imageURL string) (ollamaapi.ImageData, error) {
if data := resolveImageURLForOllama(imageURL); data != nil {
return data
return data, nil
}
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
client := &http.Client{Timeout: 30 * time.Second}
if err := secutils.ValidateURLForSSRF(imageURL); err != nil {
return nil, errors.New("ollama image URL is not allowed")
}
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
Proxy: nil,
DialContext: secutils.SSRFSafeDialContext,
},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Get(imageURL)
if err != nil {
return nil
return nil, errors.New("ollama image download failed")
}
defer resp.Body.Close()
data, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024))
if err != nil {
return nil
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return nil, errors.New("ollama image download failed")
}
return data
const maxOllamaImageBytes = 20 * 1024 * 1024
data, err := io.ReadAll(io.LimitReader(resp.Body, maxOllamaImageBytes+1))
if err != nil {
return nil, errors.New("ollama image download failed")
}
if len(data) == 0 || len(data) > maxOllamaImageBytes {
return nil, errors.New("ollama image payload is invalid")
}
return data, nil
}
return nil
return nil, errors.New("ollama image payload is invalid")
}
// buildChatRequest 构建聊天请求参数
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) *ollamaapi.ChatRequest {
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) (*ollamaapi.ChatRequest, error) {
// 设置流式标志
streamFlag := isStream
// 构建请求参数
convertedMessages, err := c.convertMessages(messages)
if err != nil {
return nil, err
}
chatReq := &ollamaapi.ChatRequest{
Model: c.modelName,
Messages: c.convertMessages(messages),
Messages: convertedMessages,
Stream: &streamFlag,
Options: make(map[string]interface{}),
}
@@ -113,7 +141,7 @@ func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isS
}
}
return chatReq
return chatReq, nil
}
// Chat 进行非流式聊天
@@ -124,7 +152,10 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
}
// 构建请求参数
chatReq := c.buildChatRequest(messages, opts, false)
chatReq, err := c.buildChatRequest(messages, opts, false)
if err != nil {
return nil, err
}
// 记录请求日志
logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName)
@@ -134,7 +165,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
var promptTokens, completionTokens int
// 使用 Ollama 客户端发送请求
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
err = c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
responseContent = resp.Message.Content
// 当 Content 为空但 Thinking 有内容时(如推理模型未正确配置 thinking 参数),使用 Thinking 作为兜底
if responseContent == "" && resp.Message.Thinking != "" {
@@ -151,7 +182,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
return nil
})
if err != nil {
return nil, fmt.Errorf("聊天请求失败: %w", err)
return nil, errors.New("ollama chat failed")
}
usage := types.TokenUsage{
@@ -180,7 +211,10 @@ func (c *OllamaChat) ChatStream(
}
// 构建请求参数
chatReq := c.buildChatRequest(messages, opts, true)
chatReq, err := c.buildChatRequest(messages, opts, true)
if err != nil {
return nil, err
}
// 记录请求日志
logger.GetLogger(ctx).Infof("发送流式聊天请求到模型 %s", c.modelName)
@@ -193,6 +227,8 @@ func (c *OllamaChat) ChatStream(
defer close(streamChan)
var thinking thinkingEmitter
var terminalUsage *types.TokenUsage
providerDone := false
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
// 发送思考内容(支持 Qwen3、DeepSeek 等推理模型)
if resp.Message.Thinking != "" {
@@ -247,32 +283,32 @@ func (c *OllamaChat) ChatStream(
}
if resp.Done {
var usage *types.TokenUsage
providerDone = true
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
usage = &types.TokenUsage{
terminalUsage = &types.TokenUsage{
PromptTokens: resp.PromptEvalCount,
CompletionTokens: resp.EvalCount,
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
}
}
logUsage(ctx, c.modelName, usage)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Done: true,
Usage: usage,
}
}
return nil
})
if err != nil {
logger.GetLogger(ctx).Errorf("流式聊天请求失败: %v", err)
// 发送错误响应
if err != nil || !providerDone {
logger.GetLogger(ctx).Error("流式聊天请求失败")
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: err.Error(),
Content: "ollama stream failed",
Done: true,
}
return
}
logUsage(ctx, c.modelName, terminalUsage)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Done: true,
Usage: terminalUsage,
}
}()
+36 -38
View File
@@ -122,7 +122,7 @@ func (c *RemoteAPIChat) processStream(
for {
response, err := stream.Recv()
if err != nil {
if err == io.EOF {
if err == io.EOF && state.lastFinishReason != "" {
logUsage(ctx, c.modelName, state.usage)
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
@@ -134,9 +134,10 @@ func (c *RemoteAPIChat) processStream(
FinishReason: state.lastFinishReason,
}
} else {
logger.Warnf(ctx, "Chat completion stream ended before finish reason")
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: err.Error(),
Content: "chat completion stream failed",
Done: true,
}
}
@@ -178,23 +179,26 @@ func (c *RemoteAPIChat) processRawHTTPStream(
for {
event, err := reader.ReadEvent()
if err != nil {
if err == io.EOF {
if err == io.EOF && state.lastFinishReason != "" {
logUsage(ctx, c.modelName, state.usage)
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
ToolCalls: toolCalls,
ToolCalls: state.buildOrderedToolCalls(),
Usage: state.usage,
FinishReason: state.lastFinishReason,
}
return
}
if err != io.EOF {
logger.Warnf(ctx, "Chat stream read failed")
} else {
logger.Errorf(ctx, "Stream read error: %v", err)
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: err.Error(),
Done: true,
}
logger.Warnf(ctx, "Chat stream ended before finish reason")
}
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: "chat stream ended before completion",
Done: true,
}
return
}
@@ -204,6 +208,14 @@ func (c *RemoteAPIChat) processRawHTTPStream(
}
if event.Done {
if state.lastFinishReason == "" {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: "chat stream ended before completion",
Done: true,
}
return
}
logUsage(ctx, c.modelName, state.usage)
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
@@ -212,6 +224,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(
Done: true,
ToolCalls: toolCalls,
Usage: state.usage,
FinishReason: state.lastFinishReason,
}
return
}
@@ -241,8 +254,13 @@ func (c *RemoteAPIChat) processRawHTTPStream(
}
if err := json.Unmarshal(event.Data, &streamResp); err != nil {
logger.Errorf(ctx, "Failed to parse stream response: %v", err)
continue
logger.Warnf(ctx, "Failed to parse chat stream response")
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: "chat stream response was invalid",
Done: true,
}
return
}
if streamResp.Usage != nil {
@@ -453,38 +471,18 @@ func (c *RemoteAPIChat) processStreamDelta(
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: delta.Content,
Done: isDone,
Done: false,
ToolCalls: state.buildOrderedToolCalls(),
FinishReason: string(choice.FinishReason),
}
}
if isDone && len(state.toolCallMap) > 0 {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
ToolCalls: state.buildOrderedToolCalls(),
FinishReason: string(choice.FinishReason),
}
}
// Ensure thinking done is sent when stream finishes without any answer content
// (e.g., model only produced reasoning then hit finish_reason with empty content).
// Ensure thinking done is sent when the provider supplied a finish reason.
// The outer stream reader emits the sole terminal answer after it confirms
// the provider ended successfully.
if isDone {
state.finish(streamChan)
}
// Catch-all: isDone but none of the above branches sent a response with
// FinishReason (empty content, no tool calls, no thinking). This prevents
// the finish_reason from being lost in the streaming pipeline.
if isDone && delta.Content == "" && len(state.toolCallMap) == 0 {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Done: true,
FinishReason: string(choice.FinishReason),
}
}
}
// processToolCallsDelta 处理 tool calls 的增量更新
@@ -0,0 +1,129 @@
package chat
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestRemoteAPIProviderErrorDoesNotExposeImagePayload(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
secret := "TOPSECRETIMAGEPAYLOAD"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"error":{"message":"data:image/png;base64,` + secret + `"}}`))
}))
defer server.Close()
model, err := NewRemoteAPIChat(&ChatConfig{
Source: types.ModelSourceRemote,
BaseURL: server.URL + "/v1",
ModelName: "vision-test",
APIKey: "test-key",
Provider: "openai",
SupportsVision: true,
})
if err != nil {
t.Fatal(err)
}
_, err = model.Chat(context.Background(), []Message{{
Role: "user", Content: "inspect", Images: []string{"data:image/png;base64," + secret},
}}, nil)
if err == nil {
t.Fatal("expected provider error")
}
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
t.Fatalf("provider error leaked image payload: %v", err)
}
}
func TestProcessRawHTTPStreamPrematureEOFFailsClosed(t *testing.T) {
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
stream := make(chan types.StreamResponse)
model := &RemoteAPIChat{modelName: "cloud-vision"}
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
var got []types.StreamResponse
for chunk := range stream {
got = append(got, chunk)
}
if len(got) == 0 || got[len(got)-1].ResponseType != types.ResponseTypeError {
t.Fatalf("stream = %#v; want terminal error on premature EOF", got)
}
if got[len(got)-1].Content != "chat stream ended before completion" {
t.Fatalf("terminal error = %q", got[len(got)-1].Content)
}
}
func TestProcessRawHTTPStreamFinishReasonThenEOFCompletes(t *testing.T) {
body := "data: {\"choices\":[{\"delta\":{\"content\":\"answer\"},\"finish_reason\":\"stop\"}]}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
stream := make(chan types.StreamResponse)
model := &RemoteAPIChat{modelName: "cloud-vision"}
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
var got []types.StreamResponse
for chunk := range stream {
got = append(got, chunk)
}
if len(got) == 0 {
t.Fatal("expected stream chunks")
}
last := got[len(got)-1]
if last.ResponseType == types.ResponseTypeError || !last.Done || last.FinishReason != "stop" {
t.Fatalf("stream = %#v; want successful stop completion", got)
}
}
func TestProcessRawHTTPStreamFinishReasonThenMalformedPacketFailsWithoutSuccessTerminal(t *testing.T) {
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n" +
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
"data: {not-json}\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
stream := make(chan types.StreamResponse, 8)
model := &RemoteAPIChat{modelName: "stream-test"}
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
successTerminals, errorTerminals := 0, 0
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
successTerminals++
}
if item.Done && item.ResponseType == types.ResponseTypeError {
errorTerminals++
}
}
if successTerminals != 0 || errorTerminals != 1 {
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
}
}
func TestProcessRawHTTPStreamEmitsExactlyOneSuccessTerminal(t *testing.T) {
body := "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\n" +
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
"data: [DONE]\n\n"
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
stream := make(chan types.StreamResponse, 8)
model := &RemoteAPIChat{modelName: "stream-test"}
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
successTerminals := 0
for item := range stream {
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
successTerminals++
}
if item.ResponseType == types.ResponseTypeError {
t.Fatalf("unexpected stream error: %#v", item)
}
}
if successTerminals != 1 {
t.Fatalf("success terminals = %d, want 1", successTerminals)
}
}
+31 -35
View File
@@ -21,14 +21,15 @@ import (
// providerAdapter(见 provider.go),thinking 编码委托给 ThinkingStrategy
// (见 thinking.go)。
type RemoteAPIChat struct {
modelName string
client *openai.Client
modelID string
baseURL string
apiKey string
provider provider.ProviderName
appID string
appSecret string
modelName string
client *openai.Client
modelID string
baseURL string
apiKey string
provider provider.ProviderName
supportsVision bool
appID string
appSecret string
// customHeaders 为用户在模型配置中指定的自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
customHeaders map[string]string
@@ -38,6 +39,8 @@ type RemoteAPIChat struct {
thinkingOverride ThinkingStrategy
}
func (c *RemoteAPIChat) SupportsImageInput() bool { return c.supportsVision }
// NewRemoteAPIChat 创建远程 API 聊天实例
func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
if chatConfig.BaseURL != "" {
@@ -103,6 +106,7 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
baseURL: chatConfig.BaseURL,
apiKey: apiKey,
provider: providerName,
supportsVision: chatConfig.SupportsVision,
appID: chatConfig.AppID,
appSecret: chatConfig.AppSecret,
customHeaders: chatConfig.CustomHeaders,
@@ -120,7 +124,11 @@ func (c *RemoteAPIChat) authCreds() authCreds {
// transform and parameter shaping (but not thinking, which may wrap the body).
func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest {
req := c.BuildChatCompletionRequest(messages, opts, isStream)
req.Messages = c.adapter.TransformMessages(req.Messages)
// WeKnora Cloud text models retain the legacy plain-text downgrade, while
// vision-capable Chat models must keep the OpenAI-compatible image parts.
if c.provider != provider.ProviderWeKnoraCloud || !c.supportsVision {
req.Messages = c.adapter.TransformMessages(req.Messages)
}
c.adapter.ShapeRequest(&req, opts, isStream)
return req
}
@@ -180,15 +188,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
c.logRequest(timeoutCtx, req, false)
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
if err != nil {
if isMultimodalNotSupportedError(err) {
logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
cleaned := stripImagesFromMessages(messages)
req = c.shapedRequest(cleaned, opts, false)
resp, err = c.client.CreateChatCompletion(timeoutCtx, req)
}
if err != nil {
return nil, fmt.Errorf("create chat completion: %w", err)
}
return nil, fmt.Errorf("create chat completion failed")
}
result, err := c.parseCompletionResponse(&resp)
@@ -231,13 +231,13 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
return nil, fmt.Errorf("send chat request failed")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
return nil, remoteAPIStatusError(resp.StatusCode, body)
}
body, err := io.ReadAll(resp.Body)
@@ -259,6 +259,10 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
return result, nil
}
func remoteAPIStatusError(status int, _ []byte) error {
return fmt.Errorf("API request failed with status %d", status)
}
// ChatStream 进行流式聊天
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
// 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长,
@@ -287,17 +291,9 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
if err != nil {
if isMultimodalNotSupportedError(err) {
logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
cleaned := stripImagesFromMessages(messages)
req = c.shapedRequest(cleaned, opts, true)
stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req)
}
if err != nil {
cancel()
close(streamChan)
return nil, fmt.Errorf("create chat completion stream: %w", err)
}
cancel()
close(streamChan)
return nil, fmt.Errorf("create chat completion stream failed")
}
go func() {
@@ -363,13 +359,13 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
return nil, fmt.Errorf("send chat request failed")
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
resp.Body.Close()
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
return nil, remoteAPIStatusError(resp.StatusCode, body)
}
streamChan := make(chan types.StreamResponse)
+56
View File
@@ -3,7 +3,11 @@ package chat
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
@@ -13,6 +17,17 @@ import (
"github.com/stretchr/testify/require"
)
func TestRemoteAPIStatusErrorDoesNotExposeResponseBody(t *testing.T) {
secret := "data:image/png;base64,TOPSECRET"
err := remoteAPIStatusError(http.StatusBadRequest, []byte(secret))
if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "TOPSECRET") {
t.Fatalf("remote API error leaked response body: %v", err)
}
if !strings.Contains(err.Error(), "400") {
t.Fatalf("remote API error = %v, want status code", err)
}
}
func newTestRemoteChat(t *testing.T) *RemoteAPIChat {
t.Helper()
@@ -533,3 +548,44 @@ func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) {
assert.Contains(t, string(b), `"cached_tokens":7`)
})
}
func newMultimodalUnsupportedTestChat(t *testing.T, calls *atomic.Int32) *RemoteAPIChat {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"error":{"message":"image input is not supported","type":"invalid_request_error","code":"unsupported"}}`))
}))
t.Cleanup(server.Close)
chat := newTestRemoteChat(t)
cfg := openai.DefaultConfig("test-key")
cfg.BaseURL = server.URL
chat.client = openai.NewClientWithConfig(cfg)
return chat
}
func TestRemoteAPIChat_MultimodalUnsupportedDoesNotStripImages(t *testing.T) {
messages := []Message{{
Role: "user",
Content: "read this image",
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
}}
t.Run("non-streaming", func(t *testing.T) {
var calls atomic.Int32
chat := newMultimodalUnsupportedTestChat(t, &calls)
_, err := chat.Chat(context.Background(), messages, &ChatOptions{})
require.Error(t, err)
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
})
t.Run("streaming", func(t *testing.T) {
var calls atomic.Int32
chat := newMultimodalUnsupportedTestChat(t, &calls)
_, err := chat.ChatStream(context.Background(), messages, &ChatOptions{})
require.Error(t, err)
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
})
}
+23 -4
View File
@@ -8,6 +8,8 @@ import (
"strings"
"sync"
"time"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// streamRawDumpDir returns the directory for per-stream raw packet dumps.
@@ -43,7 +45,10 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
if dir == "" {
return nil
}
if err := os.MkdirAll(dir, 0o755); err != nil {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil
}
if err := os.Chmod(dir, 0o700); err != nil {
return nil
}
@@ -70,10 +75,14 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
return nil
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return nil
}
if err := f.Chmod(0o600); err != nil {
_ = f.Close()
return nil
}
d := &streamPacketDumper{file: f, path: path, model: modelName}
_ = d.writeRequest(request)
@@ -81,11 +90,19 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
}
func (d *streamPacketDumper) writeRequest(request any) error {
raw, err := json.Marshal(request)
if err != nil {
return err
}
var redacted any
if err := json.Unmarshal([]byte(secutils.CompactImageDataURLForLog(string(raw))), &redacted); err != nil {
return err
}
line, err := json.Marshal(map[string]any{
"type": "request",
"model": d.model,
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
"data": request,
"data": redacted,
})
if err != nil {
return err
@@ -112,6 +129,7 @@ func (d *streamPacketDumper) WritePacketRaw(raw []byte) {
if len(raw) == 0 {
return
}
raw = []byte(secutils.CompactImageDataURLForLog(string(raw)))
d.mu.Lock()
defer d.mu.Unlock()
@@ -139,6 +157,7 @@ func (d *streamPacketDumper) WriteError(message string) {
if d == nil || d.file == nil || strings.TrimSpace(message) == "" {
return
}
message = secutils.CompactImageDataURLForLog(message)
line, err := json.Marshal(map[string]any{
"type": "error",
"model": d.model,
@@ -163,7 +182,7 @@ func (d *streamPacketDumper) WriteHTTPError(statusCode int, body []byte) {
"status_code": statusCode,
}
if len(body) > 0 {
trimmed := bytesTrimSpace(body)
trimmed := []byte(secutils.CompactImageDataURLForLog(string(bytesTrimSpace(body))))
if json.Valid(trimmed) {
var parsed any
if json.Unmarshal(trimmed, &parsed) == nil {
@@ -0,0 +1,51 @@
package chat
import (
"os"
"strings"
"testing"
)
func TestStreamPacketDumperRedactsImageRequestAndUsesPrivatePermissions(t *testing.T) {
dir := t.TempDir()
if err := os.Chmod(dir, 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("WEKNORA_LLM_STREAM_RAW_DUMP_DIR", dir)
secret := strings.Repeat("TOPSECRET", 30)
dumper := newStreamPacketDumper("vision-model", map[string]any{
"messages": []any{map[string]any{
"role": "user",
"image_url": map[string]any{"url": "data:image/png;base64," + secret},
}},
})
if dumper == nil {
t.Fatal("expected dumper")
}
path := dumper.Path()
dumper.Close()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read dump: %v", err)
}
if strings.Contains(string(data), secret) || strings.Contains(string(data), "data:image/") {
t.Fatalf("image data leaked to dump: %s", data)
}
if !strings.Contains(string(data), "[image-data-redacted]") {
t.Fatalf("missing redaction marker: %s", data)
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat dump: %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Fatalf("dump permissions = %o, want 600", got)
}
dirInfo, err := os.Stat(dir)
if err != nil {
t.Fatalf("stat dump directory: %v", err)
}
if got := dirInfo.Mode().Perm(); got != 0o700 {
t.Fatalf("dump directory permissions = %o, want 700", got)
}
}
@@ -0,0 +1,80 @@
package chat
import (
"testing"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
func TestRemoteAPIChat_WeKnoraCloudTransformsImagesByCapability(t *testing.T) {
messages := []Message{{
Role: "user",
Content: "请识别图片",
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
}}
build := func(t *testing.T, supportsVision bool) *RemoteAPIChat {
t.Helper()
model, err := NewRemoteAPIChat(&ChatConfig{
Source: types.ModelSourceRemote,
ModelName: "cloud-chat",
ModelID: "cloud-chat",
Provider: string(provider.ProviderWeKnoraCloud),
AppID: "test-app",
AppSecret: "test-secret",
SupportsVision: supportsVision,
})
require.NoError(t, err)
return model
}
t.Run("vision chat preserves image parts", func(t *testing.T) {
req := build(t, true).shapedRequest(messages, &ChatOptions{}, false)
require.Len(t, req.Messages, 1)
require.Len(t, req.Messages[0].MultiContent, 2)
require.NotNil(t, req.Messages[0].MultiContent[0].ImageURL)
require.Equal(t, messages[0].Images[0], req.Messages[0].MultiContent[0].ImageURL.URL)
})
t.Run("text chat keeps legacy downgrade", func(t *testing.T) {
req := build(t, false).shapedRequest(messages, &ChatOptions{}, false)
require.Len(t, req.Messages, 1)
require.Empty(t, req.Messages[0].MultiContent)
require.Equal(t, messages[0].Content, req.Messages[0].Content)
})
}
func TestConfigFromModelRecognizesExistingWeKnoraCloudChatAsVision(t *testing.T) {
cloudChat := &types.Model{
Name: "chat",
Type: types.ModelTypeKnowledgeQA,
Source: types.ModelSourceRemote,
Parameters: types.ModelParameters{
Provider: string(provider.ProviderWeKnoraCloud),
},
}
genericChat := &types.Model{
Name: "chat",
Type: types.ModelTypeKnowledgeQA,
Source: types.ModelSourceRemote,
}
require.True(t, ConfigFromModel(cloudChat, "", "").SupportsVision)
require.False(t, ConfigFromModel(genericChat, "", "").SupportsVision)
}
func TestSupportsImageInputUsesRuntimeTransportCapability(t *testing.T) {
visionRemote := &RemoteAPIChat{supportsVision: true}
textRemote := &RemoteAPIChat{supportsVision: false}
anthropicText := &AnthropicChat{}
anthropicVision := &AnthropicChat{supportsVision: true}
require.True(t, SupportsImageInput(visionRemote))
require.False(t, SupportsImageInput(textRemote))
require.False(t, SupportsImageInput(anthropicText))
require.True(t, SupportsImageInput(anthropicVision))
require.True(t, SupportsImageInput(&debugChat{inner: visionRemote}))
require.True(t, SupportsImageInput(&langfuseChat{inner: visionRemote}))
}