From 4d2b8707ffa688a78e4b313baa5cdbb1ba800e81 Mon Sep 17 00:00:00 2001 From: langcaiye Date: Tue, 12 May 2026 19:24:28 +0800 Subject: [PATCH] fix: support Anthropic gateway streaming --- internal/models/chat/anthropic.go | 255 ++++++++++++++++++++++++- internal/models/chat/anthropic_test.go | 165 +++++++++++++++- 2 files changed, 417 insertions(+), 3 deletions(-) diff --git a/internal/models/chat/anthropic.go b/internal/models/chat/anthropic.go index 7fb13e5f..01568526 100644 --- a/internal/models/chat/anthropic.go +++ b/internal/models/chat/anthropic.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "github.com/Tencent/WeKnora/internal/models/provider" @@ -32,6 +33,7 @@ type anthropicMessage struct { type anthropicRequest struct { Model string `json:"model"` MaxTokens int `json:"max_tokens"` + Stream bool `json:"stream,omitempty"` System string `json:"system,omitempty"` Messages []anthropicMessage `json:"messages"` Temperature *float64 `json:"temperature,omitempty"` @@ -57,6 +59,33 @@ type anthropicResponse struct { } `json:"error,omitempty"` } +type anthropicStreamEvent struct { + Type string `json:"type"` + Message *struct { + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + } `json:"usage"` + } `json:"message,omitempty"` + Delta *struct { + Type string `json:"type"` + Text string `json:"text"` + StopReason string `json:"stop_reason"` + } `json:"delta,omitempty"` + Usage *struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + } `json:"usage,omitempty"` + Error *struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) { if config.BaseURL != "" { if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil { @@ -91,7 +120,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat ctx, cancel := withLLMTimeout(ctx, defaultChatTimeout) defer cancel() - endpoint := c.baseURL + "/messages" + endpoint := c.endpoint() if err := secutils.ValidateURLForSSRF(endpoint); err != nil { return nil, fmt.Errorf("endpoint SSRF check failed: %w", err) } @@ -116,6 +145,17 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat return nil, fmt.Errorf("read response: %w", err) } + if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") { + chatResp, err := parseAnthropicSSE(bytes.NewReader(body)) + if err != nil { + return nil, err + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content) + } + return chatResp, nil + } + var chatResp anthropicResponse if err := json.Unmarshal(body, &chatResp); err != nil { return nil, fmt.Errorf("decode response: %w", err) @@ -131,7 +171,41 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat } func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) { - return nil, fmt.Errorf("Anthropic streaming chat is not supported yet") + reqBody := c.buildRequest(messages, opts) + reqBody.Stream = true + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + endpoint := c.endpoint() + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return nil, fmt.Errorf("endpoint SSRF check failed: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "text/event-stream") + httpReq.Header.Set("x-api-key", c.apiKey) + httpReq.Header.Set("anthropic-version", anthropicVersion) + secutils.ApplyCustomHeaders(httpReq, c.customHeaders) + + resp, err := rawHTTPClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("send request: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + streamChan := make(chan types.StreamResponse) + go processAnthropicStream(resp, streamChan) + return streamChan, nil } func (c *AnthropicChat) GetModelName() string { @@ -142,6 +216,35 @@ func (c *AnthropicChat) GetModelID() string { return c.modelID } +func (c *AnthropicChat) endpoint() string { + baseURL := strings.TrimRight(c.baseURL, "/") + if isAnthropicMessagesEndpoint(baseURL) { + return baseURL + } + if isAnthropicVersionedBaseURL(baseURL) { + return baseURL + "/messages" + } + return baseURL + "/v1/messages" +} + +func isAnthropicMessagesEndpoint(baseURL string) bool { + u, err := url.Parse(baseURL) + if err != nil { + return false + } + path := strings.TrimRight(u.Path, "/") + return strings.HasSuffix(path, "/messages") +} + +func isAnthropicVersionedBaseURL(baseURL string) bool { + u, err := url.Parse(baseURL) + if err != nil { + return false + } + path := strings.TrimRight(u.Path, "/") + return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta") +} + func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest { req := anthropicRequest{ Model: c.modelName, @@ -220,3 +323,151 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon }, } } + +func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) { + sseReader := NewSSEReader(reader) + var contentParts []string + var finishReason string + var inputTokens int + var outputTokens int + + for { + event, err := sseReader.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read SSE response: %w", err) + } + if event.Done { + break + } + if len(event.Data) == 0 { + continue + } + + var streamEvent anthropicStreamEvent + if err := json.Unmarshal(event.Data, &streamEvent); err != nil { + return nil, fmt.Errorf("decode SSE response: %w", err) + } + if streamEvent.Error != nil && streamEvent.Error.Message != "" { + return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message) + } + if streamEvent.Message != nil { + inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens) + outputTokens = max(outputTokens, streamEvent.Message.Usage.OutputTokens) + } + if streamEvent.Delta != nil { + if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" { + contentParts = append(contentParts, streamEvent.Delta.Text) + } + if streamEvent.Delta.StopReason != "" { + finishReason = streamEvent.Delta.StopReason + } + } + if streamEvent.Usage != nil { + inputTokens = max(inputTokens, streamEvent.Usage.InputTokens) + outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens) + } + } + + return &types.ChatResponse{ + Content: strings.Join(contentParts, ""), + FinishReason: finishReason, + Usage: types.TokenUsage{ + PromptTokens: inputTokens, + CompletionTokens: outputTokens, + TotalTokens: inputTokens + outputTokens, + }, + }, nil +} + +func processAnthropicStream(resp *http.Response, streamChan chan types.StreamResponse) { + defer close(streamChan) + defer resp.Body.Close() + + sseReader := NewSSEReader(resp.Body) + var usage *types.TokenUsage + var finishReason string + + for { + event, err := sseReader.ReadEvent() + if err != nil { + if err == io.EOF { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + Usage: usage, + FinishReason: finishReason, + } + } else { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeError, + Content: err.Error(), + Done: true, + } + } + return + } + if event.Done { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + Usage: usage, + FinishReason: finishReason, + } + return + } + if len(event.Data) == 0 { + continue + } + + var streamEvent anthropicStreamEvent + if err := json.Unmarshal(event.Data, &streamEvent); err != nil { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeError, + Content: fmt.Sprintf("decode SSE response: %v", err), + Done: true, + } + return + } + if streamEvent.Error != nil && streamEvent.Error.Message != "" { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeError, + Content: streamEvent.Error.Message, + Done: true, + } + return + } + if streamEvent.Message != nil { + usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens) + } + if streamEvent.Delta != nil { + if streamEvent.Delta.StopReason != "" { + finishReason = streamEvent.Delta.StopReason + } + if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: streamEvent.Delta.Text, + Done: false, + } + } + } + if streamEvent.Usage != nil { + usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens, streamEvent.Usage.OutputTokens) + } + } +} + +func mergeAnthropicUsage(current *types.TokenUsage, inputTokens, outputTokens int) *types.TokenUsage { + if current == nil { + current = &types.TokenUsage{} + } + current.PromptTokens = max(current.PromptTokens, inputTokens) + current.CompletionTokens = max(current.CompletionTokens, outputTokens) + current.TotalTokens = current.PromptTokens + current.CompletionTokens + return current +} diff --git a/internal/models/chat/anthropic_test.go b/internal/models/chat/anthropic_test.go index 277ff178..f188177f 100644 --- a/internal/models/chat/anthropic_test.go +++ b/internal/models/chat/anthropic_test.go @@ -20,7 +20,7 @@ func TestAnthropicChat(t *testing.T) { var capturedRequest anthropicRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/messages", r.URL.Path) + assert.Equal(t, "/v1/messages", r.URL.Path) capturedHeaders = r.Header.Clone() require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest)) @@ -70,6 +70,169 @@ func TestAnthropicChat(t *testing.T) { assert.Equal(t, 5, resp.Usage.TotalTokens) } +func TestAnthropicChat_FullEndpoint(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + + var capturedPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id":"msg_123", + "type":"message", + "role":"assistant", + "content":[{"type":"text","text":"hello"}], + "stop_reason":"end_turn", + "usage":{"input_tokens":3,"output_tokens":2} + }`)) + })) + defer server.Close() + + chat, err := NewAnthropicChat(&ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: server.URL + "/api/proxy/forward", + ModelName: "gpt-5.5", + APIKey: "test-key", + Provider: string(provider.ProviderAnthropic), + }) + require.NoError(t, err) + + resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil) + require.NoError(t, err) + + assert.Equal(t, "/api/proxy/forward/v1/messages", capturedPath) + assert.Equal(t, "hello", resp.Content) +} + +func TestAnthropicChat_MessagesEndpoint(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + + var capturedPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id":"msg_123", + "type":"message", + "role":"assistant", + "content":[{"type":"text","text":"hello"}], + "stop_reason":"end_turn", + "usage":{"input_tokens":3,"output_tokens":2} + }`)) + })) + defer server.Close() + + chat, err := NewAnthropicChat(&ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: server.URL + "/api/v1/messages", + ModelName: "gpt-5.5", + APIKey: "test-key", + Provider: string(provider.ProviderAnthropic), + }) + require.NoError(t, err) + + resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil) + require.NoError(t, err) + + assert.Equal(t, "/api/v1/messages", capturedPath) + assert.Equal(t, "hello", resp.Content) +} + +func TestAnthropicChat_SSEResponse(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/v1/messages", r.URL.Path) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(`event: message_start +data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_delta +data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}} + +event: message_stop +data: {"type":"message_stop"} + +`)) + })) + defer server.Close() + + chat, err := NewAnthropicChat(&ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: server.URL, + ModelName: "gpt-5.5", + APIKey: "test-key", + Provider: string(provider.ProviderAnthropic), + }) + require.NoError(t, err) + + resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil) + require.NoError(t, err) + + assert.Equal(t, "pong", resp.Content) + assert.Equal(t, "end_turn", resp.FinishReason) + assert.Equal(t, 114, resp.Usage.PromptTokens) + assert.Equal(t, 5, resp.Usage.CompletionTokens) + assert.Equal(t, 119, resp.Usage.TotalTokens) +} + +func TestAnthropicChat_ChatStream(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + + var capturedRequest anthropicRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/v1/messages", r.URL.Path) + assert.Equal(t, "text/event-stream", r.Header.Get("Accept")) + require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest)) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(`event: message_start +data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_delta +data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}} + +event: message_stop +data: {"type":"message_stop"} + +`)) + })) + defer server.Close() + + chat, err := NewAnthropicChat(&ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: server.URL, + ModelName: "gpt-5.5", + APIKey: "test-key", + Provider: string(provider.ProviderAnthropic), + }) + require.NoError(t, err) + + ch, err := chat.ChatStream(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil) + require.NoError(t, err) + + var chunks []types.StreamResponse + for chunk := range ch { + chunks = append(chunks, chunk) + } + + require.Len(t, chunks, 2) + assert.True(t, capturedRequest.Stream) + assert.Equal(t, "pong", chunks[0].Content) + assert.False(t, chunks[0].Done) + assert.True(t, chunks[1].Done) + assert.Equal(t, "end_turn", chunks[1].FinishReason) + require.NotNil(t, chunks[1].Usage) + assert.Equal(t, 114, chunks[1].Usage.PromptTokens) + assert.Equal(t, 5, chunks[1].Usage.CompletionTokens) + assert.Equal(t, 119, chunks[1].Usage.TotalTokens) +} + func TestNewRemoteChat_AnthropicProvider(t *testing.T) { chat, err := NewRemoteChat(&ChatConfig{ Source: types.ModelSourceRemote,