fix: support Anthropic gateway streaming

This commit is contained in:
langcaiye
2026-05-12 19:24:28 +08:00
committed by lyingbug
parent 55caeefc3d
commit 4d2b8707ff
2 changed files with 417 additions and 3 deletions
+253 -2
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"github.com/Tencent/WeKnora/internal/models/provider"
@@ -32,6 +33,7 @@ type anthropicMessage struct {
type anthropicRequest struct {
Model string `json:"model"`
MaxTokens int `json:"max_tokens"`
Stream bool `json:"stream,omitempty"`
System string `json:"system,omitempty"`
Messages []anthropicMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
@@ -57,6 +59,33 @@ type anthropicResponse struct {
} `json:"error,omitempty"`
}
type anthropicStreamEvent struct {
Type string `json:"type"`
Message *struct {
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
CacheReadInputTokens int `json:"cache_read_input_tokens"`
} `json:"usage"`
} `json:"message,omitempty"`
Delta *struct {
Type string `json:"type"`
Text string `json:"text"`
StopReason string `json:"stop_reason"`
} `json:"delta,omitempty"`
Usage *struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
CacheReadInputTokens int `json:"cache_read_input_tokens"`
} `json:"usage,omitempty"`
Error *struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error,omitempty"`
}
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
if config.BaseURL != "" {
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
@@ -91,7 +120,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
ctx, cancel := withLLMTimeout(ctx, defaultChatTimeout)
defer cancel()
endpoint := c.baseURL + "/messages"
endpoint := c.endpoint()
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
}
@@ -116,6 +145,17 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
return nil, fmt.Errorf("read response: %w", err)
}
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
chatResp, err := parseAnthropicSSE(bytes.NewReader(body))
if err != nil {
return nil, err
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content)
}
return chatResp, nil
}
var chatResp anthropicResponse
if err := json.Unmarshal(body, &chatResp); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
@@ -131,7 +171,41 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
}
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
return nil, fmt.Errorf("Anthropic streaming chat is not supported yet")
reqBody := c.buildRequest(messages, opts)
reqBody.Stream = true
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
endpoint := c.endpoint()
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Accept", "text/event-stream")
httpReq.Header.Set("x-api-key", c.apiKey)
httpReq.Header.Set("anthropic-version", anthropicVersion)
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
}
streamChan := make(chan types.StreamResponse)
go processAnthropicStream(resp, streamChan)
return streamChan, nil
}
func (c *AnthropicChat) GetModelName() string {
@@ -142,6 +216,35 @@ func (c *AnthropicChat) GetModelID() string {
return c.modelID
}
func (c *AnthropicChat) endpoint() string {
baseURL := strings.TrimRight(c.baseURL, "/")
if isAnthropicMessagesEndpoint(baseURL) {
return baseURL
}
if isAnthropicVersionedBaseURL(baseURL) {
return baseURL + "/messages"
}
return baseURL + "/v1/messages"
}
func isAnthropicMessagesEndpoint(baseURL string) bool {
u, err := url.Parse(baseURL)
if err != nil {
return false
}
path := strings.TrimRight(u.Path, "/")
return strings.HasSuffix(path, "/messages")
}
func isAnthropicVersionedBaseURL(baseURL string) bool {
u, err := url.Parse(baseURL)
if err != nil {
return false
}
path := strings.TrimRight(u.Path, "/")
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
}
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
req := anthropicRequest{
Model: c.modelName,
@@ -220,3 +323,151 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
},
}
}
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
sseReader := NewSSEReader(reader)
var contentParts []string
var finishReason string
var inputTokens int
var outputTokens int
for {
event, err := sseReader.ReadEvent()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read SSE response: %w", err)
}
if event.Done {
break
}
if len(event.Data) == 0 {
continue
}
var streamEvent anthropicStreamEvent
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
return nil, fmt.Errorf("decode SSE response: %w", err)
}
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message)
}
if streamEvent.Message != nil {
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
outputTokens = max(outputTokens, streamEvent.Message.Usage.OutputTokens)
}
if streamEvent.Delta != nil {
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
contentParts = append(contentParts, streamEvent.Delta.Text)
}
if streamEvent.Delta.StopReason != "" {
finishReason = streamEvent.Delta.StopReason
}
}
if streamEvent.Usage != nil {
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
}
}
return &types.ChatResponse{
Content: strings.Join(contentParts, ""),
FinishReason: finishReason,
Usage: types.TokenUsage{
PromptTokens: inputTokens,
CompletionTokens: outputTokens,
TotalTokens: inputTokens + outputTokens,
},
}, nil
}
func processAnthropicStream(resp *http.Response, streamChan chan types.StreamResponse) {
defer close(streamChan)
defer resp.Body.Close()
sseReader := NewSSEReader(resp.Body)
var usage *types.TokenUsage
var finishReason string
for {
event, err := sseReader.ReadEvent()
if err != nil {
if err == io.EOF {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
Usage: usage,
FinishReason: finishReason,
}
} else {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: err.Error(),
Done: true,
}
}
return
}
if event.Done {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: "",
Done: true,
Usage: usage,
FinishReason: finishReason,
}
return
}
if len(event.Data) == 0 {
continue
}
var streamEvent anthropicStreamEvent
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: fmt.Sprintf("decode SSE response: %v", err),
Done: true,
}
return
}
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeError,
Content: streamEvent.Error.Message,
Done: true,
}
return
}
if streamEvent.Message != nil {
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
}
if streamEvent.Delta != nil {
if streamEvent.Delta.StopReason != "" {
finishReason = streamEvent.Delta.StopReason
}
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Content: streamEvent.Delta.Text,
Done: false,
}
}
}
if streamEvent.Usage != nil {
usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens, streamEvent.Usage.OutputTokens)
}
}
}
func mergeAnthropicUsage(current *types.TokenUsage, inputTokens, outputTokens int) *types.TokenUsage {
if current == nil {
current = &types.TokenUsage{}
}
current.PromptTokens = max(current.PromptTokens, inputTokens)
current.CompletionTokens = max(current.CompletionTokens, outputTokens)
current.TotalTokens = current.PromptTokens + current.CompletionTokens
return current
}
+164 -1
View File
@@ -20,7 +20,7 @@ func TestAnthropicChat(t *testing.T) {
var capturedRequest anthropicRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/messages", r.URL.Path)
assert.Equal(t, "/v1/messages", r.URL.Path)
capturedHeaders = r.Header.Clone()
require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
@@ -70,6 +70,169 @@ func TestAnthropicChat(t *testing.T) {
assert.Equal(t, 5, resp.Usage.TotalTokens)
}
func TestAnthropicChat_FullEndpoint(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var capturedPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
capturedPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"id":"msg_123",
"type":"message",
"role":"assistant",
"content":[{"type":"text","text":"hello"}],
"stop_reason":"end_turn",
"usage":{"input_tokens":3,"output_tokens":2}
}`))
}))
defer server.Close()
chat, err := NewAnthropicChat(&ChatConfig{
Source: types.ModelSourceRemote,
BaseURL: server.URL + "/api/proxy/forward",
ModelName: "gpt-5.5",
APIKey: "test-key",
Provider: string(provider.ProviderAnthropic),
})
require.NoError(t, err)
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil)
require.NoError(t, err)
assert.Equal(t, "/api/proxy/forward/v1/messages", capturedPath)
assert.Equal(t, "hello", resp.Content)
}
func TestAnthropicChat_MessagesEndpoint(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var capturedPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
capturedPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"id":"msg_123",
"type":"message",
"role":"assistant",
"content":[{"type":"text","text":"hello"}],
"stop_reason":"end_turn",
"usage":{"input_tokens":3,"output_tokens":2}
}`))
}))
defer server.Close()
chat, err := NewAnthropicChat(&ChatConfig{
Source: types.ModelSourceRemote,
BaseURL: server.URL + "/api/v1/messages",
ModelName: "gpt-5.5",
APIKey: "test-key",
Provider: string(provider.ProviderAnthropic),
})
require.NoError(t, err)
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil)
require.NoError(t, err)
assert.Equal(t, "/api/v1/messages", capturedPath)
assert.Equal(t, "hello", resp.Content)
}
func TestAnthropicChat_SSEResponse(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v1/messages", r.URL.Path)
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
event: message_stop
data: {"type":"message_stop"}
`))
}))
defer server.Close()
chat, err := NewAnthropicChat(&ChatConfig{
Source: types.ModelSourceRemote,
BaseURL: server.URL,
ModelName: "gpt-5.5",
APIKey: "test-key",
Provider: string(provider.ProviderAnthropic),
})
require.NoError(t, err)
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil)
require.NoError(t, err)
assert.Equal(t, "pong", resp.Content)
assert.Equal(t, "end_turn", resp.FinishReason)
assert.Equal(t, 114, resp.Usage.PromptTokens)
assert.Equal(t, 5, resp.Usage.CompletionTokens)
assert.Equal(t, 119, resp.Usage.TotalTokens)
}
func TestAnthropicChat_ChatStream(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var capturedRequest anthropicRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v1/messages", r.URL.Path)
assert.Equal(t, "text/event-stream", r.Header.Get("Accept"))
require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
event: message_stop
data: {"type":"message_stop"}
`))
}))
defer server.Close()
chat, err := NewAnthropicChat(&ChatConfig{
Source: types.ModelSourceRemote,
BaseURL: server.URL,
ModelName: "gpt-5.5",
APIKey: "test-key",
Provider: string(provider.ProviderAnthropic),
})
require.NoError(t, err)
ch, err := chat.ChatStream(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil)
require.NoError(t, err)
var chunks []types.StreamResponse
for chunk := range ch {
chunks = append(chunks, chunk)
}
require.Len(t, chunks, 2)
assert.True(t, capturedRequest.Stream)
assert.Equal(t, "pong", chunks[0].Content)
assert.False(t, chunks[0].Done)
assert.True(t, chunks[1].Done)
assert.Equal(t, "end_turn", chunks[1].FinishReason)
require.NotNil(t, chunks[1].Usage)
assert.Equal(t, 114, chunks[1].Usage.PromptTokens)
assert.Equal(t, 5, chunks[1].Usage.CompletionTokens)
assert.Equal(t, 119, chunks[1].Usage.TotalTokens)
}
func TestNewRemoteChat_AnthropicProvider(t *testing.T) {
chat, err := NewRemoteChat(&ChatConfig{
Source: types.ModelSourceRemote,