fix: support Anthropic gateway streaming
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
@@ -32,6 +33,7 @@ type anthropicMessage struct {
|
||||
type anthropicRequest struct {
|
||||
Model string `json:"model"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
System string `json:"system,omitempty"`
|
||||
Messages []anthropicMessage `json:"messages"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
@@ -57,6 +59,33 @@ type anthropicResponse struct {
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type anthropicStreamEvent struct {
|
||||
Type string `json:"type"`
|
||||
Message *struct {
|
||||
Usage struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
|
||||
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
||||
} `json:"usage"`
|
||||
} `json:"message,omitempty"`
|
||||
Delta *struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
StopReason string `json:"stop_reason"`
|
||||
} `json:"delta,omitempty"`
|
||||
Usage *struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
|
||||
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
||||
} `json:"usage,omitempty"`
|
||||
Error *struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
||||
if config.BaseURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
|
||||
@@ -91,7 +120,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
ctx, cancel := withLLMTimeout(ctx, defaultChatTimeout)
|
||||
defer cancel()
|
||||
|
||||
endpoint := c.baseURL + "/messages"
|
||||
endpoint := c.endpoint()
|
||||
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
||||
}
|
||||
@@ -116,6 +145,17 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
||||
chatResp, err := parseAnthropicSSE(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content)
|
||||
}
|
||||
return chatResp, nil
|
||||
}
|
||||
|
||||
var chatResp anthropicResponse
|
||||
if err := json.Unmarshal(body, &chatResp); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
@@ -131,7 +171,41 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
return nil, fmt.Errorf("Anthropic streaming chat is not supported yet")
|
||||
reqBody := c.buildRequest(messages, opts)
|
||||
reqBody.Stream = true
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
endpoint := c.endpoint()
|
||||
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Accept", "text/event-stream")
|
||||
httpReq.Header.Set("x-api-key", c.apiKey)
|
||||
httpReq.Header.Set("anthropic-version", anthropicVersion)
|
||||
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
|
||||
|
||||
resp, err := rawHTTPClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
streamChan := make(chan types.StreamResponse)
|
||||
go processAnthropicStream(resp, streamChan)
|
||||
return streamChan, nil
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) GetModelName() string {
|
||||
@@ -142,6 +216,35 @@ func (c *AnthropicChat) GetModelID() string {
|
||||
return c.modelID
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) endpoint() string {
|
||||
baseURL := strings.TrimRight(c.baseURL, "/")
|
||||
if isAnthropicMessagesEndpoint(baseURL) {
|
||||
return baseURL
|
||||
}
|
||||
if isAnthropicVersionedBaseURL(baseURL) {
|
||||
return baseURL + "/messages"
|
||||
}
|
||||
return baseURL + "/v1/messages"
|
||||
}
|
||||
|
||||
func isAnthropicMessagesEndpoint(baseURL string) bool {
|
||||
u, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
path := strings.TrimRight(u.Path, "/")
|
||||
return strings.HasSuffix(path, "/messages")
|
||||
}
|
||||
|
||||
func isAnthropicVersionedBaseURL(baseURL string) bool {
|
||||
u, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
path := strings.TrimRight(u.Path, "/")
|
||||
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
|
||||
req := anthropicRequest{
|
||||
Model: c.modelName,
|
||||
@@ -220,3 +323,151 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
||||
sseReader := NewSSEReader(reader)
|
||||
var contentParts []string
|
||||
var finishReason string
|
||||
var inputTokens int
|
||||
var outputTokens int
|
||||
|
||||
for {
|
||||
event, err := sseReader.ReadEvent()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read SSE response: %w", err)
|
||||
}
|
||||
if event.Done {
|
||||
break
|
||||
}
|
||||
if len(event.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var streamEvent anthropicStreamEvent
|
||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||
return nil, fmt.Errorf("decode SSE response: %w", err)
|
||||
}
|
||||
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||
return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message)
|
||||
}
|
||||
if streamEvent.Message != nil {
|
||||
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
|
||||
outputTokens = max(outputTokens, streamEvent.Message.Usage.OutputTokens)
|
||||
}
|
||||
if streamEvent.Delta != nil {
|
||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||
contentParts = append(contentParts, streamEvent.Delta.Text)
|
||||
}
|
||||
if streamEvent.Delta.StopReason != "" {
|
||||
finishReason = streamEvent.Delta.StopReason
|
||||
}
|
||||
}
|
||||
if streamEvent.Usage != nil {
|
||||
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
|
||||
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
|
||||
}
|
||||
}
|
||||
|
||||
return &types.ChatResponse{
|
||||
Content: strings.Join(contentParts, ""),
|
||||
FinishReason: finishReason,
|
||||
Usage: types.TokenUsage{
|
||||
PromptTokens: inputTokens,
|
||||
CompletionTokens: outputTokens,
|
||||
TotalTokens: inputTokens + outputTokens,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func processAnthropicStream(resp *http.Response, streamChan chan types.StreamResponse) {
|
||||
defer close(streamChan)
|
||||
defer resp.Body.Close()
|
||||
|
||||
sseReader := NewSSEReader(resp.Body)
|
||||
var usage *types.TokenUsage
|
||||
var finishReason string
|
||||
|
||||
for {
|
||||
event, err := sseReader.ReadEvent()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
FinishReason: finishReason,
|
||||
}
|
||||
} else {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: err.Error(),
|
||||
Done: true,
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if event.Done {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
FinishReason: finishReason,
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(event.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var streamEvent anthropicStreamEvent
|
||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: fmt.Sprintf("decode SSE response: %v", err),
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: streamEvent.Error.Message,
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
if streamEvent.Message != nil {
|
||||
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
|
||||
}
|
||||
if streamEvent.Delta != nil {
|
||||
if streamEvent.Delta.StopReason != "" {
|
||||
finishReason = streamEvent.Delta.StopReason
|
||||
}
|
||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: streamEvent.Delta.Text,
|
||||
Done: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
if streamEvent.Usage != nil {
|
||||
usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens, streamEvent.Usage.OutputTokens)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mergeAnthropicUsage(current *types.TokenUsage, inputTokens, outputTokens int) *types.TokenUsage {
|
||||
if current == nil {
|
||||
current = &types.TokenUsage{}
|
||||
}
|
||||
current.PromptTokens = max(current.PromptTokens, inputTokens)
|
||||
current.CompletionTokens = max(current.CompletionTokens, outputTokens)
|
||||
current.TotalTokens = current.PromptTokens + current.CompletionTokens
|
||||
return current
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestAnthropicChat(t *testing.T) {
|
||||
var capturedRequest anthropicRequest
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/messages", r.URL.Path)
|
||||
assert.Equal(t, "/v1/messages", r.URL.Path)
|
||||
capturedHeaders = r.Header.Clone()
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
|
||||
|
||||
@@ -70,6 +70,169 @@ func TestAnthropicChat(t *testing.T) {
|
||||
assert.Equal(t, 5, resp.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
func TestAnthropicChat_FullEndpoint(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
|
||||
var capturedPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
capturedPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id":"msg_123",
|
||||
"type":"message",
|
||||
"role":"assistant",
|
||||
"content":[{"type":"text","text":"hello"}],
|
||||
"stop_reason":"end_turn",
|
||||
"usage":{"input_tokens":3,"output_tokens":2}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
chat, err := NewAnthropicChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
BaseURL: server.URL + "/api/proxy/forward",
|
||||
ModelName: "gpt-5.5",
|
||||
APIKey: "test-key",
|
||||
Provider: string(provider.ProviderAnthropic),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "/api/proxy/forward/v1/messages", capturedPath)
|
||||
assert.Equal(t, "hello", resp.Content)
|
||||
}
|
||||
|
||||
func TestAnthropicChat_MessagesEndpoint(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
|
||||
var capturedPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
capturedPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id":"msg_123",
|
||||
"type":"message",
|
||||
"role":"assistant",
|
||||
"content":[{"type":"text","text":"hello"}],
|
||||
"stop_reason":"end_turn",
|
||||
"usage":{"input_tokens":3,"output_tokens":2}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
chat, err := NewAnthropicChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
BaseURL: server.URL + "/api/v1/messages",
|
||||
ModelName: "gpt-5.5",
|
||||
APIKey: "test-key",
|
||||
Provider: string(provider.ProviderAnthropic),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "/api/v1/messages", capturedPath)
|
||||
assert.Equal(t, "hello", resp.Content)
|
||||
}
|
||||
|
||||
func TestAnthropicChat_SSEResponse(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/v1/messages", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte(`event: message_start
|
||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||
|
||||
event: message_stop
|
||||
data: {"type":"message_stop"}
|
||||
|
||||
`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
chat, err := NewAnthropicChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
BaseURL: server.URL,
|
||||
ModelName: "gpt-5.5",
|
||||
APIKey: "test-key",
|
||||
Provider: string(provider.ProviderAnthropic),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "pong", resp.Content)
|
||||
assert.Equal(t, "end_turn", resp.FinishReason)
|
||||
assert.Equal(t, 114, resp.Usage.PromptTokens)
|
||||
assert.Equal(t, 5, resp.Usage.CompletionTokens)
|
||||
assert.Equal(t, 119, resp.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
func TestAnthropicChat_ChatStream(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
|
||||
var capturedRequest anthropicRequest
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/v1/messages", r.URL.Path)
|
||||
assert.Equal(t, "text/event-stream", r.Header.Get("Accept"))
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte(`event: message_start
|
||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||
|
||||
event: message_stop
|
||||
data: {"type":"message_stop"}
|
||||
|
||||
`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
chat, err := NewAnthropicChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
BaseURL: server.URL,
|
||||
ModelName: "gpt-5.5",
|
||||
APIKey: "test-key",
|
||||
Provider: string(provider.ProviderAnthropic),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
ch, err := chat.ChatStream(context.Background(), []Message{{Role: "user", Content: "ping"}}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
var chunks []types.StreamResponse
|
||||
for chunk := range ch {
|
||||
chunks = append(chunks, chunk)
|
||||
}
|
||||
|
||||
require.Len(t, chunks, 2)
|
||||
assert.True(t, capturedRequest.Stream)
|
||||
assert.Equal(t, "pong", chunks[0].Content)
|
||||
assert.False(t, chunks[0].Done)
|
||||
assert.True(t, chunks[1].Done)
|
||||
assert.Equal(t, "end_turn", chunks[1].FinishReason)
|
||||
require.NotNil(t, chunks[1].Usage)
|
||||
assert.Equal(t, 114, chunks[1].Usage.PromptTokens)
|
||||
assert.Equal(t, 5, chunks[1].Usage.CompletionTokens)
|
||||
assert.Equal(t, 119, chunks[1].Usage.TotalTokens)
|
||||
}
|
||||
|
||||
func TestNewRemoteChat_AnthropicProvider(t *testing.T) {
|
||||
chat, err := NewRemoteChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
|
||||
Reference in New Issue
Block a user