687 lines
20 KiB
Go
687 lines
20 KiB
Go
package chat
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/models/provider"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
)
|
|
|
|
const (
|
|
anthropicVersion = "2023-06-01"
|
|
anthropicResponseBodyLimit = 1 << 20
|
|
)
|
|
|
|
type AnthropicChat struct {
|
|
modelName string
|
|
modelID string
|
|
baseURL string
|
|
apiKey string
|
|
supportsVision bool
|
|
customHeaders map[string]string
|
|
}
|
|
|
|
func (c *AnthropicChat) SupportsImageInput() bool { return c.supportsVision }
|
|
|
|
type anthropicMessage struct {
|
|
Role string `json:"role"`
|
|
Content any `json:"content"`
|
|
}
|
|
|
|
type anthropicContentBlock struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text,omitempty"`
|
|
Source *anthropicImageSource `json:"source,omitempty"`
|
|
}
|
|
|
|
type anthropicImageSource struct {
|
|
Type string `json:"type"`
|
|
MediaType string `json:"media_type"`
|
|
Data string `json:"data"`
|
|
}
|
|
|
|
type anthropicRequest struct {
|
|
Model string `json:"model"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
System string `json:"system,omitempty"`
|
|
Messages []anthropicMessage `json:"messages"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
}
|
|
|
|
type anthropicResponse struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Role string `json:"role"`
|
|
Content []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
} `json:"content"`
|
|
StopReason string `json:"stop_reason"`
|
|
Usage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
} `json:"usage"`
|
|
Error *struct {
|
|
Type string `json:"type"`
|
|
Message string `json:"message"`
|
|
} `json:"error,omitempty"`
|
|
}
|
|
|
|
type anthropicStreamEvent struct {
|
|
Type string `json:"type"`
|
|
Index *int `json:"index,omitempty"`
|
|
Message *struct {
|
|
Usage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
|
|
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
|
} `json:"usage"`
|
|
} `json:"message,omitempty"`
|
|
ContentBlock *struct {
|
|
Type string `json:"type"`
|
|
} `json:"content_block,omitempty"`
|
|
Delta *struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
StopReason string `json:"stop_reason"`
|
|
} `json:"delta,omitempty"`
|
|
Usage *struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
|
|
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
|
} `json:"usage,omitempty"`
|
|
Error *struct {
|
|
Type string `json:"type"`
|
|
Message string `json:"message"`
|
|
} `json:"error,omitempty"`
|
|
}
|
|
|
|
type anthropicStreamProtocolState struct {
|
|
messageStarted bool
|
|
finishReason string
|
|
messageStopped bool
|
|
activeBlocks map[int]string
|
|
completedBlocks map[int]struct{}
|
|
nextBlockIndex int
|
|
}
|
|
|
|
func validAnthropicStopReason(reason string) bool {
|
|
switch reason {
|
|
case "end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func anthropicDeltaMatchesBlock(blockType, deltaType string) bool {
|
|
switch deltaType {
|
|
case "text_delta", "citations_delta":
|
|
return blockType == "text"
|
|
case "thinking_delta", "signature_delta":
|
|
return blockType == "thinking"
|
|
case "input_json_delta":
|
|
return blockType == "tool_use" || blockType == "server_tool_use" || blockType == "mcp_tool_use"
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (s *anthropicStreamProtocolState) observe(event *anthropicStreamEvent) error {
|
|
if event == nil {
|
|
return errors.New("anthropic stream event is missing")
|
|
}
|
|
if event.Type == "error" || event.Error != nil {
|
|
return errors.New("anthropic stream failed")
|
|
}
|
|
if s.messageStopped {
|
|
if event.Type == "ping" {
|
|
return nil
|
|
}
|
|
return errors.New("anthropic event followed message_stop")
|
|
}
|
|
if event.Delta != nil && event.Delta.StopReason != "" && event.Type != "message_delta" {
|
|
return errors.New("anthropic stop reason appeared on an invalid event")
|
|
}
|
|
|
|
switch event.Type {
|
|
case "message_start":
|
|
if s.messageStarted || s.finishReason != "" {
|
|
return errors.New("anthropic message_start is out of order")
|
|
}
|
|
s.messageStarted = true
|
|
s.activeBlocks = make(map[int]string)
|
|
s.completedBlocks = make(map[int]struct{})
|
|
case "message_delta":
|
|
if !s.messageStarted || s.finishReason != "" || len(s.activeBlocks) != 0 || event.Delta == nil || !validAnthropicStopReason(event.Delta.StopReason) {
|
|
return errors.New("anthropic message_delta has an invalid stop reason")
|
|
}
|
|
s.finishReason = event.Delta.StopReason
|
|
case "message_stop":
|
|
if !s.messageStarted || s.finishReason == "" || len(s.activeBlocks) != 0 {
|
|
return errors.New("anthropic message_stop preceded valid completion")
|
|
}
|
|
s.messageStopped = true
|
|
case "content_block_start":
|
|
if !s.messageStarted || s.finishReason != "" || event.Index == nil || *event.Index < 0 || event.ContentBlock == nil || strings.TrimSpace(event.ContentBlock.Type) == "" {
|
|
return errors.New("anthropic content_block_start is invalid")
|
|
}
|
|
index := *event.Index
|
|
if len(s.activeBlocks) != 0 || index != s.nextBlockIndex {
|
|
return errors.New("anthropic content block index is out of order")
|
|
}
|
|
if _, active := s.activeBlocks[index]; active {
|
|
return errors.New("anthropic content block started twice")
|
|
}
|
|
if _, completed := s.completedBlocks[index]; completed {
|
|
return errors.New("anthropic completed content block restarted")
|
|
}
|
|
s.activeBlocks[index] = event.ContentBlock.Type
|
|
case "content_block_delta":
|
|
if !s.messageStarted || s.finishReason != "" || event.Index == nil || event.Delta == nil {
|
|
return errors.New("anthropic content_block_delta is invalid")
|
|
}
|
|
blockType, active := s.activeBlocks[*event.Index]
|
|
if !active || !anthropicDeltaMatchesBlock(blockType, event.Delta.Type) {
|
|
return errors.New("anthropic content block delta is out of order")
|
|
}
|
|
case "content_block_stop":
|
|
if !s.messageStarted || s.finishReason != "" || event.Index == nil {
|
|
return errors.New("anthropic content_block_stop is invalid")
|
|
}
|
|
index := *event.Index
|
|
if _, active := s.activeBlocks[index]; !active {
|
|
return errors.New("anthropic content block stop is out of order")
|
|
}
|
|
delete(s.activeBlocks, index)
|
|
s.completedBlocks[index] = struct{}{}
|
|
s.nextBlockIndex++
|
|
case "ping":
|
|
default:
|
|
return errors.New("anthropic stream event type is invalid")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *anthropicStreamProtocolState) complete() bool {
|
|
return s.messageStarted && s.messageStopped && len(s.activeBlocks) == 0 && validAnthropicStopReason(s.finishReason)
|
|
}
|
|
|
|
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
|
if config.BaseURL != "" {
|
|
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
|
|
return nil, fmt.Errorf("baseURL SSRF check failed: %w", err)
|
|
}
|
|
}
|
|
if strings.TrimSpace(config.APIKey) == "" {
|
|
return nil, fmt.Errorf("Anthropic provider: API key is required")
|
|
}
|
|
|
|
baseURL := strings.TrimRight(config.BaseURL, "/")
|
|
if baseURL == "" {
|
|
baseURL = provider.AnthropicBaseURL
|
|
}
|
|
|
|
return &AnthropicChat{
|
|
modelName: config.ModelName,
|
|
modelID: config.ModelID,
|
|
baseURL: baseURL,
|
|
apiKey: config.APIKey,
|
|
supportsVision: config.SupportsVision,
|
|
customHeaders: config.CustomHeaders,
|
|
}, nil
|
|
}
|
|
|
|
func readAnthropicResponseBody(body io.ReadCloser) ([]byte, error) {
|
|
data, err := io.ReadAll(io.LimitReader(body, anthropicResponseBodyLimit+1))
|
|
if err != nil {
|
|
return nil, errors.New("read Anthropic response failed")
|
|
}
|
|
if len(data) > anthropicResponseBodyLimit {
|
|
return nil, errors.New("Anthropic response body too large")
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
|
reqBody, err := c.buildRequest(messages, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
jsonData, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
|
|
ctx, cancel := withLLMTimeout(ctx, defaultChatTimeout)
|
|
defer cancel()
|
|
|
|
endpoint := c.endpoint()
|
|
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
|
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(jsonData))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
httpReq.Header.Set("x-api-key", c.apiKey)
|
|
httpReq.Header.Set("anthropic-version", anthropicVersion)
|
|
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
|
|
|
|
resp, err := rawHTTPClient.Do(httpReq)
|
|
if err != nil {
|
|
return nil, errors.New("send Anthropic request failed")
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := readAnthropicResponseBody(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
|
chatResp, err := parseAnthropicSSE(bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
|
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
|
}
|
|
logUsage(ctx, c.modelName, &chatResp.Usage)
|
|
return chatResp, nil
|
|
}
|
|
|
|
var chatResp anthropicResponse
|
|
if err := json.Unmarshal(body, &chatResp); err != nil {
|
|
return nil, errors.New("decode Anthropic response failed")
|
|
}
|
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
|
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
|
}
|
|
|
|
result := c.parseResponse(&chatResp)
|
|
logUsage(ctx, c.modelName, &result.Usage)
|
|
return result, nil
|
|
}
|
|
|
|
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
|
reqBody, err := c.buildRequest(messages, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqBody.Stream = true
|
|
jsonData, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
|
|
endpoint := c.endpoint()
|
|
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
|
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(jsonData))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
httpReq.Header.Set("Accept", "text/event-stream")
|
|
httpReq.Header.Set("x-api-key", c.apiKey)
|
|
httpReq.Header.Set("anthropic-version", anthropicVersion)
|
|
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
|
|
|
|
resp, err := rawHTTPClient.Do(httpReq)
|
|
if err != nil {
|
|
return nil, errors.New("send Anthropic stream request failed")
|
|
}
|
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
|
body, readErr := readAnthropicResponseBody(resp.Body)
|
|
resp.Body.Close()
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
|
}
|
|
|
|
streamChan := make(chan types.StreamResponse)
|
|
go processAnthropicStream(ctx, c.modelName, resp, streamChan)
|
|
return streamChan, nil
|
|
}
|
|
|
|
func (c *AnthropicChat) GetModelName() string {
|
|
return c.modelName
|
|
}
|
|
|
|
func (c *AnthropicChat) GetModelID() string {
|
|
return c.modelID
|
|
}
|
|
|
|
func (c *AnthropicChat) endpoint() string {
|
|
baseURL := strings.TrimRight(c.baseURL, "/")
|
|
if isAnthropicMessagesEndpoint(baseURL) {
|
|
return baseURL
|
|
}
|
|
if isAnthropicVersionedBaseURL(baseURL) {
|
|
return baseURL + "/messages"
|
|
}
|
|
return baseURL + "/v1/messages"
|
|
}
|
|
|
|
func isAnthropicMessagesEndpoint(baseURL string) bool {
|
|
u, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
path := strings.TrimRight(u.Path, "/")
|
|
return strings.HasSuffix(path, "/messages")
|
|
}
|
|
|
|
func isAnthropicVersionedBaseURL(baseURL string) bool {
|
|
u, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
path := strings.TrimRight(u.Path, "/")
|
|
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
|
|
}
|
|
|
|
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) (anthropicRequest, error) {
|
|
req := anthropicRequest{
|
|
Model: c.modelName,
|
|
MaxTokens: 1024,
|
|
Messages: make([]anthropicMessage, 0, len(messages)),
|
|
}
|
|
if opts != nil {
|
|
if opts.MaxTokens > 0 {
|
|
req.MaxTokens = opts.MaxTokens
|
|
} else if opts.MaxCompletionTokens > 0 {
|
|
req.MaxTokens = opts.MaxCompletionTokens
|
|
}
|
|
if opts.Temperature > 0 {
|
|
temperature := opts.Temperature
|
|
req.Temperature = &temperature
|
|
}
|
|
if opts.TopP > 0 {
|
|
topP := opts.TopP
|
|
req.TopP = &topP
|
|
}
|
|
}
|
|
|
|
var systemParts []string
|
|
for _, msg := range messages {
|
|
content := strings.TrimSpace(msg.Content)
|
|
if content == "" {
|
|
content = textFromMultiContent(msg.MultiContent)
|
|
}
|
|
images := append([]string(nil), msg.Images...)
|
|
for _, part := range msg.MultiContent {
|
|
if part.Type == "image_url" && part.ImageURL != nil && strings.TrimSpace(part.ImageURL.URL) != "" {
|
|
images = append(images, part.ImageURL.URL)
|
|
}
|
|
}
|
|
if len(images) == 0 && content == "" {
|
|
continue
|
|
}
|
|
switch msg.Role {
|
|
case "system":
|
|
if len(images) > 0 {
|
|
return anthropicRequest{}, fmt.Errorf("anthropic system messages cannot contain images")
|
|
}
|
|
systemParts = append(systemParts, content)
|
|
case "assistant":
|
|
if len(images) > 0 {
|
|
return anthropicRequest{}, fmt.Errorf("anthropic assistant messages cannot contain images")
|
|
}
|
|
req.Messages = append(req.Messages, anthropicMessage{Role: "assistant", Content: content})
|
|
default:
|
|
if len(images) == 0 {
|
|
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
|
continue
|
|
}
|
|
blocks := make([]anthropicContentBlock, 0, len(images)+1)
|
|
if content != "" {
|
|
blocks = append(blocks, anthropicContentBlock{Type: "text", Text: content})
|
|
}
|
|
for _, imageURL := range images {
|
|
block, err := anthropicImageBlock(imageURL)
|
|
if err != nil {
|
|
return anthropicRequest{}, err
|
|
}
|
|
blocks = append(blocks, block)
|
|
}
|
|
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: blocks})
|
|
}
|
|
}
|
|
req.System = strings.Join(systemParts, "\n\n")
|
|
return req, nil
|
|
}
|
|
|
|
func anthropicImageBlock(raw string) (anthropicContentBlock, error) {
|
|
metadata, payload, ok := strings.Cut(strings.TrimSpace(raw), ",")
|
|
if !ok {
|
|
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
|
}
|
|
metadata = strings.ToLower(metadata)
|
|
var mediaType string
|
|
switch metadata {
|
|
case "data:image/jpeg;base64", "data:image/jpg;base64":
|
|
mediaType = "image/jpeg"
|
|
case "data:image/png;base64":
|
|
mediaType = "image/png"
|
|
default:
|
|
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
|
}
|
|
if _, err := base64.StdEncoding.DecodeString(payload); err != nil {
|
|
return anthropicContentBlock{}, fmt.Errorf("anthropic image data is invalid")
|
|
}
|
|
return anthropicContentBlock{
|
|
Type: "image",
|
|
Source: &anthropicImageSource{
|
|
Type: "base64",
|
|
MediaType: mediaType,
|
|
Data: payload,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func textFromMultiContent(parts []MessageContentPart) string {
|
|
if len(parts) == 0 {
|
|
return ""
|
|
}
|
|
textParts := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
if part.Type == "text" && strings.TrimSpace(part.Text) != "" {
|
|
textParts = append(textParts, strings.TrimSpace(part.Text))
|
|
}
|
|
}
|
|
return strings.Join(textParts, "\n")
|
|
}
|
|
|
|
func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatResponse {
|
|
parts := make([]string, 0, len(resp.Content))
|
|
for _, part := range resp.Content {
|
|
if part.Type == "text" && part.Text != "" {
|
|
parts = append(parts, part.Text)
|
|
}
|
|
}
|
|
inputTokens := resp.Usage.InputTokens
|
|
outputTokens := resp.Usage.OutputTokens
|
|
return &types.ChatResponse{
|
|
Content: strings.Join(parts, ""),
|
|
FinishReason: resp.StopReason,
|
|
Usage: types.TokenUsage{
|
|
PromptTokens: inputTokens,
|
|
CompletionTokens: outputTokens,
|
|
TotalTokens: inputTokens + outputTokens,
|
|
},
|
|
}
|
|
}
|
|
|
|
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
|
sseReader := NewSSEReader(reader)
|
|
var contentParts []string
|
|
var protocol anthropicStreamProtocolState
|
|
var inputTokens int
|
|
var outputTokens int
|
|
|
|
for {
|
|
event, err := sseReader.ReadEvent()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read SSE response: %w", err)
|
|
}
|
|
if event.Done {
|
|
break
|
|
}
|
|
if len(event.Data) == 0 {
|
|
continue
|
|
}
|
|
|
|
var streamEvent anthropicStreamEvent
|
|
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
|
return nil, fmt.Errorf("decode SSE response: %w", err)
|
|
}
|
|
if err := protocol.observe(&streamEvent); err != nil {
|
|
return nil, errors.New("anthropic stream protocol failed")
|
|
}
|
|
if streamEvent.Message != nil {
|
|
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
|
|
outputTokens = max(outputTokens, streamEvent.Message.Usage.OutputTokens)
|
|
}
|
|
if streamEvent.Delta != nil {
|
|
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
|
contentParts = append(contentParts, streamEvent.Delta.Text)
|
|
}
|
|
}
|
|
if streamEvent.Usage != nil {
|
|
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
|
|
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
|
|
}
|
|
}
|
|
if !protocol.complete() {
|
|
return nil, errors.New("anthropic stream ended before completion")
|
|
}
|
|
|
|
return &types.ChatResponse{
|
|
Content: strings.Join(contentParts, ""),
|
|
FinishReason: protocol.finishReason,
|
|
Usage: types.TokenUsage{
|
|
PromptTokens: inputTokens,
|
|
CompletionTokens: outputTokens,
|
|
TotalTokens: inputTokens + outputTokens,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func processAnthropicStream(ctx context.Context, model string, resp *http.Response, streamChan chan types.StreamResponse) {
|
|
defer close(streamChan)
|
|
defer resp.Body.Close()
|
|
|
|
sseReader := NewSSEReader(resp.Body)
|
|
var usage *types.TokenUsage
|
|
var protocol anthropicStreamProtocolState
|
|
|
|
emitTerminal := func() {
|
|
if !protocol.complete() {
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeError,
|
|
Content: "anthropic stream ended before completion",
|
|
Done: true,
|
|
}
|
|
return
|
|
}
|
|
logUsage(ctx, model, usage)
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeAnswer,
|
|
Content: "",
|
|
Done: true,
|
|
Usage: usage,
|
|
FinishReason: protocol.finishReason,
|
|
}
|
|
}
|
|
|
|
for {
|
|
event, err := sseReader.ReadEvent()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
emitTerminal()
|
|
} else {
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeError,
|
|
Content: "anthropic stream read failed",
|
|
Done: true,
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if event.Done {
|
|
emitTerminal()
|
|
return
|
|
}
|
|
if len(event.Data) == 0 {
|
|
continue
|
|
}
|
|
|
|
var streamEvent anthropicStreamEvent
|
|
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeError,
|
|
Content: "anthropic stream response was invalid",
|
|
Done: true,
|
|
}
|
|
return
|
|
}
|
|
if err := protocol.observe(&streamEvent); err != nil {
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeError,
|
|
Content: "anthropic stream protocol failed",
|
|
Done: true,
|
|
}
|
|
return
|
|
}
|
|
if streamEvent.Message != nil {
|
|
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
|
|
}
|
|
if streamEvent.Delta != nil {
|
|
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
|
streamChan <- types.StreamResponse{
|
|
ResponseType: types.ResponseTypeAnswer,
|
|
Content: streamEvent.Delta.Text,
|
|
Done: false,
|
|
}
|
|
}
|
|
}
|
|
if streamEvent.Usage != nil {
|
|
usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens, streamEvent.Usage.OutputTokens)
|
|
}
|
|
}
|
|
}
|
|
|
|
func mergeAnthropicUsage(current *types.TokenUsage, inputTokens, outputTokens int) *types.TokenUsage {
|
|
if current == nil {
|
|
current = &types.TokenUsage{}
|
|
}
|
|
current.PromptTokens = max(current.PromptTokens, inputTokens)
|
|
current.CompletionTokens = max(current.CompletionTokens, outputTokens)
|
|
current.TotalTokens = current.PromptTokens + current.CompletionTokens
|
|
return current
|
|
}
|