feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
@@ -3,7 +3,9 @@ package chat
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -15,19 +17,37 @@ import (
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const anthropicVersion = "2023-06-01"
|
||||
const (
|
||||
anthropicVersion = "2023-06-01"
|
||||
anthropicResponseBodyLimit = 1 << 20
|
||||
)
|
||||
|
||||
type AnthropicChat struct {
|
||||
modelName string
|
||||
modelID string
|
||||
baseURL string
|
||||
apiKey string
|
||||
customHeaders map[string]string
|
||||
modelName string
|
||||
modelID string
|
||||
baseURL string
|
||||
apiKey string
|
||||
supportsVision bool
|
||||
customHeaders map[string]string
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) SupportsImageInput() bool { return c.supportsVision }
|
||||
|
||||
type anthropicMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type anthropicContentBlock struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Source *anthropicImageSource `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
type anthropicImageSource struct {
|
||||
Type string `json:"type"`
|
||||
MediaType string `json:"media_type"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type anthropicRequest struct {
|
||||
@@ -61,6 +81,7 @@ type anthropicResponse struct {
|
||||
|
||||
type anthropicStreamEvent struct {
|
||||
Type string `json:"type"`
|
||||
Index *int `json:"index,omitempty"`
|
||||
Message *struct {
|
||||
Usage struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
@@ -69,6 +90,9 @@ type anthropicStreamEvent struct {
|
||||
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
||||
} `json:"usage"`
|
||||
} `json:"message,omitempty"`
|
||||
ContentBlock *struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"content_block,omitempty"`
|
||||
Delta *struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
@@ -86,6 +110,117 @@ type anthropicStreamEvent struct {
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type anthropicStreamProtocolState struct {
|
||||
messageStarted bool
|
||||
finishReason string
|
||||
messageStopped bool
|
||||
activeBlocks map[int]string
|
||||
completedBlocks map[int]struct{}
|
||||
nextBlockIndex int
|
||||
}
|
||||
|
||||
func validAnthropicStopReason(reason string) bool {
|
||||
switch reason {
|
||||
case "end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func anthropicDeltaMatchesBlock(blockType, deltaType string) bool {
|
||||
switch deltaType {
|
||||
case "text_delta", "citations_delta":
|
||||
return blockType == "text"
|
||||
case "thinking_delta", "signature_delta":
|
||||
return blockType == "thinking"
|
||||
case "input_json_delta":
|
||||
return blockType == "tool_use" || blockType == "server_tool_use" || blockType == "mcp_tool_use"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *anthropicStreamProtocolState) observe(event *anthropicStreamEvent) error {
|
||||
if event == nil {
|
||||
return errors.New("anthropic stream event is missing")
|
||||
}
|
||||
if event.Type == "error" || event.Error != nil {
|
||||
return errors.New("anthropic stream failed")
|
||||
}
|
||||
if s.messageStopped {
|
||||
if event.Type == "ping" {
|
||||
return nil
|
||||
}
|
||||
return errors.New("anthropic event followed message_stop")
|
||||
}
|
||||
if event.Delta != nil && event.Delta.StopReason != "" && event.Type != "message_delta" {
|
||||
return errors.New("anthropic stop reason appeared on an invalid event")
|
||||
}
|
||||
|
||||
switch event.Type {
|
||||
case "message_start":
|
||||
if s.messageStarted || s.finishReason != "" {
|
||||
return errors.New("anthropic message_start is out of order")
|
||||
}
|
||||
s.messageStarted = true
|
||||
s.activeBlocks = make(map[int]string)
|
||||
s.completedBlocks = make(map[int]struct{})
|
||||
case "message_delta":
|
||||
if !s.messageStarted || s.finishReason != "" || len(s.activeBlocks) != 0 || event.Delta == nil || !validAnthropicStopReason(event.Delta.StopReason) {
|
||||
return errors.New("anthropic message_delta has an invalid stop reason")
|
||||
}
|
||||
s.finishReason = event.Delta.StopReason
|
||||
case "message_stop":
|
||||
if !s.messageStarted || s.finishReason == "" || len(s.activeBlocks) != 0 {
|
||||
return errors.New("anthropic message_stop preceded valid completion")
|
||||
}
|
||||
s.messageStopped = true
|
||||
case "content_block_start":
|
||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil || *event.Index < 0 || event.ContentBlock == nil || strings.TrimSpace(event.ContentBlock.Type) == "" {
|
||||
return errors.New("anthropic content_block_start is invalid")
|
||||
}
|
||||
index := *event.Index
|
||||
if len(s.activeBlocks) != 0 || index != s.nextBlockIndex {
|
||||
return errors.New("anthropic content block index is out of order")
|
||||
}
|
||||
if _, active := s.activeBlocks[index]; active {
|
||||
return errors.New("anthropic content block started twice")
|
||||
}
|
||||
if _, completed := s.completedBlocks[index]; completed {
|
||||
return errors.New("anthropic completed content block restarted")
|
||||
}
|
||||
s.activeBlocks[index] = event.ContentBlock.Type
|
||||
case "content_block_delta":
|
||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil || event.Delta == nil {
|
||||
return errors.New("anthropic content_block_delta is invalid")
|
||||
}
|
||||
blockType, active := s.activeBlocks[*event.Index]
|
||||
if !active || !anthropicDeltaMatchesBlock(blockType, event.Delta.Type) {
|
||||
return errors.New("anthropic content block delta is out of order")
|
||||
}
|
||||
case "content_block_stop":
|
||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil {
|
||||
return errors.New("anthropic content_block_stop is invalid")
|
||||
}
|
||||
index := *event.Index
|
||||
if _, active := s.activeBlocks[index]; !active {
|
||||
return errors.New("anthropic content block stop is out of order")
|
||||
}
|
||||
delete(s.activeBlocks, index)
|
||||
s.completedBlocks[index] = struct{}{}
|
||||
s.nextBlockIndex++
|
||||
case "ping":
|
||||
default:
|
||||
return errors.New("anthropic stream event type is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *anthropicStreamProtocolState) complete() bool {
|
||||
return s.messageStarted && s.messageStopped && len(s.activeBlocks) == 0 && validAnthropicStopReason(s.finishReason)
|
||||
}
|
||||
|
||||
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
||||
if config.BaseURL != "" {
|
||||
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
|
||||
@@ -102,16 +237,31 @@ func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
||||
}
|
||||
|
||||
return &AnthropicChat{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
baseURL: baseURL,
|
||||
apiKey: config.APIKey,
|
||||
customHeaders: config.CustomHeaders,
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
baseURL: baseURL,
|
||||
apiKey: config.APIKey,
|
||||
supportsVision: config.SupportsVision,
|
||||
customHeaders: config.CustomHeaders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readAnthropicResponseBody(body io.ReadCloser) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(body, anthropicResponseBodyLimit+1))
|
||||
if err != nil {
|
||||
return nil, errors.New("read Anthropic response failed")
|
||||
}
|
||||
if len(data) > anthropicResponseBodyLimit {
|
||||
return nil, errors.New("Anthropic response body too large")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
reqBody := c.buildRequest(messages, opts)
|
||||
reqBody, err := c.buildRequest(messages, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
@@ -136,13 +286,13 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
|
||||
resp, err := rawHTTPClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
return nil, errors.New("send Anthropic request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
body, err := readAnthropicResponseBody(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
||||
@@ -151,7 +301,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content)
|
||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
||||
}
|
||||
logUsage(ctx, c.modelName, &chatResp.Usage)
|
||||
return chatResp, nil
|
||||
@@ -159,13 +309,10 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
|
||||
var chatResp anthropicResponse
|
||||
if err := json.Unmarshal(body, &chatResp); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
return nil, errors.New("decode Anthropic response failed")
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
if chatResp.Error != nil && chatResp.Error.Message != "" {
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Error.Message)
|
||||
}
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
||||
}
|
||||
|
||||
result := c.parseResponse(&chatResp)
|
||||
@@ -174,7 +321,10 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
reqBody := c.buildRequest(messages, opts)
|
||||
reqBody, err := c.buildRequest(messages, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqBody.Stream = true
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
@@ -198,12 +348,15 @@ func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
|
||||
resp, err := rawHTTPClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
return nil, errors.New("send Anthropic stream request failed")
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
body, readErr := readAnthropicResponseBody(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
if readErr != nil {
|
||||
return nil, readErr
|
||||
}
|
||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
||||
}
|
||||
|
||||
streamChan := make(chan types.StreamResponse)
|
||||
@@ -248,7 +401,7 @@ func isAnthropicVersionedBaseURL(baseURL string) bool {
|
||||
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
|
||||
}
|
||||
|
||||
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
|
||||
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) (anthropicRequest, error) {
|
||||
req := anthropicRequest{
|
||||
Model: c.modelName,
|
||||
MaxTokens: 1024,
|
||||
@@ -276,22 +429,75 @@ func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anth
|
||||
if content == "" {
|
||||
content = textFromMultiContent(msg.MultiContent)
|
||||
}
|
||||
if content == "" {
|
||||
images := append([]string(nil), msg.Images...)
|
||||
for _, part := range msg.MultiContent {
|
||||
if part.Type == "image_url" && part.ImageURL != nil && strings.TrimSpace(part.ImageURL.URL) != "" {
|
||||
images = append(images, part.ImageURL.URL)
|
||||
}
|
||||
}
|
||||
if len(images) == 0 && content == "" {
|
||||
continue
|
||||
}
|
||||
switch msg.Role {
|
||||
case "system":
|
||||
if len(images) > 0 {
|
||||
return anthropicRequest{}, fmt.Errorf("anthropic system messages cannot contain images")
|
||||
}
|
||||
systemParts = append(systemParts, content)
|
||||
case "assistant":
|
||||
if len(images) > 0 {
|
||||
return anthropicRequest{}, fmt.Errorf("anthropic assistant messages cannot contain images")
|
||||
}
|
||||
req.Messages = append(req.Messages, anthropicMessage{Role: "assistant", Content: content})
|
||||
case "user":
|
||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
||||
default:
|
||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
||||
if len(images) == 0 {
|
||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
||||
continue
|
||||
}
|
||||
blocks := make([]anthropicContentBlock, 0, len(images)+1)
|
||||
if content != "" {
|
||||
blocks = append(blocks, anthropicContentBlock{Type: "text", Text: content})
|
||||
}
|
||||
for _, imageURL := range images {
|
||||
block, err := anthropicImageBlock(imageURL)
|
||||
if err != nil {
|
||||
return anthropicRequest{}, err
|
||||
}
|
||||
blocks = append(blocks, block)
|
||||
}
|
||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: blocks})
|
||||
}
|
||||
}
|
||||
req.System = strings.Join(systemParts, "\n\n")
|
||||
return req
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func anthropicImageBlock(raw string) (anthropicContentBlock, error) {
|
||||
metadata, payload, ok := strings.Cut(strings.TrimSpace(raw), ",")
|
||||
if !ok {
|
||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
||||
}
|
||||
metadata = strings.ToLower(metadata)
|
||||
var mediaType string
|
||||
switch metadata {
|
||||
case "data:image/jpeg;base64", "data:image/jpg;base64":
|
||||
mediaType = "image/jpeg"
|
||||
case "data:image/png;base64":
|
||||
mediaType = "image/png"
|
||||
default:
|
||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
||||
}
|
||||
if _, err := base64.StdEncoding.DecodeString(payload); err != nil {
|
||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image data is invalid")
|
||||
}
|
||||
return anthropicContentBlock{
|
||||
Type: "image",
|
||||
Source: &anthropicImageSource{
|
||||
Type: "base64",
|
||||
MediaType: mediaType,
|
||||
Data: payload,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func textFromMultiContent(parts []MessageContentPart) string {
|
||||
@@ -330,7 +536,7 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
|
||||
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
||||
sseReader := NewSSEReader(reader)
|
||||
var contentParts []string
|
||||
var finishReason string
|
||||
var protocol anthropicStreamProtocolState
|
||||
var inputTokens int
|
||||
var outputTokens int
|
||||
|
||||
@@ -353,8 +559,8 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||
return nil, fmt.Errorf("decode SSE response: %w", err)
|
||||
}
|
||||
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||
return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message)
|
||||
if err := protocol.observe(&streamEvent); err != nil {
|
||||
return nil, errors.New("anthropic stream protocol failed")
|
||||
}
|
||||
if streamEvent.Message != nil {
|
||||
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
|
||||
@@ -364,19 +570,19 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||
contentParts = append(contentParts, streamEvent.Delta.Text)
|
||||
}
|
||||
if streamEvent.Delta.StopReason != "" {
|
||||
finishReason = streamEvent.Delta.StopReason
|
||||
}
|
||||
}
|
||||
if streamEvent.Usage != nil {
|
||||
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
|
||||
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
|
||||
}
|
||||
}
|
||||
if !protocol.complete() {
|
||||
return nil, errors.New("anthropic stream ended before completion")
|
||||
}
|
||||
|
||||
return &types.ChatResponse{
|
||||
Content: strings.Join(contentParts, ""),
|
||||
FinishReason: finishReason,
|
||||
FinishReason: protocol.finishReason,
|
||||
Usage: types.TokenUsage{
|
||||
PromptTokens: inputTokens,
|
||||
CompletionTokens: outputTokens,
|
||||
@@ -391,38 +597,43 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
||||
|
||||
sseReader := NewSSEReader(resp.Body)
|
||||
var usage *types.TokenUsage
|
||||
var finishReason string
|
||||
var protocol anthropicStreamProtocolState
|
||||
|
||||
emitTerminal := func() {
|
||||
if !protocol.complete() {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: "anthropic stream ended before completion",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
logUsage(ctx, model, usage)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
FinishReason: protocol.finishReason,
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
event, err := sseReader.ReadEvent()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
logUsage(ctx, model, usage)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
FinishReason: finishReason,
|
||||
}
|
||||
emitTerminal()
|
||||
} else {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: err.Error(),
|
||||
Content: "anthropic stream read failed",
|
||||
Done: true,
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if event.Done {
|
||||
logUsage(ctx, model, usage)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
FinishReason: finishReason,
|
||||
}
|
||||
emitTerminal()
|
||||
return
|
||||
}
|
||||
if len(event.Data) == 0 {
|
||||
@@ -433,15 +644,15 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: fmt.Sprintf("decode SSE response: %v", err),
|
||||
Content: "anthropic stream response was invalid",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||
if err := protocol.observe(&streamEvent); err != nil {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: streamEvent.Error.Message,
|
||||
Content: "anthropic stream protocol failed",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
@@ -450,9 +661,6 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
||||
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
|
||||
}
|
||||
if streamEvent.Delta != nil {
|
||||
if streamEvent.Delta.StopReason != "" {
|
||||
finishReason = streamEvent.Delta.StopReason
|
||||
}
|
||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestAnthropicProviderErrorDoesNotExposeImagePayload(t *testing.T) {
|
||||
secret := "TOPSECRETIMAGEPAYLOAD"
|
||||
payload := "data: {\"type\":\"error\",\"error\":{\"message\":\"data:image/png;base64," + secret + "\"}}\n\n"
|
||||
|
||||
_, err := parseAnthropicSSE(strings.NewReader(payload))
|
||||
if err == nil {
|
||||
t.Fatal("expected Anthropic SSE error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
|
||||
t.Fatalf("non-stream error leaked provider payload: %v", err)
|
||||
}
|
||||
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse)
|
||||
go processAnthropicStream(context.Background(), "vision-test", resp, stream)
|
||||
var terminal types.StreamResponse
|
||||
for chunk := range stream {
|
||||
terminal = chunk
|
||||
}
|
||||
if terminal.ResponseType != types.ResponseTypeError {
|
||||
t.Fatalf("terminal response = %#v; want error", terminal)
|
||||
}
|
||||
if strings.Contains(terminal.Content, secret) || strings.Contains(strings.ToLower(terminal.Content), "data:image") {
|
||||
t.Fatalf("stream error leaked provider payload: %q", terminal.Content)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
const anthropicTestMessageStart = "data: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":1,\"output_tokens\":0}}}\n\n"
|
||||
const anthropicTestBlockStart0 = "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"
|
||||
const anthropicTestBlockDelta0 = "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n"
|
||||
const anthropicTestBlockStop0 = "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n"
|
||||
|
||||
func invalidAnthropicBlockPayloads() map[string]string {
|
||||
return map[string]string{
|
||||
"stop before start": anthropicTestMessageStart + anthropicTestBlockStop0,
|
||||
"delta before start": anthropicTestMessageStart + anthropicTestBlockDelta0,
|
||||
"delta after stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockDelta0,
|
||||
"duplicate start": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStart0,
|
||||
"duplicate stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockStop0,
|
||||
"mismatched index": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n",
|
||||
"index gap": anthropicTestMessageStart + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
|
||||
"concurrent block start": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
|
||||
"mismatched delta type": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\"}}\n\n",
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicContentBlockLifecycleRejectedByBothParsers(t *testing.T) {
|
||||
for name, payload := range invalidAnthropicBlockPayloads() {
|
||||
t.Run(name+"/aggregate", func(t *testing.T) {
|
||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
||||
t.Fatal("aggregate parser accepted invalid content-block lifecycle")
|
||||
}
|
||||
})
|
||||
t.Run(name+"/stream", func(t *testing.T) {
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 8)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
sawError := false
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
t.Fatalf("stream accepted invalid content-block lifecycle: %#v", item)
|
||||
}
|
||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
||||
sawError = true
|
||||
}
|
||||
}
|
||||
if !sawError {
|
||||
t.Fatal("stream did not emit an error terminal")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicContentBlockLifecycleAcceptsSequentialBlocks(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 + anthropicTestBlockStop0 +
|
||||
"data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\n" +
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"thinking_delta\"}}\n\n" +
|
||||
"data: {\"type\":\"content_block_stop\",\"index\":1}\n\n" +
|
||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"data: {\"type\":\"message_stop\"}\n\n"
|
||||
response, err := parseAnthropicSSE(strings.NewReader(payload))
|
||||
if err != nil || response.FinishReason != "end_turn" || response.Content != "x" {
|
||||
t.Fatalf("valid sequential blocks rejected: response=%#v err=%v", response, err)
|
||||
}
|
||||
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 8)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
successTerminals := 0
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
||||
t.Fatalf("valid stream rejected: %#v", item)
|
||||
}
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
successTerminals++
|
||||
}
|
||||
}
|
||||
if successTerminals != 1 {
|
||||
t.Fatalf("success terminals=%d, want 1", successTerminals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicProtocolAcceptsOfficialStopReasons(t *testing.T) {
|
||||
reasons := []string{"end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded"}
|
||||
for _, reason := range reasons {
|
||||
t.Run(reason, func(t *testing.T) {
|
||||
var state anthropicStreamProtocolState
|
||||
start := anthropicStreamEvent{Type: "message_start"}
|
||||
if err := state.observe(&start); err != nil {
|
||||
t.Fatalf("message_start: %v", err)
|
||||
}
|
||||
delta := anthropicStreamEvent{Type: "message_delta"}
|
||||
delta.Delta = &struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
StopReason string `json:"stop_reason"`
|
||||
}{StopReason: reason}
|
||||
if err := state.observe(&delta); err != nil {
|
||||
t.Fatalf("message_delta: %v", err)
|
||||
}
|
||||
stop := anthropicStreamEvent{Type: "message_stop"}
|
||||
if err := state.observe(&stop); err != nil || !state.complete() {
|
||||
t.Fatalf("message_stop err=%v complete=%v", err, state.complete())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAnthropicSSERejectsPrematureEOF(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
|
||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
||||
t.Fatal("expected premature Anthropic EOF to fail closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAnthropicSSERejectsMalformedPacketAfterFinishReason(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + "event: message_delta\n" +
|
||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"event: broken\n" +
|
||||
"data: {not-json}\n\n"
|
||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
||||
t.Fatal("expected trailing malformed packet to override finish reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAnthropicSSERejectsInvalidOrOutOfOrderCompletion(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"invalid stop reason": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"made_up\"}}\n\n" +
|
||||
"data: {\"type\":\"message_stop\"}\n\n",
|
||||
"message stop first": "data: {\"type\":\"message_stop\"}\n\n" +
|
||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n",
|
||||
"stop reason wrong event": anthropicTestMessageStart + "data: {\"type\":\"content_block_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"data: {\"type\":\"message_stop\"}\n\n",
|
||||
"empty provider error after completion": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"data: {\"type\":\"message_stop\"}\n\n" +
|
||||
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n",
|
||||
}
|
||||
for name, payload := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
||||
t.Fatal("expected invalid Anthropic completion protocol to fail")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessAnthropicStreamRejectsPrematureDone(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 +
|
||||
"data: [DONE]\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 4)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
t.Fatalf("premature [DONE] produced success: %#v", item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessAnthropicStreamRejectsEmptyProviderErrorAfterCompletion(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"data: {\"type\":\"message_stop\"}\n\n" +
|
||||
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 4)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
t.Fatalf("empty-message provider error was ignored: %#v", item)
|
||||
}
|
||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("expected provider error terminal")
|
||||
}
|
||||
|
||||
func TestProcessAnthropicStreamRejectsPrematureEOF(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 4)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
|
||||
successTerminals, errorTerminals := 0, 0
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
successTerminals++
|
||||
}
|
||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
||||
errorTerminals++
|
||||
}
|
||||
}
|
||||
if successTerminals != 0 || errorTerminals != 1 {
|
||||
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessAnthropicStreamRejectsMalformedPacketAfterFinishReason(t *testing.T) {
|
||||
payload := anthropicTestMessageStart + "event: message_delta\n" +
|
||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
||||
"event: broken\n" +
|
||||
"data: {not-json}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
||||
stream := make(chan types.StreamResponse, 4)
|
||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
||||
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
t.Fatalf("unexpected success terminal before trailing malformed packet: %#v", item)
|
||||
}
|
||||
if item.ResponseType == types.ResponseTypeError {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("expected terminal stream error")
|
||||
}
|
||||
@@ -13,6 +13,33 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAnthropicBuildRequestPreservesImageDataURI(t *testing.T) {
|
||||
chat := &AnthropicChat{modelName: "claude-test"}
|
||||
req, err := chat.buildRequest([]Message{{
|
||||
Role: "user",
|
||||
Content: "describe this image",
|
||||
Images: []string{"data:image/png;base64,YWJj"},
|
||||
}}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, req.Messages, 1)
|
||||
blocks, ok := req.Messages[0].Content.([]anthropicContentBlock)
|
||||
require.True(t, ok, "content type = %T", req.Messages[0].Content)
|
||||
require.Len(t, blocks, 2)
|
||||
assert.Equal(t, "text", blocks[0].Type)
|
||||
assert.Equal(t, "describe this image", blocks[0].Text)
|
||||
assert.Equal(t, "image", blocks[1].Type)
|
||||
require.NotNil(t, blocks[1].Source)
|
||||
assert.Equal(t, "base64", blocks[1].Source.Type)
|
||||
assert.Equal(t, "image/png", blocks[1].Source.MediaType)
|
||||
assert.Equal(t, "YWJj", blocks[1].Source.Data)
|
||||
}
|
||||
|
||||
func TestAnthropicBuildRequestRejectsMalformedImage(t *testing.T) {
|
||||
chat := &AnthropicChat{modelName: "claude-test"}
|
||||
_, err := chat.buildRequest([]Message{{Role: "user", Content: "describe", Images: []string{"not-an-image"}}}, nil)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestAnthropicChat(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
|
||||
@@ -147,8 +174,14 @@ func TestAnthropicChat_SSEResponse(t *testing.T) {
|
||||
_, _ = w.Write([]byte(`event: message_start
|
||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
|
||||
event: content_block_start
|
||||
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: content_block_stop
|
||||
data: {"type":"content_block_stop","index":0}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||
@@ -191,8 +224,14 @@ func TestAnthropicChat_ChatStream(t *testing.T) {
|
||||
_, _ = w.Write([]byte(`event: message_start
|
||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
|
||||
event: content_block_start
|
||||
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: content_block_stop
|
||||
data: {"type":"content_block_stop","index":0}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||
|
||||
@@ -98,14 +98,27 @@ type Chat interface {
|
||||
GetModelID() string
|
||||
}
|
||||
|
||||
type imageInputCapable interface {
|
||||
SupportsImageInput() bool
|
||||
}
|
||||
|
||||
// SupportsImageInput reports whether the concrete Chat transport can preserve
|
||||
// image input. Unknown implementations fail closed instead of trusting model
|
||||
// metadata alone.
|
||||
func SupportsImageInput(model Chat) bool {
|
||||
capable, ok := model.(imageInputCapable)
|
||||
return ok && capable.SupportsImageInput()
|
||||
}
|
||||
|
||||
type ChatConfig struct {
|
||||
Source types.ModelSource
|
||||
BaseURL string
|
||||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
Provider string
|
||||
ExtraConfig map[string]string
|
||||
Source types.ModelSource
|
||||
BaseURL string
|
||||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
Provider string
|
||||
SupportsVision bool
|
||||
ExtraConfig map[string]string
|
||||
// CustomHeaders 允许在调用远程 OpenAI 兼容 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string
|
||||
AppID string
|
||||
@@ -120,17 +133,23 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) *ChatConfig {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
supportsVision := m.Parameters.SupportsVision
|
||||
if provider.ProviderName(m.Parameters.Provider) == provider.ProviderWeKnoraCloud &&
|
||||
m.Type == types.ModelTypeKnowledgeQA && strings.EqualFold(strings.TrimSpace(m.Name), "chat") {
|
||||
supportsVision = true
|
||||
}
|
||||
return &ChatConfig{
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
Provider: m.Parameters.Provider,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
Provider: m.Parameters.Provider,
|
||||
SupportsVision: supportsVision,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -74,24 +74,3 @@ func readLocalStorageBytes(storagePath string) []byte {
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// isMultimodalNotSupportedError checks if an error indicates the model does not
|
||||
// support multimodal/image input.
|
||||
func isMultimodalNotSupportedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return (strings.Contains(msg, "multimodal") || strings.Contains(msg, "image") || strings.Contains(msg, "vision")) &&
|
||||
(strings.Contains(msg, "not support") || strings.Contains(msg, "unsupported") || strings.Contains(msg, "400"))
|
||||
}
|
||||
|
||||
// stripImagesFromMessages returns a copy of messages with all image data removed.
|
||||
func stripImagesFromMessages(messages []Message) []Message {
|
||||
cleaned := make([]Message, len(messages))
|
||||
for i, msg := range messages {
|
||||
cleaned[i] = msg
|
||||
cleaned[i].Images = nil
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildLangfuseMessagesRedactsMultiContentImage(t *testing.T) {
|
||||
secret := "TOPSECRETBASE64"
|
||||
msgs := buildLangfuseMessages([]Message{{
|
||||
Role: "user",
|
||||
MultiContent: []MessageContentPart{
|
||||
{Type: "image_url", ImageURL: &ImageURL{URL: "data:image/png;base64," + secret}},
|
||||
{Type: "text", Text: "describe"},
|
||||
},
|
||||
}})
|
||||
encoded, err := json.Marshal(msgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serialized := string(encoded)
|
||||
if strings.Contains(serialized, secret) || strings.Contains(strings.ToLower(serialized), "data:image") {
|
||||
t.Fatalf("Langfuse input leaked image payload: %s", serialized)
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,12 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
|
||||
@@ -16,8 +18,9 @@ type langfuseChat struct {
|
||||
inner Chat
|
||||
}
|
||||
|
||||
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
||||
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
||||
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
||||
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
||||
func (l *langfuseChat) SupportsImageInput() bool { return SupportsImageInput(l.inner) }
|
||||
|
||||
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
mgr := langfuse.GetManager()
|
||||
@@ -47,7 +50,7 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
|
||||
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
|
||||
)
|
||||
}
|
||||
gen.Finish(output, usage, err)
|
||||
gen.Finish(output, usage, sanitizeLangfuseError(err))
|
||||
return resp, err
|
||||
}
|
||||
|
||||
@@ -71,7 +74,7 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
|
||||
ch, err := l.inner.ChatStream(genCtx, messages, opts)
|
||||
if err != nil {
|
||||
gen.Finish(nil, nil, err)
|
||||
gen.Finish(nil, nil, sanitizeLangfuseError(err))
|
||||
return ch, err
|
||||
}
|
||||
if ch == nil {
|
||||
@@ -131,10 +134,21 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
||||
"role": m.Role,
|
||||
}
|
||||
if m.Content != "" {
|
||||
entry["content"] = m.Content
|
||||
entry["content"] = secutils.RedactDataURLsForLog(m.Content)
|
||||
}
|
||||
if len(m.MultiContent) > 0 {
|
||||
entry["content"] = m.MultiContent
|
||||
parts := make([]MessageContentPart, len(m.MultiContent))
|
||||
copy(parts, m.MultiContent)
|
||||
for i := range parts {
|
||||
parts[i].Text = secutils.RedactDataURLsForLog(parts[i].Text)
|
||||
if parts[i].ImageURL == nil {
|
||||
continue
|
||||
}
|
||||
imageURL := *parts[i].ImageURL
|
||||
imageURL.URL = "[image-url-redacted]"
|
||||
parts[i].ImageURL = &imageURL
|
||||
}
|
||||
entry["content"] = parts
|
||||
}
|
||||
if m.Name != "" {
|
||||
entry["name"] = m.Name
|
||||
@@ -143,10 +157,15 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
||||
entry["tool_call_id"] = m.ToolCallID
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
entry["tool_calls"] = m.ToolCalls
|
||||
calls := make([]ToolCall, len(m.ToolCalls))
|
||||
copy(calls, m.ToolCalls)
|
||||
for i := range calls {
|
||||
calls[i].Function.Arguments = secutils.RedactDataURLsForLog(calls[i].Function.Arguments)
|
||||
}
|
||||
entry["tool_calls"] = calls
|
||||
}
|
||||
if m.ReasoningContent != "" {
|
||||
entry["reasoning_content"] = m.ReasoningContent
|
||||
entry["reasoning_content"] = secutils.RedactDataURLsForLog(m.ReasoningContent)
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
@@ -157,17 +176,32 @@ func buildLangfuseGenerationOutput(
|
||||
content, reasoningContent, finishReason string,
|
||||
toolCalls []types.LLMToolCall,
|
||||
) map[string]interface{} {
|
||||
var safeToolCalls []types.LLMToolCall
|
||||
if toolCalls != nil {
|
||||
safeToolCalls = make([]types.LLMToolCall, len(toolCalls))
|
||||
copy(safeToolCalls, toolCalls)
|
||||
}
|
||||
for i := range safeToolCalls {
|
||||
safeToolCalls[i].Function.Arguments = secutils.RedactDataURLsForLog(safeToolCalls[i].Function.Arguments)
|
||||
}
|
||||
output := map[string]interface{}{
|
||||
"content": content,
|
||||
"tool_calls": toolCalls,
|
||||
"content": secutils.RedactDataURLsForLog(content),
|
||||
"tool_calls": safeToolCalls,
|
||||
"finish_reason": finishReason,
|
||||
}
|
||||
if reasoningContent != "" {
|
||||
output["reasoning_content"] = reasoningContent
|
||||
output["reasoning_content"] = secutils.RedactDataURLsForLog(reasoningContent)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func sanitizeLangfuseError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return errors.New(secutils.RedactDataURLsForLog(err.Error()))
|
||||
}
|
||||
|
||||
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
|
||||
if opts == nil {
|
||||
return nil
|
||||
|
||||
@@ -29,7 +29,7 @@ func buildLLMMessages(messages []Message) []logger.LLMMessage {
|
||||
parts = append(parts, mc.Text)
|
||||
case "image_url":
|
||||
if mc.ImageURL != nil {
|
||||
parts = append(parts, fmt.Sprintf("[image_url: %s]", truncateForDebug(mc.ImageURL.URL, 120)))
|
||||
parts = append(parts, fmt.Sprintf("[image_url: redacted (%d bytes)]", len(mc.ImageURL.URL)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildLLMMessagesRedactsMultiContentImageURL(t *testing.T) {
|
||||
secret := "TOPSECRETBASE64"
|
||||
messages := buildLLMMessages([]Message{{
|
||||
Role: "user",
|
||||
MultiContent: []MessageContentPart{{
|
||||
Type: "image_url",
|
||||
ImageURL: &ImageURL{URL: "data:image/png;base64," + secret},
|
||||
}},
|
||||
}})
|
||||
if len(messages) != 1 {
|
||||
t.Fatalf("messages length = %d", len(messages))
|
||||
}
|
||||
got := messages[0].Content
|
||||
if strings.Contains(got, secret) || strings.Contains(strings.ToLower(got), "data:image") {
|
||||
t.Fatalf("buildLLMMessages leaked image URL: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "redacted") {
|
||||
t.Fatalf("buildLLMMessages missing redaction marker: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -16,8 +16,9 @@ type debugChat struct {
|
||||
inner Chat
|
||||
}
|
||||
|
||||
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
|
||||
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
|
||||
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
|
||||
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
|
||||
func (d *debugChat) SupportsImageInput() bool { return SupportsImageInput(d.inner) }
|
||||
|
||||
func (d *debugChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
callStart := time.Now()
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOllamaBuildChatRequestRejectsAnyUnresolvableImage(t *testing.T) {
|
||||
c := &OllamaChat{supportsVision: true}
|
||||
_, err := c.buildChatRequest([]Message{{
|
||||
Role: "user",
|
||||
Content: "describe",
|
||||
Images: []string{"data:image/png;base64,%%%INVALID%%%"},
|
||||
}}, nil, false)
|
||||
if err == nil {
|
||||
t.Fatal("expected malformed image to fail closed")
|
||||
}
|
||||
if strings.Contains(err.Error(), "%%%INVALID%%%") {
|
||||
t.Fatalf("image value leaked in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAnthropicResponseBodyRejectsOversize(t *testing.T) {
|
||||
_, err := readAnthropicResponseBody(io.NopCloser(strings.NewReader(strings.Repeat("x", anthropicResponseBodyLimit+1))))
|
||||
if err == nil {
|
||||
t.Fatal("expected oversized Anthropic response rejection")
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package chat
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -13,27 +13,33 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
ollamaapi "github.com/ollama/ollama/api"
|
||||
)
|
||||
|
||||
// OllamaChat 实现了基于 Ollama 的聊天
|
||||
type OllamaChat struct {
|
||||
modelName string
|
||||
modelID string
|
||||
ollamaService *ollama.OllamaService
|
||||
modelName string
|
||||
modelID string
|
||||
supportsVision bool
|
||||
ollamaService *ollama.OllamaService
|
||||
}
|
||||
|
||||
func (c *OllamaChat) SupportsImageInput() bool { return c.supportsVision }
|
||||
|
||||
// NewOllamaChat 创建 Ollama 聊天实例
|
||||
func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*OllamaChat, error) {
|
||||
return &OllamaChat{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
ollamaService: ollamaService,
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
supportsVision: config.SupportsVision,
|
||||
ollamaService: ollamaService,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// convertMessages 转换消息格式为Ollama API格式
|
||||
func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message {
|
||||
// convertMessages converts messages and fails closed if any declared image
|
||||
// cannot be resolved. Silently dropping one image would change the user's input.
|
||||
func (c *OllamaChat) convertMessages(messages []Message) ([]ollamaapi.Message, error) {
|
||||
ollamaMessages := make([]ollamaapi.Message, 0, len(messages))
|
||||
for _, msg := range messages {
|
||||
msgOllama := ollamaapi.Message{
|
||||
@@ -46,47 +52,69 @@ func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message {
|
||||
}
|
||||
if len(msg.Images) > 0 && msg.Role == "user" {
|
||||
for _, imgURL := range msg.Images {
|
||||
if imgData := resolveImageForOllama(imgURL); imgData != nil {
|
||||
msgOllama.Images = append(msgOllama.Images, imgData)
|
||||
imgData, err := resolveImageForOllama(imgURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msgOllama.Images = append(msgOllama.Images, imgData)
|
||||
}
|
||||
}
|
||||
ollamaMessages = append(ollamaMessages, msgOllama)
|
||||
}
|
||||
return ollamaMessages
|
||||
return ollamaMessages, nil
|
||||
}
|
||||
|
||||
// resolveImageForOllama resolves an image URL into raw bytes for Ollama.
|
||||
// Handles local serving paths (/files/...), data URIs, and remote HTTP URLs.
|
||||
func resolveImageForOllama(imageURL string) ollamaapi.ImageData {
|
||||
func resolveImageForOllama(imageURL string) (ollamaapi.ImageData, error) {
|
||||
if data := resolveImageURLForOllama(imageURL); data != nil {
|
||||
return data
|
||||
return data, nil
|
||||
}
|
||||
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
if err := secutils.ValidateURLForSSRF(imageURL); err != nil {
|
||||
return nil, errors.New("ollama image URL is not allowed")
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: secutils.SSRFSafeDialContext,
|
||||
},
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
resp, err := client.Get(imageURL)
|
||||
if err != nil {
|
||||
return nil
|
||||
return nil, errors.New("ollama image download failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024))
|
||||
if err != nil {
|
||||
return nil
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, errors.New("ollama image download failed")
|
||||
}
|
||||
return data
|
||||
const maxOllamaImageBytes = 20 * 1024 * 1024
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxOllamaImageBytes+1))
|
||||
if err != nil {
|
||||
return nil, errors.New("ollama image download failed")
|
||||
}
|
||||
if len(data) == 0 || len(data) > maxOllamaImageBytes {
|
||||
return nil, errors.New("ollama image payload is invalid")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
return nil
|
||||
return nil, errors.New("ollama image payload is invalid")
|
||||
}
|
||||
|
||||
// buildChatRequest 构建聊天请求参数
|
||||
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) *ollamaapi.ChatRequest {
|
||||
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) (*ollamaapi.ChatRequest, error) {
|
||||
// 设置流式标志
|
||||
streamFlag := isStream
|
||||
|
||||
// 构建请求参数
|
||||
convertedMessages, err := c.convertMessages(messages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chatReq := &ollamaapi.ChatRequest{
|
||||
Model: c.modelName,
|
||||
Messages: c.convertMessages(messages),
|
||||
Messages: convertedMessages,
|
||||
Stream: &streamFlag,
|
||||
Options: make(map[string]interface{}),
|
||||
}
|
||||
@@ -113,7 +141,7 @@ func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isS
|
||||
}
|
||||
}
|
||||
|
||||
return chatReq
|
||||
return chatReq, nil
|
||||
}
|
||||
|
||||
// Chat 进行非流式聊天
|
||||
@@ -124,7 +152,10 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
||||
}
|
||||
|
||||
// 构建请求参数
|
||||
chatReq := c.buildChatRequest(messages, opts, false)
|
||||
chatReq, err := c.buildChatRequest(messages, opts, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 记录请求日志
|
||||
logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName)
|
||||
@@ -134,7 +165,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
||||
var promptTokens, completionTokens int
|
||||
|
||||
// 使用 Ollama 客户端发送请求
|
||||
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
||||
err = c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
||||
responseContent = resp.Message.Content
|
||||
// 当 Content 为空但 Thinking 有内容时(如推理模型未正确配置 thinking 参数),使用 Thinking 作为兜底
|
||||
if responseContent == "" && resp.Message.Thinking != "" {
|
||||
@@ -151,7 +182,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("聊天请求失败: %w", err)
|
||||
return nil, errors.New("ollama chat failed")
|
||||
}
|
||||
|
||||
usage := types.TokenUsage{
|
||||
@@ -180,7 +211,10 @@ func (c *OllamaChat) ChatStream(
|
||||
}
|
||||
|
||||
// 构建请求参数
|
||||
chatReq := c.buildChatRequest(messages, opts, true)
|
||||
chatReq, err := c.buildChatRequest(messages, opts, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 记录请求日志
|
||||
logger.GetLogger(ctx).Infof("发送流式聊天请求到模型 %s", c.modelName)
|
||||
@@ -193,6 +227,8 @@ func (c *OllamaChat) ChatStream(
|
||||
defer close(streamChan)
|
||||
|
||||
var thinking thinkingEmitter
|
||||
var terminalUsage *types.TokenUsage
|
||||
providerDone := false
|
||||
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
||||
// 发送思考内容(支持 Qwen3、DeepSeek 等推理模型)
|
||||
if resp.Message.Thinking != "" {
|
||||
@@ -247,32 +283,32 @@ func (c *OllamaChat) ChatStream(
|
||||
}
|
||||
|
||||
if resp.Done {
|
||||
var usage *types.TokenUsage
|
||||
providerDone = true
|
||||
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
|
||||
usage = &types.TokenUsage{
|
||||
terminalUsage = &types.TokenUsage{
|
||||
PromptTokens: resp.PromptEvalCount,
|
||||
CompletionTokens: resp.EvalCount,
|
||||
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
|
||||
}
|
||||
}
|
||||
logUsage(ctx, c.modelName, usage)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("流式聊天请求失败: %v", err)
|
||||
// 发送错误响应
|
||||
if err != nil || !providerDone {
|
||||
logger.GetLogger(ctx).Error("流式聊天请求失败")
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: err.Error(),
|
||||
Content: "ollama stream failed",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
logUsage(ctx, c.modelName, terminalUsage)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Done: true,
|
||||
Usage: terminalUsage,
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ func (c *RemoteAPIChat) processStream(
|
||||
for {
|
||||
response, err := stream.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if err == io.EOF && state.lastFinishReason != "" {
|
||||
logUsage(ctx, c.modelName, state.usage)
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -134,9 +134,10 @@ func (c *RemoteAPIChat) processStream(
|
||||
FinishReason: state.lastFinishReason,
|
||||
}
|
||||
} else {
|
||||
logger.Warnf(ctx, "Chat completion stream ended before finish reason")
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: err.Error(),
|
||||
Content: "chat completion stream failed",
|
||||
Done: true,
|
||||
}
|
||||
}
|
||||
@@ -178,23 +179,26 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
||||
for {
|
||||
event, err := reader.ReadEvent()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if err == io.EOF && state.lastFinishReason != "" {
|
||||
logUsage(ctx, c.modelName, state.usage)
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
ToolCalls: toolCalls,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
Usage: state.usage,
|
||||
FinishReason: state.lastFinishReason,
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != io.EOF {
|
||||
logger.Warnf(ctx, "Chat stream read failed")
|
||||
} else {
|
||||
logger.Errorf(ctx, "Stream read error: %v", err)
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: err.Error(),
|
||||
Done: true,
|
||||
}
|
||||
logger.Warnf(ctx, "Chat stream ended before finish reason")
|
||||
}
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: "chat stream ended before completion",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -204,6 +208,14 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
||||
}
|
||||
|
||||
if event.Done {
|
||||
if state.lastFinishReason == "" {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: "chat stream ended before completion",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
logUsage(ctx, c.modelName, state.usage)
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -212,6 +224,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
||||
Done: true,
|
||||
ToolCalls: toolCalls,
|
||||
Usage: state.usage,
|
||||
FinishReason: state.lastFinishReason,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -241,8 +254,13 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(event.Data, &streamResp); err != nil {
|
||||
logger.Errorf(ctx, "Failed to parse stream response: %v", err)
|
||||
continue
|
||||
logger.Warnf(ctx, "Failed to parse chat stream response")
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeError,
|
||||
Content: "chat stream response was invalid",
|
||||
Done: true,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if streamResp.Usage != nil {
|
||||
@@ -453,38 +471,18 @@ func (c *RemoteAPIChat) processStreamDelta(
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: delta.Content,
|
||||
Done: isDone,
|
||||
Done: false,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
|
||||
if isDone && len(state.toolCallMap) > 0 {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Content: "",
|
||||
Done: true,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure thinking done is sent when stream finishes without any answer content
|
||||
// (e.g., model only produced reasoning then hit finish_reason with empty content).
|
||||
// Ensure thinking done is sent when the provider supplied a finish reason.
|
||||
// The outer stream reader emits the sole terminal answer after it confirms
|
||||
// the provider ended successfully.
|
||||
if isDone {
|
||||
state.finish(streamChan)
|
||||
}
|
||||
|
||||
// Catch-all: isDone but none of the above branches sent a response with
|
||||
// FinishReason (empty content, no tool calls, no thinking). This prevents
|
||||
// the finish_reason from being lost in the streaming pipeline.
|
||||
if isDone && delta.Content == "" && len(state.toolCallMap) == 0 {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Done: true,
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processToolCallsDelta 处理 tool calls 的增量更新
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestRemoteAPIProviderErrorDoesNotExposeImagePayload(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
secret := "TOPSECRETIMAGEPAYLOAD"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"data:image/png;base64,` + secret + `"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
model, err := NewRemoteAPIChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
BaseURL: server.URL + "/v1",
|
||||
ModelName: "vision-test",
|
||||
APIKey: "test-key",
|
||||
Provider: "openai",
|
||||
SupportsVision: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = model.Chat(context.Background(), []Message{{
|
||||
Role: "user", Content: "inspect", Images: []string{"data:image/png;base64," + secret},
|
||||
}}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected provider error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
|
||||
t.Fatalf("provider error leaked image payload: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessRawHTTPStreamPrematureEOFFailsClosed(t *testing.T) {
|
||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
||||
stream := make(chan types.StreamResponse)
|
||||
model := &RemoteAPIChat{modelName: "cloud-vision"}
|
||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
||||
|
||||
var got []types.StreamResponse
|
||||
for chunk := range stream {
|
||||
got = append(got, chunk)
|
||||
}
|
||||
if len(got) == 0 || got[len(got)-1].ResponseType != types.ResponseTypeError {
|
||||
t.Fatalf("stream = %#v; want terminal error on premature EOF", got)
|
||||
}
|
||||
if got[len(got)-1].Content != "chat stream ended before completion" {
|
||||
t.Fatalf("terminal error = %q", got[len(got)-1].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessRawHTTPStreamFinishReasonThenEOFCompletes(t *testing.T) {
|
||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"answer\"},\"finish_reason\":\"stop\"}]}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
||||
stream := make(chan types.StreamResponse)
|
||||
model := &RemoteAPIChat{modelName: "cloud-vision"}
|
||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
||||
|
||||
var got []types.StreamResponse
|
||||
for chunk := range stream {
|
||||
got = append(got, chunk)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("expected stream chunks")
|
||||
}
|
||||
last := got[len(got)-1]
|
||||
if last.ResponseType == types.ResponseTypeError || !last.Done || last.FinishReason != "stop" {
|
||||
t.Fatalf("stream = %#v; want successful stop completion", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessRawHTTPStreamFinishReasonThenMalformedPacketFailsWithoutSuccessTerminal(t *testing.T) {
|
||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n" +
|
||||
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
|
||||
"data: {not-json}\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
||||
stream := make(chan types.StreamResponse, 8)
|
||||
model := &RemoteAPIChat{modelName: "stream-test"}
|
||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
||||
|
||||
successTerminals, errorTerminals := 0, 0
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
successTerminals++
|
||||
}
|
||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
||||
errorTerminals++
|
||||
}
|
||||
}
|
||||
if successTerminals != 0 || errorTerminals != 1 {
|
||||
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessRawHTTPStreamEmitsExactlyOneSuccessTerminal(t *testing.T) {
|
||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\n" +
|
||||
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
|
||||
"data: [DONE]\n\n"
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
||||
stream := make(chan types.StreamResponse, 8)
|
||||
model := &RemoteAPIChat{modelName: "stream-test"}
|
||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
||||
|
||||
successTerminals := 0
|
||||
for item := range stream {
|
||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
||||
successTerminals++
|
||||
}
|
||||
if item.ResponseType == types.ResponseTypeError {
|
||||
t.Fatalf("unexpected stream error: %#v", item)
|
||||
}
|
||||
}
|
||||
if successTerminals != 1 {
|
||||
t.Fatalf("success terminals = %d, want 1", successTerminals)
|
||||
}
|
||||
}
|
||||
@@ -21,14 +21,15 @@ import (
|
||||
// providerAdapter(见 provider.go),thinking 编码委托给 ThinkingStrategy
|
||||
// (见 thinking.go)。
|
||||
type RemoteAPIChat struct {
|
||||
modelName string
|
||||
client *openai.Client
|
||||
modelID string
|
||||
baseURL string
|
||||
apiKey string
|
||||
provider provider.ProviderName
|
||||
appID string
|
||||
appSecret string
|
||||
modelName string
|
||||
client *openai.Client
|
||||
modelID string
|
||||
baseURL string
|
||||
apiKey string
|
||||
provider provider.ProviderName
|
||||
supportsVision bool
|
||||
appID string
|
||||
appSecret string
|
||||
// customHeaders 为用户在模型配置中指定的自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
customHeaders map[string]string
|
||||
|
||||
@@ -38,6 +39,8 @@ type RemoteAPIChat struct {
|
||||
thinkingOverride ThinkingStrategy
|
||||
}
|
||||
|
||||
func (c *RemoteAPIChat) SupportsImageInput() bool { return c.supportsVision }
|
||||
|
||||
// NewRemoteAPIChat 创建远程 API 聊天实例
|
||||
func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
||||
if chatConfig.BaseURL != "" {
|
||||
@@ -103,6 +106,7 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
||||
baseURL: chatConfig.BaseURL,
|
||||
apiKey: apiKey,
|
||||
provider: providerName,
|
||||
supportsVision: chatConfig.SupportsVision,
|
||||
appID: chatConfig.AppID,
|
||||
appSecret: chatConfig.AppSecret,
|
||||
customHeaders: chatConfig.CustomHeaders,
|
||||
@@ -120,7 +124,11 @@ func (c *RemoteAPIChat) authCreds() authCreds {
|
||||
// transform and parameter shaping (but not thinking, which may wrap the body).
|
||||
func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest {
|
||||
req := c.BuildChatCompletionRequest(messages, opts, isStream)
|
||||
req.Messages = c.adapter.TransformMessages(req.Messages)
|
||||
// WeKnora Cloud text models retain the legacy plain-text downgrade, while
|
||||
// vision-capable Chat models must keep the OpenAI-compatible image parts.
|
||||
if c.provider != provider.ProviderWeKnoraCloud || !c.supportsVision {
|
||||
req.Messages = c.adapter.TransformMessages(req.Messages)
|
||||
}
|
||||
c.adapter.ShapeRequest(&req, opts, isStream)
|
||||
return req
|
||||
}
|
||||
@@ -180,15 +188,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
c.logRequest(timeoutCtx, req, false)
|
||||
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
|
||||
if err != nil {
|
||||
if isMultimodalNotSupportedError(err) {
|
||||
logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
cleaned := stripImagesFromMessages(messages)
|
||||
req = c.shapedRequest(cleaned, opts, false)
|
||||
resp, err = c.client.CreateChatCompletion(timeoutCtx, req)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create chat completion: %w", err)
|
||||
}
|
||||
return nil, fmt.Errorf("create chat completion failed")
|
||||
}
|
||||
|
||||
result, err := c.parseCompletionResponse(&resp)
|
||||
@@ -231,13 +231,13 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
|
||||
|
||||
resp, err := rawHTTPClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
return nil, fmt.Errorf("send chat request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
@@ -259,6 +259,10 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func remoteAPIStatusError(status int, _ []byte) error {
|
||||
return fmt.Errorf("API request failed with status %d", status)
|
||||
}
|
||||
|
||||
// ChatStream 进行流式聊天
|
||||
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
// 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长,
|
||||
@@ -287,17 +291,9 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
|
||||
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||
if err != nil {
|
||||
if isMultimodalNotSupportedError(err) {
|
||||
logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
cleaned := stripImagesFromMessages(messages)
|
||||
req = c.shapedRequest(cleaned, opts, true)
|
||||
stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||
}
|
||||
if err != nil {
|
||||
cancel()
|
||||
close(streamChan)
|
||||
return nil, fmt.Errorf("create chat completion stream: %w", err)
|
||||
}
|
||||
cancel()
|
||||
close(streamChan)
|
||||
return nil, fmt.Errorf("create chat completion stream failed")
|
||||
}
|
||||
|
||||
go func() {
|
||||
@@ -363,13 +359,13 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri
|
||||
|
||||
resp, err := rawHTTPClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
return nil, fmt.Errorf("send chat request failed")
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
||||
}
|
||||
|
||||
streamChan := make(chan types.StreamResponse)
|
||||
|
||||
@@ -3,7 +3,11 @@ package chat
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +17,17 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRemoteAPIStatusErrorDoesNotExposeResponseBody(t *testing.T) {
|
||||
secret := "data:image/png;base64,TOPSECRET"
|
||||
err := remoteAPIStatusError(http.StatusBadRequest, []byte(secret))
|
||||
if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "TOPSECRET") {
|
||||
t.Fatalf("remote API error leaked response body: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "400") {
|
||||
t.Fatalf("remote API error = %v, want status code", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRemoteChat(t *testing.T) *RemoteAPIChat {
|
||||
t.Helper()
|
||||
|
||||
@@ -533,3 +548,44 @@ func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) {
|
||||
assert.Contains(t, string(b), `"cached_tokens":7`)
|
||||
})
|
||||
}
|
||||
|
||||
func newMultimodalUnsupportedTestChat(t *testing.T, calls *atomic.Int32) *RemoteAPIChat {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
calls.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"image input is not supported","type":"invalid_request_error","code":"unsupported"}}`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
chat := newTestRemoteChat(t)
|
||||
cfg := openai.DefaultConfig("test-key")
|
||||
cfg.BaseURL = server.URL
|
||||
chat.client = openai.NewClientWithConfig(cfg)
|
||||
return chat
|
||||
}
|
||||
|
||||
func TestRemoteAPIChat_MultimodalUnsupportedDoesNotStripImages(t *testing.T) {
|
||||
messages := []Message{{
|
||||
Role: "user",
|
||||
Content: "read this image",
|
||||
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
|
||||
}}
|
||||
|
||||
t.Run("non-streaming", func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
chat := newMultimodalUnsupportedTestChat(t, &calls)
|
||||
_, err := chat.Chat(context.Background(), messages, &ChatOptions{})
|
||||
require.Error(t, err)
|
||||
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
|
||||
})
|
||||
|
||||
t.Run("streaming", func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
chat := newMultimodalUnsupportedTestChat(t, &calls)
|
||||
_, err := chat.ChatStream(context.Background(), messages, &ChatOptions{})
|
||||
require.Error(t, err)
|
||||
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// streamRawDumpDir returns the directory for per-stream raw packet dumps.
|
||||
@@ -43,7 +45,10 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil
|
||||
}
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -70,10 +75,14 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if err := f.Chmod(0o600); err != nil {
|
||||
_ = f.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
d := &streamPacketDumper{file: f, path: path, model: modelName}
|
||||
_ = d.writeRequest(request)
|
||||
@@ -81,11 +90,19 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
||||
}
|
||||
|
||||
func (d *streamPacketDumper) writeRequest(request any) error {
|
||||
raw, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var redacted any
|
||||
if err := json.Unmarshal([]byte(secutils.CompactImageDataURLForLog(string(raw))), &redacted); err != nil {
|
||||
return err
|
||||
}
|
||||
line, err := json.Marshal(map[string]any{
|
||||
"type": "request",
|
||||
"model": d.model,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"data": request,
|
||||
"data": redacted,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -112,6 +129,7 @@ func (d *streamPacketDumper) WritePacketRaw(raw []byte) {
|
||||
if len(raw) == 0 {
|
||||
return
|
||||
}
|
||||
raw = []byte(secutils.CompactImageDataURLForLog(string(raw)))
|
||||
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
@@ -139,6 +157,7 @@ func (d *streamPacketDumper) WriteError(message string) {
|
||||
if d == nil || d.file == nil || strings.TrimSpace(message) == "" {
|
||||
return
|
||||
}
|
||||
message = secutils.CompactImageDataURLForLog(message)
|
||||
line, err := json.Marshal(map[string]any{
|
||||
"type": "error",
|
||||
"model": d.model,
|
||||
@@ -163,7 +182,7 @@ func (d *streamPacketDumper) WriteHTTPError(statusCode int, body []byte) {
|
||||
"status_code": statusCode,
|
||||
}
|
||||
if len(body) > 0 {
|
||||
trimmed := bytesTrimSpace(body)
|
||||
trimmed := []byte(secutils.CompactImageDataURLForLog(string(bytesTrimSpace(body))))
|
||||
if json.Valid(trimmed) {
|
||||
var parsed any
|
||||
if json.Unmarshal(trimmed, &parsed) == nil {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStreamPacketDumperRedactsImageRequestAndUsesPrivatePermissions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.Chmod(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("WEKNORA_LLM_STREAM_RAW_DUMP_DIR", dir)
|
||||
secret := strings.Repeat("TOPSECRET", 30)
|
||||
dumper := newStreamPacketDumper("vision-model", map[string]any{
|
||||
"messages": []any{map[string]any{
|
||||
"role": "user",
|
||||
"image_url": map[string]any{"url": "data:image/png;base64," + secret},
|
||||
}},
|
||||
})
|
||||
if dumper == nil {
|
||||
t.Fatal("expected dumper")
|
||||
}
|
||||
path := dumper.Path()
|
||||
dumper.Close()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read dump: %v", err)
|
||||
}
|
||||
if strings.Contains(string(data), secret) || strings.Contains(string(data), "data:image/") {
|
||||
t.Fatalf("image data leaked to dump: %s", data)
|
||||
}
|
||||
if !strings.Contains(string(data), "[image-data-redacted]") {
|
||||
t.Fatalf("missing redaction marker: %s", data)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat dump: %v", err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != 0o600 {
|
||||
t.Fatalf("dump permissions = %o, want 600", got)
|
||||
}
|
||||
dirInfo, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("stat dump directory: %v", err)
|
||||
}
|
||||
if got := dirInfo.Mode().Perm(); got != 0o700 {
|
||||
t.Fatalf("dump directory permissions = %o, want 700", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRemoteAPIChat_WeKnoraCloudTransformsImagesByCapability(t *testing.T) {
|
||||
messages := []Message{{
|
||||
Role: "user",
|
||||
Content: "请识别图片",
|
||||
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
|
||||
}}
|
||||
|
||||
build := func(t *testing.T, supportsVision bool) *RemoteAPIChat {
|
||||
t.Helper()
|
||||
model, err := NewRemoteAPIChat(&ChatConfig{
|
||||
Source: types.ModelSourceRemote,
|
||||
ModelName: "cloud-chat",
|
||||
ModelID: "cloud-chat",
|
||||
Provider: string(provider.ProviderWeKnoraCloud),
|
||||
AppID: "test-app",
|
||||
AppSecret: "test-secret",
|
||||
SupportsVision: supportsVision,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return model
|
||||
}
|
||||
|
||||
t.Run("vision chat preserves image parts", func(t *testing.T) {
|
||||
req := build(t, true).shapedRequest(messages, &ChatOptions{}, false)
|
||||
require.Len(t, req.Messages, 1)
|
||||
require.Len(t, req.Messages[0].MultiContent, 2)
|
||||
require.NotNil(t, req.Messages[0].MultiContent[0].ImageURL)
|
||||
require.Equal(t, messages[0].Images[0], req.Messages[0].MultiContent[0].ImageURL.URL)
|
||||
})
|
||||
|
||||
t.Run("text chat keeps legacy downgrade", func(t *testing.T) {
|
||||
req := build(t, false).shapedRequest(messages, &ChatOptions{}, false)
|
||||
require.Len(t, req.Messages, 1)
|
||||
require.Empty(t, req.Messages[0].MultiContent)
|
||||
require.Equal(t, messages[0].Content, req.Messages[0].Content)
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigFromModelRecognizesExistingWeKnoraCloudChatAsVision(t *testing.T) {
|
||||
cloudChat := &types.Model{
|
||||
Name: "chat",
|
||||
Type: types.ModelTypeKnowledgeQA,
|
||||
Source: types.ModelSourceRemote,
|
||||
Parameters: types.ModelParameters{
|
||||
Provider: string(provider.ProviderWeKnoraCloud),
|
||||
},
|
||||
}
|
||||
genericChat := &types.Model{
|
||||
Name: "chat",
|
||||
Type: types.ModelTypeKnowledgeQA,
|
||||
Source: types.ModelSourceRemote,
|
||||
}
|
||||
|
||||
require.True(t, ConfigFromModel(cloudChat, "", "").SupportsVision)
|
||||
require.False(t, ConfigFromModel(genericChat, "", "").SupportsVision)
|
||||
}
|
||||
|
||||
func TestSupportsImageInputUsesRuntimeTransportCapability(t *testing.T) {
|
||||
visionRemote := &RemoteAPIChat{supportsVision: true}
|
||||
textRemote := &RemoteAPIChat{supportsVision: false}
|
||||
anthropicText := &AnthropicChat{}
|
||||
anthropicVision := &AnthropicChat{supportsVision: true}
|
||||
|
||||
require.True(t, SupportsImageInput(visionRemote))
|
||||
require.False(t, SupportsImageInput(textRemote))
|
||||
require.False(t, SupportsImageInput(anthropicText))
|
||||
require.True(t, SupportsImageInput(anthropicVision))
|
||||
require.True(t, SupportsImageInput(&debugChat{inner: visionRemote}))
|
||||
require.True(t, SupportsImageInput(&langfuseChat{inner: visionRemote}))
|
||||
}
|
||||
Reference in New Issue
Block a user