From e2a41ecf0a058f896e1f5ec451d8870ce5e55aff Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 5 Jun 2026 14:42:08 +0800 Subject: [PATCH] feat(thinkingControl): implement thinking mode configuration in model editor - Added a new thinking mode control field in the ModelEditorDialog, allowing users to specify how the agent's thinking mode is formatted for API requests. - Introduced localization strings for thinking mode labels and descriptions in English, Korean, Russian, and Chinese, enhancing user experience across multiple languages. - Updated the model form data structure to include thinking control settings, ensuring proper handling during model edits and saves. - Implemented utility functions for resolving and defaulting thinking control values based on provider and model, improving backend integration. - Added tests for the new thinking control functionality to ensure reliability and correctness in handling user configurations. --- frontend/src/components/ModelEditorDialog.vue | 225 ++++- frontend/src/i18n/locales/en-US.ts | 21 + frontend/src/i18n/locales/ko-KR.ts | 21 + frontend/src/i18n/locales/ru-RU.ts | 21 + frontend/src/i18n/locales/zh-CN.ts | 21 + frontend/src/utils/thinkingControl.test.ts | 36 + frontend/src/utils/thinkingControl.ts | 71 ++ frontend/src/views/agent/AgentEditorModal.vue | 11 + frontend/src/views/settings/ModelSettings.vue | 26 +- .../views/settings/StorageEngineSettings.vue | 1 + internal/models/chat/anthropic.go | 11 +- internal/models/chat/chat.go | 28 +- internal/models/chat/chat_provider_spec.go | 252 ----- internal/models/chat/ollama.go | 35 +- internal/models/chat/openai_request.go | 183 ++++ internal/models/chat/openai_stream.go | 539 ++++++++++ internal/models/chat/provider.go | 247 +++++ internal/models/chat/provider_test.go | 159 +++ internal/models/chat/remote_api.go | 939 ++---------------- internal/models/chat/remote_api_test.go | 4 +- internal/models/chat/stream_emit.go | 36 + internal/models/chat/stream_raw_dump.go | 48 + internal/models/chat/thinking.go | 140 +++ internal/models/chat/thinking_test.go | 127 +++ internal/models/chat/transport.go | 59 ++ internal/models/chat/usage.go | 20 + 26 files changed, 2075 insertions(+), 1206 deletions(-) create mode 100644 frontend/src/utils/thinkingControl.test.ts create mode 100644 frontend/src/utils/thinkingControl.ts delete mode 100644 internal/models/chat/chat_provider_spec.go create mode 100644 internal/models/chat/openai_request.go create mode 100644 internal/models/chat/openai_stream.go create mode 100644 internal/models/chat/provider.go create mode 100644 internal/models/chat/provider_test.go create mode 100644 internal/models/chat/stream_emit.go create mode 100644 internal/models/chat/thinking.go create mode 100644 internal/models/chat/thinking_test.go create mode 100644 internal/models/chat/transport.go create mode 100644 internal/models/chat/usage.go diff --git a/frontend/src/components/ModelEditorDialog.vue b/frontend/src/components/ModelEditorDialog.vue index 3df8e39c..5e97d3ee 100644 --- a/frontend/src/components/ModelEditorDialog.vue +++ b/frontend/src/components/ModelEditorDialog.vue @@ -324,6 +324,31 @@ {{ $t('model.editor.supportsVisionDesc') }} + + +
+ + + +
+ {{ opt.label }} + {{ opt.hint }} +
+
+
+

{{ $t('model.editor.thinkingControlDesc') }}

+
@@ -342,6 +367,11 @@ import { } from '@/api/model' import { useI18n } from 'vue-i18n' import { useUIStore } from '@/stores/ui' +import { + defaultThinkingControl, + resolveThinkingControl, + type ThinkingControlValue, +} from '@/utils/thinkingControl' import SettingDrawer from '@/components/settings/SettingDrawer.vue' import CredentialResource, { type CredentialFieldDef, @@ -367,6 +397,8 @@ interface ModelFormData { interfaceType?: 'ollama' | 'openai' isDefault: boolean supportsVision?: boolean + /** extra_config.thinking_control — how agent thinking on/off maps to API fields. */ + thinkingControl?: string // 自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers) customHeaders?: CustomHeaderItem[] /** LKEAP Rerank:腾讯云 SecretKey(创建时写入 app_secret) */ @@ -564,6 +596,51 @@ const dialogVisible = computed({ const isEdit = computed(() => !!props.modelData) +const showThinkingControlField = computed(() => + props.modelType === 'chat' && formData.value.source === 'remote', +) + +const resolvedThinkingControl = (): ThinkingControlValue => + defaultThinkingControl( + formData.value.provider || '', + formData.value.modelName || '', + ) + +/** 用户是否手动改过思考参数格式(改过则不再自动覆盖,直到换服务商) */ +const thinkingControlManual = ref(false) +/** 正在从 modelData 灌入表单,忽略厂商/来源控件的程序化 change 副作用 */ +const hydratingForm = ref(false) + +const onThinkingControlManualPick = () => { + thinkingControlManual.value = true +} + +const syncThinkingControlToForm = (force = false) => { + if (!showThinkingControlField.value) return + if (!force && !isEdit.value && thinkingControlManual.value) return + formData.value.thinkingControl = resolvedThinkingControl() +} + +const applyThinkingControlFromModelData = () => { + if (!props.modelData || props.modelType !== 'chat' || formData.value.source !== 'remote') return + thinkingControlManual.value = !!props.modelData.thinkingControl + formData.value.thinkingControl = resolveThinkingControl( + props.modelData.thinkingControl, + formData.value.provider || props.modelData.provider || '', + formData.value.modelName || props.modelData.modelName || '', + ) +} + +const thinkingControlOptions = computed(() => { + const keys = ['none', 'chatTemplateKwargs', 'enableThinking', 'thinkingType'] as const + const values = ['none', 'chat_template_kwargs', 'enable_thinking', 'thinking_type'] as const + return keys.map((key, i) => ({ + value: values[i], + label: t(`model.editor.thinkingControl.${key}.label`), + hint: t(`model.editor.thinkingControl.${key}.hint`), + })) +}) + // Header icon for the SettingDrawer — uses the same TDesign icon name table // as the model card list, so the drawer's leading badge visually matches the // card the user just clicked on. @@ -697,6 +774,7 @@ const formData = ref({ interfaceType: 'ollama', isDefault: false, supportsVision: false, + thinkingControl: defaultThinkingControl('generic', ''), customHeaders: [], appSecret: '', lkeapRegion: 'ap-guangzhou', @@ -838,39 +916,53 @@ watch(() => props.visible, (val) => { const currentId = props.modelData?.id ?? null - if (props.modelData) { - // 编辑:始终用最新的 modelData 覆盖。apiKey field is left blank — in - // edit mode the credential is owned by the card, - // not by this form's apiKey field. - formData.value = { - ...props.modelData, - apiKey: '', - customHeaders: Array.isArray(props.modelData.customHeaders) - ? props.modelData.customHeaders.map(h => ({ key: h.key, value: h.value })) - : [] + hydratingForm.value = true + try { + if (props.modelData) { + // 编辑:始终用最新的 modelData 覆盖。apiKey field is left blank — in + // edit mode the credential is owned by the card, + // not by this form's apiKey field. + formData.value = { + ...props.modelData, + apiKey: '', + customHeaders: Array.isArray(props.modelData.customHeaders) + ? props.modelData.customHeaders.map(h => ({ key: h.key, value: h.value })) + : [], + } + applyThinkingControlFromModelData() + } else if (lastOpenedModelId.value !== null || !formData.value.id) { + // 上次是编辑某个模型,或第一次新增 → 重置成空白 + resetForm() } - } else if (lastOpenedModelId.value !== null || !formData.value.id) { - // 上次是编辑某个模型,或第一次新增 → 重置成空白 - resetForm() - } - // 否则:连续两次"新增"打开(中间是点遮罩/ESC 关闭的)→ 保留上次填写 + // 否则:连续两次"新增"打开(中间是点遮罩/ESC 关闭的)→ 保留上次填写 - lastOpenedModelId.value = currentId + lastOpenedModelId.value = currentId - // ReRank 模型强制使用 remote 来源(Ollama 不支持 ReRank) - if (props.modelType === 'rerank') { - formData.value.source = 'remote' - } + // ReRank 模型强制使用 remote 来源(Ollama 不支持 ReRank) + if (props.modelType === 'rerank') { + formData.value.source = 'remote' + } - // 如果当前 provider 是 WeKnoraCloud,检查凭证状态 - if (formData.value.provider === 'weknoracloud') { - checkWkcCredentialStatus() + // 如果当前 provider 是 WeKnoraCloud,检查凭证状态 + if (formData.value.provider === 'weknoracloud') { + checkWkcCredentialStatus() + } + + if (showThinkingControlField.value && !isEdit.value) { + thinkingControlManual.value = false + syncThinkingControlToForm(true) + } + } finally { + nextTick(() => { + hydratingForm.value = false + }) } } }) // 重置表单 const resetForm = () => { + thinkingControlManual.value = false formData.value = { id: generateId(), name: '', // 保留字段但不使用,保存时用 modelName @@ -884,6 +976,7 @@ const resetForm = () => { interfaceType: undefined, isDefault: false, supportsVision: false, + thinkingControl: defaultThinkingControl('generic', ''), customHeaders: [], appSecret: '', lkeapRegion: 'ap-guangzhou', @@ -920,8 +1013,43 @@ const handleProviderChange = (value: string) => { if (value === 'weknoracloud') { checkWkcCredentialStatus() } + if (hydratingForm.value) return + if (props.modelType !== 'chat' || formData.value.source !== 'remote') return + if (!isEdit.value) { + thinkingControlManual.value = false + syncThinkingControlToForm(true) + return + } + // 编辑时仅用户主动换厂商才跟随默认 + thinkingControlManual.value = false + syncThinkingControlToForm(true) } +watch( + () => [formData.value.source, formData.value.provider, formData.value.modelName] as const, + ([source, provider, modelName], [prevSource, prevProvider, prevModelName]) => { + if (hydratingForm.value || isEdit.value) return + if (props.modelType !== 'chat' || source !== 'remote') return + if (source === prevSource && provider === prevProvider && modelName === prevModelName) return + + const providerChanged = provider !== prevProvider + + if (providerChanged) { + thinkingControlManual.value = false + syncThinkingControlToForm(true) + return + } + if (!thinkingControlManual.value) { + syncThinkingControlToForm(true) + return + } + const prevDefault = defaultThinkingControl(prevProvider || '', prevModelName || '') + if (formData.value.thinkingControl === prevDefault) { + syncThinkingControlToForm(true) + } + }, +) + // 监听来源变化,重置校验状态(已合并到下面的 watch) // 生成唯一ID @@ -1388,6 +1516,16 @@ watch(() => formData.value.source, () => { downloading.value = false downloadProgress.value = 0 currentDownloadModel.value = '' + + if ( + !hydratingForm.value + && !isEdit.value + && formData.value.source === 'remote' + && props.modelType === 'chat' + ) { + thinkingControlManual.value = false + syncThinkingControlToForm(true) + } }) // 监听模型名称变化,清理维度检测状态 @@ -1871,6 +2009,14 @@ const handleCancel = () => { &--inline { margin: 0; } + + &--recommend { + color: var(--td-brand-color); + } + + &--warn { + color: var(--td-warning-color); + } } .vision-toggle { @@ -1969,6 +2115,39 @@ const handleCancel = () => { + diff --git a/internal/models/chat/anthropic.go b/internal/models/chat/anthropic.go index 01568526..8fbfc84b 100644 --- a/internal/models/chat/anthropic.go +++ b/internal/models/chat/anthropic.go @@ -153,6 +153,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content) } + logUsage(ctx, c.modelName, &chatResp.Usage) return chatResp, nil } @@ -167,7 +168,9 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } - return c.parseResponse(&chatResp), nil + result := c.parseResponse(&chatResp) + logUsage(ctx, c.modelName, &result.Usage) + return result, nil } func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) { @@ -204,7 +207,7 @@ func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts } streamChan := make(chan types.StreamResponse) - go processAnthropicStream(resp, streamChan) + go processAnthropicStream(ctx, c.modelName, resp, streamChan) return streamChan, nil } @@ -382,7 +385,7 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) { }, nil } -func processAnthropicStream(resp *http.Response, streamChan chan types.StreamResponse) { +func processAnthropicStream(ctx context.Context, model string, resp *http.Response, streamChan chan types.StreamResponse) { defer close(streamChan) defer resp.Body.Close() @@ -394,6 +397,7 @@ func processAnthropicStream(resp *http.Response, streamChan chan types.StreamRes event, err := sseReader.ReadEvent() if err != nil { if err == io.EOF { + logUsage(ctx, model, usage) streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, Content: "", @@ -411,6 +415,7 @@ func processAnthropicStream(resp *http.Response, streamChan chan types.StreamRes return } if event.Done { + logUsage(ctx, model, usage) streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, Content: "", diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go index 74bf1f7b..98e7ef75 100644 --- a/internal/models/chat/chat.go +++ b/internal/models/chat/chat.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" "strings" "github.com/Tencent/WeKnora/internal/models/provider" @@ -150,7 +149,9 @@ func NewChat(config *ChatConfig, ollamaService *ollama.OllamaService) (Chat, err return wrapChatLangfuse(c, err) } -// NewRemoteChat 根据 provider 创建远程聊天实例 +// NewRemoteChat 根据 provider 创建远程聊天实例。 +// Anthropic 走独立的 Messages 协议实现;其余 OpenAI 兼容供应商统一由 +// RemoteAPIChat 处理,provider 特定行为在构造时通过 providerAdapter 解析。 func NewRemoteChat(config *ChatConfig) (Chat, error) { providerName := provider.ProviderName(config.Provider) if providerName == "" { @@ -159,26 +160,5 @@ func NewRemoteChat(config *ChatConfig) (Chat, error) { if providerName == provider.ProviderAnthropic { return NewAnthropicChat(config) } - - remoteChat, err := NewRemoteAPIChat(config) - if err != nil { - return nil, err - } - - // Look up provider-specific behavior from spec registry - if spec := findProviderSpec(providerName, config.ModelName); spec != nil { - if spec.RequestCustomizer != nil { - remoteChat.SetRequestCustomizer(spec.RequestCustomizer) - } - if spec.EndpointCustomizer != nil { - remoteChat.SetEndpointCustomizer(spec.EndpointCustomizer) - } - if spec.HeaderCustomizer != nil { - remoteChat.SetHeaderCustomizer(func(req *http.Request, body []byte) error { - return spec.HeaderCustomizer(remoteChat, req, body) - }) - } - } - - return remoteChat, nil + return NewRemoteAPIChat(config) } diff --git a/internal/models/chat/chat_provider_spec.go b/internal/models/chat/chat_provider_spec.go deleted file mode 100644 index e5df5c0f..00000000 --- a/internal/models/chat/chat_provider_spec.go +++ /dev/null @@ -1,252 +0,0 @@ -package chat - -import ( - "context" - "net/http" - "strings" - - "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/models/provider" - modelutils "github.com/Tencent/WeKnora/internal/models/utils" - "github.com/google/uuid" - "github.com/sashabaranov/go-openai" -) - -// ProviderSpec describes provider-specific behavior for chat completions. -// Each spec is registered with a ProviderName and optionally a model matcher. -type ProviderSpec struct { - Provider provider.ProviderName - // ModelMatcher: if non-nil, this spec only applies when the model name matches. - // Used for sub-provider routing (e.g. Qwen3 within Aliyun). - ModelMatcher func(modelName string) bool - // RequestCustomizer: provider-specific request modification. - RequestCustomizer func(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) - // EndpointCustomizer: provider-specific endpoint URL override. - EndpointCustomizer func(baseURL string, modelID string, isStream bool) string - // HeaderCustomizer: provider-specific raw HTTP header customization. - HeaderCustomizer func(chat *RemoteAPIChat, req *http.Request, body []byte) error -} - -// chatProviderSpecs is the ordered list of provider specs. -// Order matters: more specific specs (with ModelMatcher) should come before generic ones. -var chatProviderSpecs = []ProviderSpec{ - // WeKnoraCloud - { - Provider: provider.ProviderWeKnoraCloud, - RequestCustomizer: weKnoraCloudRequestCustomizer, - EndpointCustomizer: func(baseURL string, _ string, _ bool) string { - return strings.TrimRight(baseURL, "/") + "/api/v1/chat/completions" - }, - HeaderCustomizer: weKnoraCloudHeaderCustomizer, - }, - // Aliyun Qwen Thinking Models (must be before generic Aliyun) - { - Provider: provider.ProviderAliyun, - ModelMatcher: func(name string) bool { return provider.IsQwenThinkingModel(name) }, - RequestCustomizer: qwenThinkingRequestCustomizer, - }, - // LKEAP - { - Provider: provider.ProviderLKEAP, - RequestCustomizer: lkeapRequestCustomizer, - }, - // DeepSeek - { - Provider: provider.ProviderDeepSeek, - RequestCustomizer: deepseekRequestCustomizer, - }, - // Generic (vLLM) - { - Provider: provider.ProviderGeneric, - RequestCustomizer: genericRequestCustomizer, - }, - // Volcengine (火山引擎 Ark) - { - Provider: provider.ProviderVolcengine, - RequestCustomizer: volcengineRequestCustomizer, - }, - // NVIDIA - { - Provider: provider.ProviderNvidia, - RequestCustomizer: genericRequestCustomizer, - }, -} - -// findProviderSpec finds the matching spec for the given provider and model name. -func findProviderSpec(providerName provider.ProviderName, modelName string) *ProviderSpec { - for i := range chatProviderSpecs { - spec := &chatProviderSpecs[i] - if spec.Provider != providerName { - continue - } - if spec.ModelMatcher != nil && !spec.ModelMatcher(modelName) { - continue - } - return spec - } - return nil -} - -// --- Type definitions (moved from wrapper files) --- - -// QwenChatCompletionRequest Qwen 模型的自定义请求结构体 -type QwenChatCompletionRequest struct { - openai.ChatCompletionRequest - EnableThinking *bool `json:"enable_thinking,omitempty"` -} - -// ThinkingConfig 思维链配置(LKEAP / Volcengine 等通用格式) -type ThinkingConfig struct { - Type string `json:"type"` // "enabled" 或 "disabled" -} - -// ThinkingChatCompletionRequest 带 thinking 字段的自定义请求结构体 -// 适用于 LKEAP、Volcengine 等使用 { "thinking": { "type": "enabled" } } 格式的 provider -type ThinkingChatCompletionRequest struct { - openai.ChatCompletionRequest - Thinking *ThinkingConfig `json:"thinking,omitempty"` -} - -// --- Customizer functions --- - -// weKnoraCloudRequestCustomizer 构造 WeKnoraCloud 请求。 -// WeKnoraCloud 走 OpenAI 兼容格式,除了 MultiContent 需要降级为纯文本 Content 之外, -// 其他字段(tools / tool_choice / parallel_tool_calls / response_format / stream_options 等)直接透传, -// 以保证 function calling 等能力可用。 -func weKnoraCloudRequestCustomizer(req *openai.ChatCompletionRequest, _ *ChatOptions, isStream bool) (any, bool) { - cloudReq := *req - cloudReq.Stream = isStream - cloudReq.Messages = convertToWeKnoraCloudMessagesFromOpenAI(req.Messages) - return cloudReq, true -} - -func weKnoraCloudHeaderCustomizer(chat *RemoteAPIChat, req *http.Request, body []byte) error { - requestID := uuid.NewString() - headers := modelutils.Sign(chat.appID, chat.appSecret, requestID, string(body)) - for k, v := range headers { - req.Header.Set(k, v) - } - return nil -} - -// convertToWeKnoraCloudMessagesFromOpenAI 将 MultiContent 降级为纯文本, -// 其它字段(tool_calls / tool_call_id / name 等)完全保留,保证 tool 协议正常。 -func convertToWeKnoraCloudMessagesFromOpenAI(messages []openai.ChatCompletionMessage) []openai.ChatCompletionMessage { - result := make([]openai.ChatCompletionMessage, 0, len(messages)) - for _, m := range messages { - msg := m - if msg.Content == "" && len(msg.MultiContent) > 0 { - var textParts []string - for _, part := range msg.MultiContent { - if part.Type == openai.ChatMessagePartTypeText && part.Text != "" { - textParts = append(textParts, part.Text) - } - } - msg.Content = strings.Join(textParts, "\n") - msg.MultiContent = nil - } - result = append(result, msg) - } - return result -} - -// qwenThinkingRequestCustomizer 自定义 Qwen 系列(阿里云)模型的思考请求 -func qwenThinkingRequestCustomizer( - req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool, -) (any, bool) { - if !isStream { - // Qwen3 模型在非流式请求时需要显式禁用 thinking - qwenReq := QwenChatCompletionRequest{ - ChatCompletionRequest: *req, - } - enableThinking := false - qwenReq.EnableThinking = &enableThinking - return qwenReq, true - } - - // 流式请求:根据 opts.Thinking 启用思考 - qwenReq := QwenChatCompletionRequest{ - ChatCompletionRequest: *req, - } - thinking := false - if opts != nil && opts.Thinking != nil { - thinking = *opts.Thinking - } - qwenReq.EnableThinking = &thinking - - // 必须返回 true 以使用 raw HTTP,否则 SDK 会过滤掉 enable_thinking 字段 - return qwenReq, true -} - -// lkeapRequestCustomizer 自定义 LKEAP 请求 -// 仅对 DeepSeek V3.x 系列模型设置 thinking 参数;R1 系列默认开启思维链 -// 参考:https://cloud.tencent.com/document/product/1772/115963 -func lkeapRequestCustomizer( - req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool, -) (any, bool) { - modelName := req.Model - if !strings.Contains(strings.ToLower(modelName), "deepseek-v3") || opts == nil || opts.Thinking == nil { - return nil, false - } - - lkeapReq := ThinkingChatCompletionRequest{ - ChatCompletionRequest: *req, - } - - thinkingType := "disabled" - if *opts.Thinking { - thinkingType = "enabled" - } - lkeapReq.Thinking = &ThinkingConfig{Type: thinkingType} - - return lkeapReq, true -} - -// deepseekRequestCustomizer 自定义 DeepSeek 请求 -// DeepSeek 模型不支持 tool_choice 参数,需要清除 -func deepseekRequestCustomizer( - req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool, -) (any, bool) { - if opts != nil && opts.ToolChoice != "" { - logger.Infof(context.Background(), "deepseek model, skip tool_choice") - req.ToolChoice = nil - } - return nil, false -} - -// genericRequestCustomizer 自定义 Generic 请求 -// Generic provider(如 vLLM)使用 ChatTemplateKwargs 传递 thinking 参数 -func genericRequestCustomizer( - req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool, -) (any, bool) { - thinking := false - if opts != nil && opts.Thinking != nil { - thinking = *opts.Thinking - } - req.ChatTemplateKwargs = map[string]interface{}{ - "enable_thinking": thinking, - } - return req, true -} - -// volcengineRequestCustomizer 自定义火山引擎请求 -// 火山引擎使用 thinking 参数控制深度思考,格式同 LKEAP: { "type": "enabled"/"disabled" } -func volcengineRequestCustomizer( - req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool, -) (any, bool) { - if opts == nil || opts.Thinking == nil { - return nil, false - } - - vcReq := ThinkingChatCompletionRequest{ - ChatCompletionRequest: *req, - } - - thinkingType := "disabled" - if *opts.Thinking { - thinkingType = "enabled" - } - vcReq.Thinking = &ThinkingConfig{Type: thinkingType} - - return vcReq, true -} diff --git a/internal/models/chat/ollama.go b/internal/models/chat/ollama.go index d0dd5985..989a06fa 100644 --- a/internal/models/chat/ollama.go +++ b/internal/models/chat/ollama.go @@ -154,18 +154,17 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt return nil, fmt.Errorf("聊天请求失败: %w", err) } - totalTokens := promptTokens + completionTokens - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, promptTokens, completionTokens, totalTokens) + usage := types.TokenUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + } + logUsage(ctx, c.modelName, &usage) return &types.ChatResponse{ Content: responseContent, ToolCalls: toolCalls, - Usage: types.TokenUsage{ - PromptTokens: promptTokens, - CompletionTokens: completionTokens, - TotalTokens: totalTokens, - }, + Usage: usage, }, nil } @@ -193,27 +192,16 @@ func (c *OllamaChat) ChatStream( go func() { defer close(streamChan) - hasThinking := false + var thinking thinkingEmitter err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error { // 发送思考内容(支持 Qwen3、DeepSeek 等推理模型) if resp.Message.Thinking != "" { - hasThinking = true - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Content: resp.Message.Thinking, - Done: false, - } + thinking.emit(streamChan, resp.Message.Thinking) } if resp.Message.Content != "" { // 思考阶段结束后,发送思考完成事件 - if hasThinking { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Done: true, - } - hasThinking = false - } + thinking.finish(streamChan) streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, Content: resp.Message.Content, @@ -266,9 +254,8 @@ func (c *OllamaChat) ChatStream( CompletionTokens: resp.EvalCount, TotalTokens: resp.PromptEvalCount + resp.EvalCount, } - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens) } + logUsage(ctx, c.modelName, usage) streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, Done: true, diff --git a/internal/models/chat/openai_request.go b/internal/models/chat/openai_request.go new file mode 100644 index 00000000..fff22905 --- /dev/null +++ b/internal/models/chat/openai_request.go @@ -0,0 +1,183 @@ +package chat + +import ( + "fmt" + + "github.com/sashabaranov/go-openai" +) + +// ConvertMessages 转换消息格式为 OpenAI 格式(导出供子类使用) +func (c *RemoteAPIChat) ConvertMessages(messages []Message) []openai.ChatCompletionMessage { + openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages)) + for _, msg := range messages { + openaiMsg := openai.ChatCompletionMessage{ + Role: msg.Role, + } + + // 优先处理多内容消息(包含图片等) + if len(msg.MultiContent) > 0 { + openaiMsg.MultiContent = make([]openai.ChatMessagePart, 0, len(msg.MultiContent)) + for _, part := range msg.MultiContent { + switch part.Type { + case "text": + openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeText, + Text: part.Text, + }) + case "image_url": + if part.ImageURL != nil { + openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: part.ImageURL.URL, + Detail: openai.ImageURLDetail(part.ImageURL.Detail), + }, + }) + } + } + } + } else if len(msg.Images) > 0 && msg.Role == "user" { + parts := make([]openai.ChatMessagePart, 0, len(msg.Images)+1) + for _, imgURL := range msg.Images { + resolved := resolveImageURLForLLM(imgURL) + parts = append(parts, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: resolved, + Detail: openai.ImageURLDetailAuto, + }, + }) + } + parts = append(parts, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeText, + Text: msg.Content, + }) + openaiMsg.MultiContent = parts + } else if msg.Content != "" { + openaiMsg.Content = msg.Content + } + + if len(msg.ToolCalls) > 0 { + openaiMsg.ToolCalls = make([]openai.ToolCall, 0, len(msg.ToolCalls)) + for _, tc := range msg.ToolCalls { + toolType := openai.ToolType(tc.Type) + openaiMsg.ToolCalls = append(openaiMsg.ToolCalls, openai.ToolCall{ + ID: tc.ID, + Type: toolType, + Function: openai.FunctionCall{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }, + }) + } + } + + if msg.Role == "tool" { + openaiMsg.ToolCallID = msg.ToolCallID + openaiMsg.Name = msg.Name + } + + // Round-trip reasoning_content on assistant turns. MiMo and DeepSeek V3.2+ + // thinking mode reject multi-turn requests where the prior assistant + // message lacks its reasoning_content with HTTP 400 ("The reasoning_content + // in the thinking mode must be passed back to the API."). Providers that + // don't recognize the field ignore it harmlessly. + if msg.Role == "assistant" && msg.ReasoningContent != "" { + openaiMsg.ReasoningContent = msg.ReasoningContent + } + + openaiMessages = append(openaiMessages, openaiMsg) + } + return openaiMessages +} + +// BuildChatCompletionRequest 构建标准聊天请求参数(导出供子类使用)。 +// +// 这是一个不含任何 provider 特定逻辑的通用实现:所有采样参数(temperature / +// top_p / penalties)与 max_tokens 都按 opts 直接映射。供应商相关的特判 +// (OpenAI o-series / GPT-5 改用 max_completion_tokens、Moonshot 固定温度等) +// 由对应的 providerAdapter.ShapeRequest 在事后施加,见 provider.go。 +func (c *RemoteAPIChat) BuildChatCompletionRequest( + messages []Message, opts *ChatOptions, isStream bool, +) openai.ChatCompletionRequest { + req := openai.ChatCompletionRequest{ + Model: c.modelName, + Messages: c.ConvertMessages(messages), + Stream: isStream, + } + + if isStream { + req.StreamOptions = &openai.StreamOptions{IncludeUsage: true} + } + + if opts == nil { + return req + } + + req.Temperature = float32(opts.Temperature) + if opts.TopP > 0 { + req.TopP = float32(opts.TopP) + } + if opts.FrequencyPenalty > 0 { + req.FrequencyPenalty = float32(opts.FrequencyPenalty) + } + if opts.PresencePenalty > 0 { + req.PresencePenalty = float32(opts.PresencePenalty) + } + + if opts.MaxTokens > 0 { + req.MaxTokens = opts.MaxTokens + } + if opts.MaxCompletionTokens > 0 { + req.MaxCompletionTokens = opts.MaxCompletionTokens + } + + // 处理 Tools + if len(opts.Tools) > 0 { + req.Tools = make([]openai.Tool, 0, len(opts.Tools)) + for _, tool := range opts.Tools { + toolType := openai.ToolType(tool.Type) + openaiTool := openai.Tool{ + Type: toolType, + Function: &openai.FunctionDefinition{ + Name: tool.Function.Name, + Description: tool.Function.Description, + }, + } + if tool.Function.Parameters != nil { + openaiTool.Function.Parameters = tool.Function.Parameters + } + req.Tools = append(req.Tools, openaiTool) + } + } + + // 处理 ParallelToolCalls + if opts.ParallelToolCalls != nil { + val := *opts.ParallelToolCalls + req.ParallelToolCalls = val + } + + // 处理 ToolChoice(标准实现) + if opts.ToolChoice != "" { + switch opts.ToolChoice { + case "none", "required", "auto": + req.ToolChoice = opts.ToolChoice + default: + req.ToolChoice = openai.ToolChoice{ + Type: "function", + Function: openai.ToolFunction{ + Name: opts.ToolChoice, + }, + } + } + } + + if len(opts.Format) > 0 { + req.ResponseFormat = &openai.ChatCompletionResponseFormat{ + Type: openai.ChatCompletionResponseFormatTypeJSONObject, + } + req.Messages[len(req.Messages)-1].Content += fmt.Sprintf("\nUse this JSON schema: %s", opts.Format) + } + + return req +} diff --git a/internal/models/chat/openai_stream.go b/internal/models/chat/openai_stream.go new file mode 100644 index 00000000..86b1c4bb --- /dev/null +++ b/internal/models/chat/openai_stream.go @@ -0,0 +1,539 @@ +package chat + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/sashabaranov/go-openai" +) + +// parseCompletionResponse 解析非流式响应 +func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionResponse) (*types.ChatResponse, error) { + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response from API") + } + + choice := resp.Choices[0] + + // 处理思考模型的输出:移除 标签包裹的思考过程 + // 为设置了 Thinking=false 但模型仍返回思考内容的情况和部分不支持Thinking=false的思考模型(例如Miniax-M2.1)提供兜底策略 + content := removeThinkingContent(choice.Message.Content) + + response := &types.ChatResponse{ + Content: content, + FinishReason: string(choice.FinishReason), + Usage: types.TokenUsage{ + PromptTokens: resp.Usage.PromptTokens, + CompletionTokens: resp.Usage.CompletionTokens, + TotalTokens: resp.Usage.TotalTokens, + CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails), + }, + } + + if len(choice.Message.ToolCalls) > 0 { + response.ToolCalls = make([]types.LLMToolCall, 0, len(choice.Message.ToolCalls)) + for _, tc := range choice.Message.ToolCalls { + response.ToolCalls = append(response.ToolCalls, types.LLMToolCall{ + ID: tc.ID, + Type: string(tc.Type), + Function: types.FunctionCall{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }, + }) + } + } + + return response, nil +} + +// removeThinkingContent 移除思考模型输出中的 思考过程 +// 仅当内容以 开头时才处理 +func removeThinkingContent(content string) string { + const thinkStartTag = "" + const thinkEndTag = "" + + trimmed := strings.TrimSpace(content) + if !strings.HasPrefix(trimmed, thinkStartTag) { + return content + } + + // 查找最后一个 标签(处理嵌套情况) + if lastEndIdx := strings.LastIndex(trimmed, thinkEndTag); lastEndIdx != -1 { + if result := strings.TrimSpace(trimmed[lastEndIdx+len(thinkEndTag):]); result != "" { + return result + } + return "" + } + + return "" // 未找到 ,可能思考内容过长被截断,返回空字符串 +} + +// processStream 处理 OpenAI SDK 流式响应 +func (c *RemoteAPIChat) processStream( + ctx context.Context, + stream *openai.ChatCompletionStream, + streamChan chan types.StreamResponse, + dumper *streamPacketDumper, +) { + defer close(streamChan) + defer stream.Close() + + state := newStreamState() + + for { + response, err := stream.Recv() + if err != nil { + if err == io.EOF { + logUsage(ctx, c.modelName, state.usage) + toolCalls := state.buildOrderedToolCalls() + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + ToolCalls: toolCalls, + Usage: state.usage, + FinishReason: state.lastFinishReason, + } + } else { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeError, + Content: err.Error(), + Done: true, + } + } + return + } + + if dumper != nil { + dumper.WritePacket(response) + } + + if response.Usage != nil { + state.usage = &types.TokenUsage{ + PromptTokens: response.Usage.PromptTokens, + CompletionTokens: response.Usage.CompletionTokens, + TotalTokens: response.Usage.TotalTokens, + CachedTokens: cachedTokens(response.Usage.PromptTokensDetails), + } + } + + if len(response.Choices) > 0 { + c.processStreamDelta(ctx, &response.Choices[0], state, streamChan, response.Choices[0].Delta.ReasoningContent) + } + } +} + +// processRawHTTPStream 处理原始 HTTP 流式响应 +func (c *RemoteAPIChat) processRawHTTPStream( + ctx context.Context, + resp *http.Response, + streamChan chan types.StreamResponse, + dumper *streamPacketDumper, +) { + defer close(streamChan) + defer resp.Body.Close() + + state := newStreamState() + reader := NewSSEReader(resp.Body) + + for { + event, err := reader.ReadEvent() + if err != nil { + if err == io.EOF { + logUsage(ctx, c.modelName, state.usage) + toolCalls := state.buildOrderedToolCalls() + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + ToolCalls: toolCalls, + Usage: state.usage, + } + } else { + logger.Errorf(ctx, "Stream read error: %v", err) + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeError, + Content: err.Error(), + Done: true, + } + } + return + } + + if event == nil { + continue + } + + if event.Done { + logUsage(ctx, c.modelName, state.usage) + toolCalls := state.buildOrderedToolCalls() + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + ToolCalls: toolCalls, + Usage: state.usage, + } + return + } + + if event.Data == nil { + continue + } + + if dumper != nil { + // 保留上游 SSE data 行的原始 JSON,不经过中间结构体裁剪。 + raw := make([]byte, len(event.Data)) + copy(raw, event.Data) + dumper.WritePacketRaw(raw) + } + + // 使用局部结构体进行一次性解析,同时捕捉标准字段和 vLLM 的 reasoning 字段,避免性能损失 + var streamResp struct { + openai.ChatCompletionStreamResponse + Choices []struct { + Index int `json:"index"` + Delta struct { + openai.ChatCompletionStreamChoiceDelta + Reasoning string `json:"reasoning,omitempty"` + } `json:"delta"` + FinishReason openai.FinishReason `json:"finish_reason"` + } `json:"choices"` + } + + if err := json.Unmarshal(event.Data, &streamResp); err != nil { + logger.Errorf(ctx, "Failed to parse stream response: %v", err) + continue + } + + if streamResp.Usage != nil { + state.usage = &types.TokenUsage{ + PromptTokens: streamResp.Usage.PromptTokens, + CompletionTokens: streamResp.Usage.CompletionTokens, + TotalTokens: streamResp.Usage.TotalTokens, + CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails), + } + } + + if len(streamResp.Choices) > 0 { + choice := streamResp.Choices[0] + // 统一获取逻辑(支持标准和 vLLM 两种路径) + reasoning := choice.Delta.Reasoning + if reasoning == "" { + reasoning = choice.Delta.ReasoningContent + } + + // 构造一个标准 SDK 兼容的 choice 对象传给下游,保证现有逻辑完全不动 + sdkChoice := openai.ChatCompletionStreamChoice{ + Index: choice.Index, + Delta: choice.Delta.ChatCompletionStreamChoiceDelta, + FinishReason: choice.FinishReason, + } + c.processStreamDelta(ctx, &sdkChoice, state, streamChan, reasoning) + } + } +} + +// streamState 流式处理状态 +type streamState struct { + thinkingEmitter + toolCallMap map[int]*types.LLMToolCall + lastFunctionName map[int]string + nameNotified map[int]bool + fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction + usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled + lastFinishReason string // last observed finish_reason for EOF handler fallback + + // Diagnostic flags (fire-once) used to log earliest signals of tool_call + // presence/absence at the OpenAI-protocol level. These are independent of + // the higher-level ResponseTypeToolCall marker (which only fires once + // function name has stabilized) and let us distinguish between + // (A) no tool_calls field ever observed (true natural-stop), and + // (B) tool_calls field observed but marker not yet emitted. + firstToolCallSeen bool // true once any delta carried tool_calls + noToolCallStopLogged bool // true once we logged "stop without tool_calls" + firstContentSeen bool // true once delta.Content first appeared + firstReasoningSeen bool // true once reasoning_content first appeared + streamStartedAt time.Time +} + +func newStreamState() *streamState { + return &streamState{ + toolCallMap: make(map[int]*types.LLMToolCall), + lastFunctionName: make(map[int]string), + nameNotified: make(map[int]bool), + fieldExtractors: make(map[int]*jsonFieldExtractor), + streamStartedAt: time.Now(), + } +} + +// elapsedMs returns the milliseconds elapsed since the stream state was +// initialized. Used to attach time-since-stream-start to fire-once diagnostic +// logs so a single grep can reveal the temporal layout of a single stream +// (TTFC / TTFT / first-tool-call / natural-stop confirmation, etc). +func (s *streamState) elapsedMs() int64 { + if s.streamStartedAt.IsZero() { + return 0 + } + return time.Since(s.streamStartedAt).Milliseconds() +} + +func (s *streamState) buildOrderedToolCalls() []types.LLMToolCall { + if len(s.toolCallMap) == 0 { + return nil + } + result := make([]types.LLMToolCall, 0, len(s.toolCallMap)) + for i := 0; i < len(s.toolCallMap); i++ { + if tc, ok := s.toolCallMap[i]; ok && tc != nil { + result = append(result, *tc) + } + } + if len(result) == 0 { + return nil + } + return result +} + +// processStreamDelta 处理流式响应的单个 delta +func (c *RemoteAPIChat) processStreamDelta( + ctx context.Context, + choice *openai.ChatCompletionStreamChoice, + state *streamState, + streamChan chan types.StreamResponse, + reasoningContent string, +) { + delta := choice.Delta + isDone := string(choice.FinishReason) != "" + + // Track finish_reason for EOF handler fallback + if isDone { + state.lastFinishReason = string(choice.FinishReason) + } + + // 处理 tool calls + if len(delta.ToolCalls) > 0 { + c.processToolCallsDelta(ctx, delta.ToolCalls, state, streamChan) + } + + // Earliest reliable "no tool_calls" signal at the OpenAI-protocol level: + // finish_reason=stop arrived AND we never observed a tool_calls field on + // any prior delta. Logged once per stream so callers can grep for the + // natural-stop entry point without waiting for the higher-level summary. + if isDone && + string(choice.FinishReason) == "stop" && + !state.firstToolCallSeen && + !state.noToolCallStopLogged { + logger.Infof(ctx, "[LLM Stream] Natural-stop at OpenAI layer "+ + "(finish=stop, tool_calls field never observed, thinking_seen=%t, "+ + "first_content_seen=%t, elapsed_ms=%d)", + state.active, state.firstContentSeen, state.elapsedMs()) + state.noToolCallStopLogged = true + } + + // 发送思考内容(ReasoningContent,支持 DeepSeek 等模型) + if reasoningContent != "" { + // Earliest reasoning_content signal at the OpenAI-protocol level. Fired + // once per stream so we can distinguish "model emitted thinking before + // answer" vs "model never produced thinking" when triaging logs. + if !state.firstReasoningSeen { + state.firstReasoningSeen = true + logger.Infof(ctx, "[LLM Stream] First reasoning_content at OpenAI layer "+ + "(len=%d, preview=%q, elapsed_ms=%d)", + len(reasoningContent), truncateForDebug(reasoningContent, 80), state.elapsedMs()) + } + state.emit(streamChan, reasoningContent) + } + + // 发送回答内容 + if delta.Content != "" { + // Earliest delta.Content signal at the OpenAI-protocol level. Fired once + // per stream so we can measure TTFC (time-to-first-content) and tell + // "answer started before any tool_call" from "tool_call came first". + if !state.firstContentSeen { + state.firstContentSeen = true + logger.Infof(ctx, "[LLM Stream] First delta.Content at OpenAI layer "+ + "(len=%d, preview=%q, tool_call_seen=%t, thinking_seen=%t, elapsed_ms=%d)", + len(delta.Content), truncateForDebug(delta.Content, 80), + state.firstToolCallSeen, state.firstReasoningSeen, state.elapsedMs()) + } + // If we had thinking content and this is the first answer chunk, + // send a thinking done event first. + state.finish(streamChan) + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: delta.Content, + Done: isDone, + ToolCalls: state.buildOrderedToolCalls(), + FinishReason: string(choice.FinishReason), + } + } + + if isDone && len(state.toolCallMap) > 0 { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + ToolCalls: state.buildOrderedToolCalls(), + FinishReason: string(choice.FinishReason), + } + } + + // Ensure thinking done is sent when stream finishes without any answer content + // (e.g., model only produced reasoning then hit finish_reason with empty content). + if isDone { + state.finish(streamChan) + } + + // Catch-all: isDone but none of the above branches sent a response with + // FinishReason (empty content, no tool calls, no thinking). This prevents + // the finish_reason from being lost in the streaming pipeline. + if isDone && delta.Content == "" && len(state.toolCallMap) == 0 { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Done: true, + FinishReason: string(choice.FinishReason), + } + } +} + +// processToolCallsDelta 处理 tool calls 的增量更新 +func (c *RemoteAPIChat) processToolCallsDelta( + ctx context.Context, + toolCalls []openai.ToolCall, + state *streamState, + streamChan chan types.StreamResponse, +) { + // Earliest signal at the OpenAI-protocol level that this stream will + // produce at least one tool call. Fires *before* the function name has + // stabilized, i.e. earlier than the higher-level ResponseTypeToolCall + // marker downstream consumers see. Useful for distinguishing + // "tool_calls field arrived but marker not yet emitted" from + // "tool_calls field truly absent" when triaging stream behavior. + if !state.firstToolCallSeen && len(toolCalls) > 0 { + state.firstToolCallSeen = true + var firstID, firstName string + for _, tc := range toolCalls { + if tc.ID != "" { + firstID = tc.ID + } + if tc.Function.Name != "" { + firstName = tc.Function.Name + } + if firstID != "" || firstName != "" { + break + } + } + logger.Infof(ctx, "[LLM Stream] First tool_calls delta at OpenAI layer "+ + "(count=%d, first_id=%q, first_name=%q, "+ + "first_content_seen=%t, thinking_seen=%t, elapsed_ms=%d)", + len(toolCalls), firstID, firstName, + state.firstContentSeen, state.firstReasoningSeen, state.elapsedMs()) + } + + for _, tc := range toolCalls { + var toolCallIndex int + if tc.Index != nil { + toolCallIndex = *tc.Index + } + toolCallEntry, exists := state.toolCallMap[toolCallIndex] + if !exists || toolCallEntry == nil { + toolCallEntry = &types.LLMToolCall{ + Type: string(tc.Type), + Function: types.FunctionCall{ + Name: "", + Arguments: "", + }, + } + state.toolCallMap[toolCallIndex] = toolCallEntry + } + + if tc.ID != "" { + toolCallEntry.ID = tc.ID + } + if tc.Type != "" { + toolCallEntry.Type = string(tc.Type) + } + if tc.Function.Name != "" { + // 防御性校验:解决部分供应商(如vLLM Ascend等)在每个流 Chunk 中重复发送完整工具名的问题。 + // 如果当前已存名字与新收到名字一致,则视为冗余重复,不进行叠加。 + if toolCallEntry.Function.Name != tc.Function.Name { + toolCallEntry.Function.Name += tc.Function.Name + } + } + + argsUpdated := false + if tc.Function.Arguments != "" { + toolCallEntry.Function.Arguments += tc.Function.Arguments + argsUpdated = true + } + + currName := toolCallEntry.Function.Name + if currName != "" && + currName == state.lastFunctionName[toolCallIndex] && + argsUpdated && + !state.nameNotified[toolCallIndex] && + toolCallEntry.ID != "" { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeToolCall, + Content: "", + Done: false, + Data: map[string]interface{}{ + "tool_name": currName, + "tool_call_id": toolCallEntry.ID, + }, + } + state.nameNotified[toolCallIndex] = true + } + + state.lastFunctionName[toolCallIndex] = currName + + // Stream thinking tool's thought field as thinking-type chunks + if toolCallEntry.Function.Name == "thinking" && argsUpdated { + extractor, exists := state.fieldExtractors[toolCallIndex] + if !exists { + extractor = newJSONFieldExtractor("thought") + state.fieldExtractors[toolCallIndex] = extractor + } + thoughtChunk := extractor.Feed(tc.Function.Arguments) + if thoughtChunk != "" { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeThinking, + Content: thoughtChunk, + Done: false, + Data: map[string]interface{}{ + "source": "thinking_tool", + "tool_call_id": toolCallEntry.ID, + }, + } + } + } + } +} + +// cachedTokens returns the cached prompt-token count from an OpenAI-compatible +// usage detail block, or zero when the provider did not report one. Some +// providers omit PromptTokensDetails entirely, so the nil guard is required. +// +// Note on provider semantics: +// - Implicit-cache providers (OpenAI, Azure OpenAI, DeepSeek, …) populate +// `cached_tokens` automatically whenever the prompt prefix matches a +// previous request — no caller opt-in is required. +// - Explicit-cache providers (Qwen on Aliyun, Anthropic Claude, …) only +// populate `cached_tokens` after the caller attaches `cache_control: +// {"type": "ephemeral"}` to the relevant message / content block. This +// helper still returns zero for those providers until that opt-in is +// applied upstream of the request. +func cachedTokens(d *openai.PromptTokensDetails) int { + if d == nil { + return 0 + } + return d.CachedTokens +} diff --git a/internal/models/chat/provider.go b/internal/models/chat/provider.go new file mode 100644 index 00000000..4c9d3f81 --- /dev/null +++ b/internal/models/chat/provider.go @@ -0,0 +1,247 @@ +package chat + +import ( + "net/http" + "strings" + + "github.com/Tencent/WeKnora/internal/models/provider" + modelutils "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" + "github.com/sashabaranov/go-openai" +) + +// authCreds carries the credentials a providerAdapter needs to authenticate a +// raw HTTP request. APIKey covers the common Bearer / api-key cases; AppID and +// AppSecret are only used by signing providers (WeKnoraCloud). +type authCreds struct { + APIKey string + AppID string + AppSecret string +} + +// providerAdapter captures everything provider-specific about an +// OpenAI-compatible chat backend. Every method has a sensible default on +// baseProvider, so a new provider is added by embedding baseProvider and +// overriding only the one or two methods that actually differ. +type providerAdapter interface { + // Name is the provider this adapter handles. + Name() provider.ProviderName + // Matches reports whether this adapter applies to the given model name. + // Used for sub-provider routing (e.g. Qwen thinking models within Aliyun, + // reasoning models within OpenAI). Default: true. + Matches(model string) bool + // Thinking is how this provider encodes ChatOptions.Thinking. Default: none. + Thinking() ThinkingStrategy + // ShapeRequest applies in-place parameter quirks to the standard request + // (stripping unsupported fields, pinning temperature, …). Default: noop. + ShapeRequest(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) + // TransformMessages rewrites the converted messages (e.g. downgrading + // multi-content to plain text). Default: identity. + TransformMessages(msgs []openai.ChatCompletionMessage) []openai.ChatCompletionMessage + // Endpoint overrides the request URL. Empty string means the standard + // "/chat/completions" handled by the caller. Default: "". + Endpoint(baseURL, modelID string, isStream bool) string + // Auth sets authentication headers on a raw HTTP request. Default: Bearer. + Auth(req *http.Request, creds authCreds, body []byte) + // ForceRawHTTP forces the raw HTTP path even when the body is standard + // (needed by providers that must sign the exact request bytes). Default: false. + ForceRawHTTP() bool +} + +// baseProvider supplies the default behavior for every providerAdapter method. +// It is also the fallback returned by resolveProvider for unknown providers: +// Bearer auth, standard endpoint, no thinking, no request shaping. +type baseProvider struct{} + +func (baseProvider) Name() provider.ProviderName { return "" } +func (baseProvider) Matches(string) bool { return true } +func (baseProvider) Thinking() ThinkingStrategy { return noThinking{} } +func (baseProvider) ShapeRequest(*openai.ChatCompletionRequest, *ChatOptions, bool) {} +func (baseProvider) TransformMessages(msgs []openai.ChatCompletionMessage) []openai.ChatCompletionMessage { + return msgs +} +func (baseProvider) Endpoint(string, string, bool) string { return "" } +func (baseProvider) Auth(req *http.Request, creds authCreds, _ []byte) { + req.Header.Set("Authorization", "Bearer "+creds.APIKey) +} +func (baseProvider) ForceRawHTTP() bool { return false } + +// --- WeKnoraCloud: custom endpoint + request signing + multi-content downgrade --- + +type weKnoraCloudProvider struct{ baseProvider } + +func (weKnoraCloudProvider) Name() provider.ProviderName { return provider.ProviderWeKnoraCloud } + +func (weKnoraCloudProvider) Endpoint(baseURL, _ string, _ bool) string { + return strings.TrimRight(baseURL, "/") + "/api/v1/chat/completions" +} + +func (weKnoraCloudProvider) ForceRawHTTP() bool { return true } + +func (weKnoraCloudProvider) Auth(req *http.Request, creds authCreds, body []byte) { + requestID := uuid.NewString() + headers := modelutils.Sign(creds.AppID, creds.AppSecret, requestID, string(body)) + for k, v := range headers { + req.Header.Set(k, v) + } +} + +// TransformMessages downgrades MultiContent to plain text while preserving +// tool_calls / tool_call_id / name so the function-calling protocol keeps working. +func (weKnoraCloudProvider) TransformMessages(messages []openai.ChatCompletionMessage) []openai.ChatCompletionMessage { + result := make([]openai.ChatCompletionMessage, 0, len(messages)) + for _, m := range messages { + msg := m + if msg.Content == "" && len(msg.MultiContent) > 0 { + var textParts []string + for _, part := range msg.MultiContent { + if part.Type == openai.ChatMessagePartTypeText && part.Text != "" { + textParts = append(textParts, part.Text) + } + } + msg.Content = strings.Join(textParts, "\n") + msg.MultiContent = nil + } + result = append(result, msg) + } + return result +} + +// --- Aliyun Qwen thinking models: enable_thinking (always sent, forced off non-stream) --- + +type qwenThinkingProvider struct{ baseProvider } + +func (qwenThinkingProvider) Name() provider.ProviderName { return provider.ProviderAliyun } +func (qwenThinkingProvider) Matches(model string) bool { return provider.IsQwenThinkingModel(model) } +func (qwenThinkingProvider) Thinking() ThinkingStrategy { + return enableThinking{alwaysSend: true, disableOnNonStream: true} +} + +// --- LKEAP: thinking via { "thinking": { "type": ... } }, only for DeepSeek V3.x --- +// R1 series enables chain-of-thought by default and is left untouched (falls +// back to baseProvider). See https://cloud.tencent.com/document/product/1772/115963 + +type lkeapProvider struct{ baseProvider } + +func (lkeapProvider) Name() provider.ProviderName { return provider.ProviderLKEAP } +func (lkeapProvider) Matches(model string) bool { + return strings.Contains(strings.ToLower(model), "deepseek-v3") +} +func (lkeapProvider) Thinking() ThinkingStrategy { return thinkingTypeField{} } + +// --- DeepSeek: does not support tool_choice --- + +type deepseekProvider struct{ baseProvider } + +func (deepseekProvider) Name() provider.ProviderName { return provider.ProviderDeepSeek } +func (deepseekProvider) ShapeRequest(req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool) { + if opts != nil && opts.ToolChoice != "" { + req.ToolChoice = nil + } +} + +// --- Generic (vLLM) / NVIDIA: thinking via chat_template_kwargs --- + +type genericProvider struct{ baseProvider } + +func (genericProvider) Name() provider.ProviderName { return provider.ProviderGeneric } +func (genericProvider) Thinking() ThinkingStrategy { return chatTemplateKwargs{} } + +type nvidiaProvider struct{ baseProvider } + +func (nvidiaProvider) Name() provider.ProviderName { return provider.ProviderNvidia } +func (nvidiaProvider) Thinking() ThinkingStrategy { return chatTemplateKwargs{} } + +// --- Volcengine (火山引擎 Ark): thinking via { "thinking": { "type": ... } } --- + +type volcengineProvider struct{ baseProvider } + +func (volcengineProvider) Name() provider.ProviderName { return provider.ProviderVolcengine } +func (volcengineProvider) Thinking() ThinkingStrategy { return thinkingTypeField{} } + +// --- Azure OpenAI: api-key auth (reasoning variant also strips sampling params) --- + +type azureProvider struct{ baseProvider } + +func (azureProvider) Name() provider.ProviderName { return provider.ProviderAzureOpenAI } +func (azureProvider) Auth(req *http.Request, creds authCreds, _ []byte) { + req.Header.Set("api-key", creds.APIKey) +} + +type azureReasoningProvider struct{ azureProvider } + +func (azureReasoningProvider) Matches(model string) bool { + return provider.IsOpenAIReasoningOrGPT5Model(model) +} +func (azureReasoningProvider) ShapeRequest(req *openai.ChatCompletionRequest, _ *ChatOptions, _ bool) { + shapeOpenAIReasoning(req) +} + +// --- OpenAI reasoning / GPT-5: no sampling params, must use max_completion_tokens --- + +type openAIReasoningProvider struct{ baseProvider } + +func (openAIReasoningProvider) Name() provider.ProviderName { return provider.ProviderOpenAI } +func (openAIReasoningProvider) Matches(model string) bool { + return provider.IsOpenAIReasoningOrGPT5Model(model) +} +func (openAIReasoningProvider) ShapeRequest(req *openai.ChatCompletionRequest, _ *ChatOptions, _ bool) { + shapeOpenAIReasoning(req) +} + +// --- Moonshot: v1 models accept only temperature=1 --- + +type moonshotProvider struct{ baseProvider } + +func (moonshotProvider) Name() provider.ProviderName { return provider.ProviderMoonshot } +func (moonshotProvider) Matches(model string) bool { + return provider.IsMoonshotFixedTempModel(model) +} +func (moonshotProvider) ShapeRequest(req *openai.ChatCompletionRequest, _ *ChatOptions, _ bool) { + // Pin temperature to 1 and drop the other sampling params, matching the + // pre-refactor behavior where these fields were never set for this model. + req.Temperature = 1 + req.TopP = 0 + req.FrequencyPenalty = 0 + req.PresencePenalty = 0 +} + +// shapeOpenAIReasoning strips sampling params (unsupported by o-series / GPT-5) +// and migrates max_tokens to max_completion_tokens. See issue #1283. +func shapeOpenAIReasoning(req *openai.ChatCompletionRequest) { + req.Temperature = 0 + req.TopP = 0 + req.FrequencyPenalty = 0 + req.PresencePenalty = 0 + if req.MaxCompletionTokens == 0 && req.MaxTokens > 0 { + req.MaxCompletionTokens = req.MaxTokens + } + req.MaxTokens = 0 +} + +// providerRegistry is ordered: more specific adapters (those with a real +// Matches predicate) must precede the generic catch-all for the same provider. +var providerRegistry = []providerAdapter{ + weKnoraCloudProvider{}, + qwenThinkingProvider{}, + lkeapProvider{}, + deepseekProvider{}, + genericProvider{}, + volcengineProvider{}, + nvidiaProvider{}, + azureReasoningProvider{}, + azureProvider{}, + openAIReasoningProvider{}, + moonshotProvider{}, +} + +// resolveProvider returns the adapter handling the given provider+model, or +// baseProvider{} (Bearer auth, standard endpoint, no thinking) when none matches. +func resolveProvider(name provider.ProviderName, model string) providerAdapter { + for _, p := range providerRegistry { + if p.Name() == name && p.Matches(model) { + return p + } + } + return baseProvider{} +} diff --git a/internal/models/chat/provider_test.go b/internal/models/chat/provider_test.go new file mode 100644 index 00000000..ea10dfff --- /dev/null +++ b/internal/models/chat/provider_test.go @@ -0,0 +1,159 @@ +package chat + +import ( + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/models/provider" + "github.com/Tencent/WeKnora/internal/types" + "github.com/sashabaranov/go-openai" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResolveProvider pins the provider+model routing table, including the +// sub-model matchers (reasoning models, Qwen thinking, LKEAP DeepSeek V3) and +// the baseProvider fallback for everything else. +func TestResolveProvider(t *testing.T) { + cases := []struct { + name string + prov provider.ProviderName + model string + want providerAdapter + }{ + {"deepseek", provider.ProviderDeepSeek, "deepseek-chat", deepseekProvider{}}, + {"lkeap v3", provider.ProviderLKEAP, "deepseek-v3.1", lkeapProvider{}}, + {"lkeap r1 falls back", provider.ProviderLKEAP, "deepseek-r1", baseProvider{}}, + {"qwen thinking", provider.ProviderAliyun, "qwen3-32b", qwenThinkingProvider{}}, + {"generic", provider.ProviderGeneric, "anything", genericProvider{}}, + {"nvidia", provider.ProviderNvidia, "anything", nvidiaProvider{}}, + {"volcengine", provider.ProviderVolcengine, "doubao", volcengineProvider{}}, + {"openai non-reasoning falls back", provider.ProviderOpenAI, "gpt-4o", baseProvider{}}, + {"openai reasoning", provider.ProviderOpenAI, "gpt-5", openAIReasoningProvider{}}, + {"azure non-reasoning", provider.ProviderAzureOpenAI, "gpt-4", azureProvider{}}, + {"azure reasoning", provider.ProviderAzureOpenAI, "gpt-5-mini", azureReasoningProvider{}}, + {"moonshot fixed temp", provider.ProviderMoonshot, "moonshot-v1-8k", moonshotProvider{}}, + {"moonshot other falls back", provider.ProviderMoonshot, "kimi-latest", baseProvider{}}, + {"weknora cloud", provider.ProviderWeKnoraCloud, "anything", weKnoraCloudProvider{}}, + {"unknown falls back", provider.ProviderName("nope"), "x", baseProvider{}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.IsType(t, tc.want, resolveProvider(tc.prov, tc.model)) + }) + } +} + +func newOutboundChat(t *testing.T, providerName, model string, extra map[string]string) *RemoteAPIChat { + t.Helper() + c, err := NewRemoteAPIChat(&ChatConfig{ + Source: types.ModelSourceRemote, + ModelName: model, + APIKey: "k", + ModelID: model, + Provider: providerName, + ExtraConfig: extra, + }) + require.NoError(t, err) + return c +} + +// TestBuildOutbound_Thinking is the characterization suite for the merged +// thinking-control path: it asserts that buildOutbound produces the same wire +// formats the pre-refactor provider customizers did. +func TestBuildOutbound_Thinking(t *testing.T) { + msgs := []Message{{Role: "user", Content: "hi"}} + + t.Run("generic explicit thinking_type overrides legacy kwargs", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderGeneric), "deepseek-v4-flash", + map[string]string{ExtraConfigThinkingControl: "thinking_type"}) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true) + require.True(t, useRaw) + js := mustJSON(t, body) + assert.Contains(t, js, `"thinking"`) + assert.Contains(t, js, `"disabled"`) + assert.NotContains(t, js, "chat_template_kwargs") + }) + + t.Run("generic legacy chat_template_kwargs", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderGeneric), "qwen", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true) + require.True(t, useRaw) + assert.Contains(t, mustJSON(t, body), "chat_template_kwargs") + }) + + t.Run("none keeps the standard SDK request", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderGeneric), "x", + map[string]string{ExtraConfigThinkingControl: "none"}) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true) + assert.False(t, useRaw) + _, ok := body.(*openai.ChatCompletionRequest) + assert.True(t, ok) + }) + + t.Run("qwen non-stream forces disabled", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderAliyun), "qwen3-32b", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, false) + require.True(t, useRaw) + assert.Contains(t, mustJSON(t, body), `"enable_thinking":false`) + }) + + t.Run("qwen stream honors requested true", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderAliyun), "qwen3-32b", nil) + body, _, _ := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, true) + assert.Contains(t, mustJSON(t, body), `"enable_thinking":true`) + }) + + t.Run("volcengine thinking enabled", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderVolcengine), "doubao", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, true) + require.True(t, useRaw) + js := mustJSON(t, body) + assert.Contains(t, js, `"thinking"`) + assert.Contains(t, js, `"enabled"`) + }) + + t.Run("lkeap deepseek-v3 emits thinking type", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderLKEAP), "deepseek-v3.1", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true) + require.True(t, useRaw) + assert.Contains(t, mustJSON(t, body), `"thinking"`) + }) + + t.Run("lkeap r1 left untouched", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderLKEAP), "deepseek-r1", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true) + assert.False(t, useRaw) + _, ok := body.(*openai.ChatCompletionRequest) + assert.True(t, ok) + }) +} + +// TestBuildOutbound_ShapeRequest covers the param-shaping providers that used +// to live inline in BuildChatCompletionRequest. +func TestBuildOutbound_ShapeRequest(t *testing.T) { + msgs := []Message{{Role: "user", Content: "hi"}} + + t.Run("deepseek strips tool_choice", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderDeepSeek), "deepseek-chat", nil) + body, _, useRaw := c.buildOutbound(msgs, &ChatOptions{ToolChoice: "auto"}, false) + assert.False(t, useRaw) + req := body.(*openai.ChatCompletionRequest) + assert.Nil(t, req.ToolChoice) + }) + + t.Run("moonshot pins temperature to 1", func(t *testing.T) { + c := newOutboundChat(t, string(provider.ProviderMoonshot), "moonshot-v1-8k", nil) + body, _, _ := c.buildOutbound(msgs, &ChatOptions{Temperature: 0.7, TopP: 0.9}, false) + req := body.(*openai.ChatCompletionRequest) + assert.EqualValues(t, 1, req.Temperature) + assert.EqualValues(t, 0, req.TopP) + }) +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return string(b) +} diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 2a6acce5..7b570d32 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -7,10 +7,7 @@ import ( "fmt" "io" "net/http" - "os" - "strconv" "strings" - "time" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/provider" @@ -19,55 +16,10 @@ import ( "github.com/sashabaranov/go-openai" ) -// LLM 调用超时配置。仅作为"上层未设置 deadline 时"的兜底,避免 hung 请求 -// 永久阻塞 worker。如果上层 ctx 已经设置了 deadline(无论比默认更短还是更长), -// 都会原样尊重,不再叠加默认超时。可通过环境变量覆盖: -// - WEKNORA_LLM_CHAT_TIMEOUT_SECONDS 非流式调用兜底超时(默认 600s) -// - WEKNORA_LLM_STREAM_TIMEOUT_SECONDS 流式调用兜底超时(默认 1800s) -var ( - defaultChatTimeout = envDurationSeconds("WEKNORA_LLM_CHAT_TIMEOUT_SECONDS", 300*time.Second) - defaultStreamTimeout = envDurationSeconds("WEKNORA_LLM_STREAM_TIMEOUT_SECONDS", 600*time.Second) -) - -// envDurationSeconds 读取以"秒"为单位的环境变量,解析失败或非正值时回退到 fallback。 -func envDurationSeconds(key string, fallback time.Duration) time.Duration { - v := strings.TrimSpace(os.Getenv(key)) - if v == "" { - return fallback - } - n, err := strconv.Atoi(v) - if err != nil || n <= 0 { - return fallback - } - return time.Duration(n) * time.Second -} - -// withLLMTimeout 仅在上层 ctx 没有 deadline 时附加一个兜底超时; -// 如果上层已显式设置 deadline(无论更短或更长),则原样返回, -// 让调用方对自己的超时策略拥有最终决定权。 -func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { - if _, ok := ctx.Deadline(); ok { - return ctx, func() {} - } - return context.WithTimeout(ctx, d) -} - -// rawHTTPClient is a shared HTTP client for raw HTTP LLM calls with connection-level timeouts. -// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout) -// rather than http.Client.Timeout, so streaming calls are not prematurely terminated. -// Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer. -var rawHTTPClient = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: secutils.SSRFSafeDialContext, - TLSHandshakeTimeout: 10 * time.Second, - IdleConnTimeout: 90 * time.Second, - MaxIdleConnsPerHost: 5, - }, -} - -// RemoteAPIChat 实现了基于 OpenAI 兼容 API 的聊天 -// 这是一个通用实现,不包含任何 provider 特定的逻辑 +// RemoteAPIChat 实现了基于 OpenAI 兼容 API 的聊天。 +// 它本身只负责通用的请求/响应/流式处理;所有 provider 特定行为都委托给 +// providerAdapter(见 provider.go),thinking 编码委托给 ThinkingStrategy +// (见 thinking.go)。 type RemoteAPIChat struct { modelName string client *openai.Client @@ -80,16 +32,10 @@ type RemoteAPIChat struct { // customHeaders 为用户在模型配置中指定的自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 customHeaders map[string]string - // requestCustomizer 允许子类自定义请求 - // 返回自定义请求体(如果为 nil 则使用标准请求)和是否需要使用原始 HTTP 请求 - requestCustomizer func(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (customReq any, useRawHTTP bool) - - // endpointCustomizer 允许子类自定义请求的 endpoint - // 返回是否使用自定义请求地址, 返回空则使用默认OpenAI格式地址 - endpointCustomizer func(baseURL string, modelID string, isStream bool) (endpoint string) - - // headerCustomizer 允许子类自定义原始 HTTP 请求头(例如签名认证) - headerCustomizer func(req *http.Request, body []byte) error + // adapter 承载所有 provider 特定行为(thinking / 参数特判 / endpoint / 鉴权 / 消息变换)。 + adapter providerAdapter + // thinkingOverride 来自 extra_config.thinking_control,非 nil 时覆盖 adapter.Thinking()。 + thinkingOverride ThinkingStrategy } // NewRemoteAPIChat 创建远程 API 聊天实例 @@ -151,233 +97,63 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { } return &RemoteAPIChat{ - modelName: modelName, - client: openai.NewClientWithConfig(config), - modelID: chatConfig.ModelID, - baseURL: chatConfig.BaseURL, - apiKey: apiKey, - provider: providerName, - appID: chatConfig.AppID, - appSecret: chatConfig.AppSecret, - customHeaders: chatConfig.CustomHeaders, + modelName: modelName, + client: openai.NewClientWithConfig(config), + modelID: chatConfig.ModelID, + baseURL: chatConfig.BaseURL, + apiKey: apiKey, + provider: providerName, + appID: chatConfig.AppID, + appSecret: chatConfig.AppSecret, + customHeaders: chatConfig.CustomHeaders, + adapter: resolveProvider(providerName, modelName), + thinkingOverride: parseThinkingOverride(chatConfig.ExtraConfig), }, nil } -// SetRequestCustomizer 设置请求自定义器 -func (c *RemoteAPIChat) SetRequestCustomizer(customizer func(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool)) { - c.requestCustomizer = customizer +// authCreds bundles the credentials passed to the adapter's Auth method. +func (c *RemoteAPIChat) authCreds() authCreds { + return authCreds{APIKey: c.apiKey, AppID: c.appID, AppSecret: c.appSecret} } -// SetEndpointCustomizer 设置请求地址自定义器 -func (c *RemoteAPIChat) SetEndpointCustomizer(customizer func(baseURL string, modelID string, isStream bool) string) { - c.endpointCustomizer = customizer -} - -// SetHeaderCustomizer 设置原始 HTTP 请求头自定义器 -func (c *RemoteAPIChat) SetHeaderCustomizer(customizer func(req *http.Request, body []byte) error) { - c.headerCustomizer = customizer -} - -// ConvertMessages 转换消息格式为 OpenAI 格式(导出供子类使用) -func (c *RemoteAPIChat) ConvertMessages(messages []Message) []openai.ChatCompletionMessage { - openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages)) - for _, msg := range messages { - openaiMsg := openai.ChatCompletionMessage{ - Role: msg.Role, - } - - // 优先处理多内容消息(包含图片等) - if len(msg.MultiContent) > 0 { - openaiMsg.MultiContent = make([]openai.ChatMessagePart, 0, len(msg.MultiContent)) - for _, part := range msg.MultiContent { - switch part.Type { - case "text": - openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, - Text: part.Text, - }) - case "image_url": - if part.ImageURL != nil { - openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: part.ImageURL.URL, - Detail: openai.ImageURLDetail(part.ImageURL.Detail), - }, - }) - } - } - } - } else if len(msg.Images) > 0 && msg.Role == "user" { - parts := make([]openai.ChatMessagePart, 0, len(msg.Images)+1) - for _, imgURL := range msg.Images { - resolved := resolveImageURLForLLM(imgURL) - parts = append(parts, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: resolved, - Detail: openai.ImageURLDetailAuto, - }, - }) - } - parts = append(parts, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, - Text: msg.Content, - }) - openaiMsg.MultiContent = parts - } else if msg.Content != "" { - openaiMsg.Content = msg.Content - } - - if len(msg.ToolCalls) > 0 { - openaiMsg.ToolCalls = make([]openai.ToolCall, 0, len(msg.ToolCalls)) - for _, tc := range msg.ToolCalls { - toolType := openai.ToolType(tc.Type) - openaiMsg.ToolCalls = append(openaiMsg.ToolCalls, openai.ToolCall{ - ID: tc.ID, - Type: toolType, - Function: openai.FunctionCall{ - Name: tc.Function.Name, - Arguments: tc.Function.Arguments, - }, - }) - } - } - - if msg.Role == "tool" { - openaiMsg.ToolCallID = msg.ToolCallID - openaiMsg.Name = msg.Name - } - - // Round-trip reasoning_content on assistant turns. MiMo and DeepSeek V3.2+ - // thinking mode reject multi-turn requests where the prior assistant - // message lacks its reasoning_content with HTTP 400 ("The reasoning_content - // in the thinking mode must be passed back to the API."). Providers that - // don't recognize the field ignore it harmlessly. - if msg.Role == "assistant" && msg.ReasoningContent != "" { - openaiMsg.ReasoningContent = msg.ReasoningContent - } - - openaiMessages = append(openaiMessages, openaiMsg) - } - return openaiMessages -} - -// BuildChatCompletionRequest 构建标准聊天请求参数(导出供子类使用) -func (c *RemoteAPIChat) BuildChatCompletionRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest { - req := openai.ChatCompletionRequest{ - Model: c.modelName, - Messages: c.ConvertMessages(messages), - Stream: isStream, - } - - if isStream { - req.StreamOptions = &openai.StreamOptions{IncludeUsage: true} - } - - if opts != nil { - // OpenAI / Azure OpenAI 的推理类模型(o-series)以及 GPT-5 系列 - // 不再支持 max_tokens,必须使用 max_completion_tokens;且不支持 - // temperature / top_p / frequency_penalty / presence_penalty 的非默认值。 - // 这里做兼容处理,避免上层无差别地传 MaxTokens 导致 400 错误: - // "this model is not supported MaxTokens, please use MaxCompletionTokens" - // 参考 issue #1283。 - isOpenAIReasoning := (c.provider == provider.ProviderOpenAI || c.provider == provider.ProviderAzureOpenAI) && - provider.IsOpenAIReasoningOrGPT5Model(c.modelName) - - // Moonshot v1 系列模型只接受 temperature=1,传入其他值会返回 400 错误。 - isMoonshotFixedTemp := c.provider == provider.ProviderMoonshot && - provider.IsMoonshotFixedTempModel(c.modelName) - - if isOpenAIReasoning { - // 不设置 temperature / top_p 等参数 - } else if isMoonshotFixedTemp { - req.Temperature = 1 - } else { - req.Temperature = float32(opts.Temperature) - if opts.TopP > 0 { - req.TopP = float32(opts.TopP) - } - if opts.FrequencyPenalty > 0 { - req.FrequencyPenalty = float32(opts.FrequencyPenalty) - } - if opts.PresencePenalty > 0 { - req.PresencePenalty = float32(opts.PresencePenalty) - } - } - - switch { - case isOpenAIReasoning: - // 强制使用 max_completion_tokens;MaxTokens 字段保持零值,依赖 omitempty 不发送。 - if opts.MaxCompletionTokens > 0 { - req.MaxCompletionTokens = opts.MaxCompletionTokens - } else if opts.MaxTokens > 0 { - req.MaxCompletionTokens = opts.MaxTokens - } - default: - if opts.MaxTokens > 0 { - req.MaxTokens = opts.MaxTokens - } - if opts.MaxCompletionTokens > 0 { - req.MaxCompletionTokens = opts.MaxCompletionTokens - } - } - - // 处理 Tools - if len(opts.Tools) > 0 { - req.Tools = make([]openai.Tool, 0, len(opts.Tools)) - for _, tool := range opts.Tools { - toolType := openai.ToolType(tool.Type) - openaiTool := openai.Tool{ - Type: toolType, - Function: &openai.FunctionDefinition{ - Name: tool.Function.Name, - Description: tool.Function.Description, - }, - } - if tool.Function.Parameters != nil { - openaiTool.Function.Parameters = tool.Function.Parameters - } - req.Tools = append(req.Tools, openaiTool) - } - } - - // 处理 ParallelToolCalls - if opts.ParallelToolCalls != nil { - val := *opts.ParallelToolCalls - req.ParallelToolCalls = val - } - - // 处理 ToolChoice(标准实现) - if opts.ToolChoice != "" { - switch opts.ToolChoice { - case "none", "required", "auto": - req.ToolChoice = opts.ToolChoice - default: - req.ToolChoice = openai.ToolChoice{ - Type: "function", - Function: openai.ToolFunction{ - Name: opts.ToolChoice, - }, - } - } - } - - if len(opts.Format) > 0 { - req.ResponseFormat = &openai.ChatCompletionResponseFormat{ - Type: openai.ChatCompletionResponseFormatTypeJSONObject, - } - req.Messages[len(req.Messages)-1].Content += fmt.Sprintf("\nUse this JSON schema: %s", opts.Format) - } - } - +// shapedRequest builds the standard request and applies the adapter's message +// transform and parameter shaping (but not thinking, which may wrap the body). +func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest { + req := c.BuildChatCompletionRequest(messages, opts, isStream) + req.Messages = c.adapter.TransformMessages(req.Messages) + c.adapter.ShapeRequest(&req, opts, isStream) return req } +// buildOutbound assembles the final outbound request: the body to send, the +// endpoint override (empty for the standard endpoint), and whether the raw HTTP +// path is required. This is the single place that composes adapter + thinking, +// replacing the former buildRequestCustomizer plumbing. +func (c *RemoteAPIChat) buildOutbound( + messages []Message, opts *ChatOptions, isStream bool, +) (body any, endpoint string, useRawHTTP bool) { + req := c.shapedRequest(messages, opts, isStream) + + thinking := c.thinkingOverride + if thinking == nil { + thinking = c.adapter.Thinking() + } + customBody, useRaw := thinking.Apply(&req, opts, isStream) + + body = &req + if customBody != nil { + body = customBody + } + endpoint = c.adapter.Endpoint(c.baseURL, c.modelID, isStream) + useRawHTTP = useRaw || c.adapter.ForceRawHTTP() || endpoint != "" + return body, endpoint, useRawHTTP +} + // logRequest 记录请求日志 func (c *RemoteAPIChat) logRequest(ctx context.Context, req any, isStream bool) { if jsonData, err := json.MarshalIndent(req, "", " "); err == nil { - logger.Infof(ctx, "[LLM Request] model=%s, stream=%v, request:\n%s", c.modelName, isStream, secutils.CompactImageDataURLForLog(string(jsonData))) + logger.Infof(ctx, "[LLM Request] model=%s, stream=%v, request:\n%s", + c.modelName, isStream, secutils.CompactImageDataURLForLog(string(jsonData))) } } @@ -388,31 +164,19 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat timeoutCtx, cancel := withLLMTimeout(ctx, defaultChatTimeout) defer cancel() - req := c.BuildChatCompletionRequest(messages, opts, false) - var customEndpoint string - if c.endpointCustomizer != nil { - customEndpoint = c.endpointCustomizer(c.baseURL, c.modelID, true) - } - // 检查是否需要自定义请求 - if c.requestCustomizer != nil { - customReq, useRawHTTP := c.requestCustomizer(&req, opts, false) - if useRawHTTP && customReq != nil { - return c.chatWithRawHTTP(timeoutCtx, customEndpoint, customReq) - } - } - - // 使用自定义请求地址 - if customEndpoint != "" { - return c.chatWithRawHTTP(timeoutCtx, customEndpoint, &req) + body, endpoint, useRawHTTP := c.buildOutbound(messages, opts, false) + if useRawHTTP { + return c.chatWithRawHTTP(timeoutCtx, endpoint, body) } + req := *(body.(*openai.ChatCompletionRequest)) c.logRequest(timeoutCtx, req, false) resp, err := c.client.CreateChatCompletion(timeoutCtx, req) if err != nil { if isMultimodalNotSupportedError(err) { logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName) cleaned := stripImagesFromMessages(messages) - req = c.BuildChatCompletionRequest(cleaned, opts, false) + req = c.shapedRequest(cleaned, opts, false) resp, err = c.client.CreateChatCompletion(timeoutCtx, req) } if err != nil { @@ -424,8 +188,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat if err != nil { return nil, err } - logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", - c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens) + logUsage(timeoutCtx, c.modelName, &result.Usage) return result, nil } @@ -451,16 +214,7 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu } httpReq.Header.Set("Content-Type", "application/json") - - if c.headerCustomizer != nil { - if err := c.headerCustomizer(httpReq, jsonData); err != nil { - return nil, fmt.Errorf("customize headers: %w", err) - } - } else if c.provider == provider.ProviderAzureOpenAI { - httpReq.Header.Set("api-key", c.apiKey) - } else { - httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) - } + c.adapter.Auth(httpReq, c.authCreds(), jsonData) // 注入用户自定义 header(保留头会在工具内部自动跳过) secutils.ApplyCustomHeaders(httpReq, c.customHeaders) @@ -488,102 +242,26 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu if err != nil { return nil, err } - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", - c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens) + logUsage(ctx, c.modelName, &result.Usage) return result, nil } -// parseCompletionResponse 解析非流式响应 -func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionResponse) (*types.ChatResponse, error) { - if len(resp.Choices) == 0 { - return nil, fmt.Errorf("no response from API") - } - - choice := resp.Choices[0] - - // 处理思考模型的输出:移除 标签包裹的思考过程 - // 为设置了 Thinking=false 但模型仍返回思考内容的情况和部分不支持Thinking=false的思考模型(例如Miniax-M2.1)提供兜底策略 - content := removeThinkingContent(choice.Message.Content) - - response := &types.ChatResponse{ - Content: content, - FinishReason: string(choice.FinishReason), - Usage: types.TokenUsage{ - PromptTokens: resp.Usage.PromptTokens, - CompletionTokens: resp.Usage.CompletionTokens, - TotalTokens: resp.Usage.TotalTokens, - CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails), - }, - } - - if len(choice.Message.ToolCalls) > 0 { - response.ToolCalls = make([]types.LLMToolCall, 0, len(choice.Message.ToolCalls)) - for _, tc := range choice.Message.ToolCalls { - response.ToolCalls = append(response.ToolCalls, types.LLMToolCall{ - ID: tc.ID, - Type: string(tc.Type), - Function: types.FunctionCall{ - Name: tc.Function.Name, - Arguments: tc.Function.Arguments, - }, - }) - } - } - - return response, nil -} - -// removeThinkingContent 移除思考模型输出中的 思考过程 -// 仅当内容以 开头时才处理 -func removeThinkingContent(content string) string { - const thinkStartTag = "" - const thinkEndTag = "" - - trimmed := strings.TrimSpace(content) - if !strings.HasPrefix(trimmed, thinkStartTag) { - return content - } - - // 查找最后一个 标签(处理嵌套情况) - if lastEndIdx := strings.LastIndex(trimmed, thinkEndTag); lastEndIdx != -1 { - if result := strings.TrimSpace(trimmed[lastEndIdx+len(thinkEndTag):]); result != "" { - return result - } - return "" - } - - return "" // 未找到 ,可能思考内容过长被截断,返回空字符串 -} - // ChatStream 进行流式聊天 func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) { // 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长, // 因为带思考/推理的模型可能数十秒甚至几分钟才产出首 token。 timeoutCtx, cancel := withLLMTimeout(ctx, defaultStreamTimeout) - req := c.BuildChatCompletionRequest(messages, opts, true) - - var customEndpoint string - if c.endpointCustomizer != nil { - customEndpoint = c.endpointCustomizer(c.baseURL, c.modelID, true) - } - - // 检查是否需要自定义请求 - if c.requestCustomizer != nil { - customReq, useRawHTTP := c.requestCustomizer(&req, opts, true) - if useRawHTTP && customReq != nil { - ch, err := c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, customReq) - return wrapStreamCancel(ch, err, cancel) - } - } - // 使用自定义请求地址 - if customEndpoint != "" { - ch, err := c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, &req) + body, endpoint, useRawHTTP := c.buildOutbound(messages, opts, true) + if useRawHTTP { + ch, err := c.chatStreamWithRawHTTP(timeoutCtx, endpoint, body) return wrapStreamCancel(ch, err, cancel) } + + req := *(body.(*openai.ChatCompletionRequest)) c.logRequest(timeoutCtx, req, true) - streamDumper := newStreamPacketDumper(c.modelName, req) + streamDumper := newStreamPacketDumper(c.modelName, &req) if streamDumper != nil { logger.Infof(timeoutCtx, "[LLM Stream Raw Dump] writing packets to %s", streamDumper.Path()) } @@ -595,7 +273,7 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts if isMultimodalNotSupportedError(err) { logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName) cleaned := stripImagesFromMessages(messages) - req = c.BuildChatCompletionRequest(cleaned, opts, true) + req = c.shapedRequest(cleaned, opts, true) stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req) } if err != nil { @@ -660,16 +338,7 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri } httpReq.Header.Set("Content-Type", "application/json") - - if c.headerCustomizer != nil { - if err := c.headerCustomizer(httpReq, jsonData); err != nil { - return nil, fmt.Errorf("customize headers: %w", err) - } - } else if c.provider == provider.ProviderAzureOpenAI { - httpReq.Header.Set("api-key", c.apiKey) - } else { - httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) - } + c.adapter.Auth(httpReq, c.authCreds(), jsonData) httpReq.Header.Set("Accept", "text/event-stream") // 注入用户自定义 header(保留头会在工具内部自动跳过) @@ -702,454 +371,6 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri return streamChan, nil } -// processStream 处理 OpenAI SDK 流式响应 -func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCompletionStream, streamChan chan types.StreamResponse, dumper *streamPacketDumper) { - defer close(streamChan) - defer stream.Close() - - state := newStreamState() - - for { - response, err := stream.Recv() - if err != nil { - if err == io.EOF { - if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) - } - toolCalls := state.buildOrderedToolCalls() - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Content: "", - Done: true, - ToolCalls: toolCalls, - Usage: state.usage, - FinishReason: state.lastFinishReason, - } - } else { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeError, - Content: err.Error(), - Done: true, - } - } - return - } - - if dumper != nil { - dumper.WritePacket(response) - } - - if response.Usage != nil { - state.usage = &types.TokenUsage{ - PromptTokens: response.Usage.PromptTokens, - CompletionTokens: response.Usage.CompletionTokens, - TotalTokens: response.Usage.TotalTokens, - CachedTokens: cachedTokens(response.Usage.PromptTokensDetails), - } - } - - if len(response.Choices) > 0 { - c.processStreamDelta(ctx, &response.Choices[0], state, streamChan, response.Choices[0].Delta.ReasoningContent) - } - } -} - -// processRawHTTPStream 处理原始 HTTP 流式响应 -func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Response, streamChan chan types.StreamResponse, dumper *streamPacketDumper) { - defer close(streamChan) - defer resp.Body.Close() - - state := newStreamState() - reader := NewSSEReader(resp.Body) - - for { - event, err := reader.ReadEvent() - if err != nil { - if err == io.EOF { - if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) - } - toolCalls := state.buildOrderedToolCalls() - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Content: "", - Done: true, - ToolCalls: toolCalls, - Usage: state.usage, - } - } else { - logger.Errorf(ctx, "Stream read error: %v", err) - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeError, - Content: err.Error(), - Done: true, - } - } - return - } - - if event == nil { - continue - } - - if event.Done { - if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) - } - toolCalls := state.buildOrderedToolCalls() - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Content: "", - Done: true, - ToolCalls: toolCalls, - Usage: state.usage, - } - return - } - - if event.Data == nil { - continue - } - - if dumper != nil { - // 保留上游 SSE data 行的原始 JSON,不经过中间结构体裁剪。 - raw := make([]byte, len(event.Data)) - copy(raw, event.Data) - dumper.WritePacketRaw(raw) - } - - // 使用局部结构体进行一次性解析,同时捕捉标准字段和 vLLM 的 reasoning 字段,避免性能损失 - var streamResp struct { - openai.ChatCompletionStreamResponse - Choices []struct { - Index int `json:"index"` - Delta struct { - openai.ChatCompletionStreamChoiceDelta - Reasoning string `json:"reasoning,omitempty"` - } `json:"delta"` - FinishReason openai.FinishReason `json:"finish_reason"` - } `json:"choices"` - } - - if err := json.Unmarshal(event.Data, &streamResp); err != nil { - logger.Errorf(ctx, "Failed to parse stream response: %v", err) - continue - } - - if streamResp.Usage != nil { - state.usage = &types.TokenUsage{ - PromptTokens: streamResp.Usage.PromptTokens, - CompletionTokens: streamResp.Usage.CompletionTokens, - TotalTokens: streamResp.Usage.TotalTokens, - CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails), - } - } - - if len(streamResp.Choices) > 0 { - choice := streamResp.Choices[0] - // 统一获取逻辑(支持标准和 vLLM 两种路径) - reasoning := choice.Delta.Reasoning - if reasoning == "" { - reasoning = choice.Delta.ReasoningContent - } - - // 构造一个标准 SDK 兼容的 choice 对象传给下游,保证现有逻辑完全不动 - sdkChoice := openai.ChatCompletionStreamChoice{ - Index: choice.Index, - Delta: choice.Delta.ChatCompletionStreamChoiceDelta, - FinishReason: choice.FinishReason, - } - c.processStreamDelta(ctx, &sdkChoice, state, streamChan, reasoning) - } - } -} - -// streamState 流式处理状态 -type streamState struct { - toolCallMap map[int]*types.LLMToolCall - lastFunctionName map[int]string - nameNotified map[int]bool - hasThinking bool - fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction - usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled - lastFinishReason string // last observed finish_reason for EOF handler fallback - - // Diagnostic flags (fire-once) used to log earliest signals of tool_call - // presence/absence at the OpenAI-protocol level. These are independent of - // the higher-level ResponseTypeToolCall marker (which only fires once - // function name has stabilized) and let us distinguish between - // (A) no tool_calls field ever observed (true natural-stop), and - // (B) tool_calls field observed but marker not yet emitted. - firstToolCallSeen bool // true once any delta carried tool_calls - noToolCallStopLogged bool // true once we logged "stop without tool_calls" - firstContentSeen bool // true once delta.Content first appeared - firstReasoningSeen bool // true once reasoning_content first appeared - streamStartedAt time.Time -} - -func newStreamState() *streamState { - return &streamState{ - toolCallMap: make(map[int]*types.LLMToolCall), - lastFunctionName: make(map[int]string), - nameNotified: make(map[int]bool), - hasThinking: false, - fieldExtractors: make(map[int]*jsonFieldExtractor), - streamStartedAt: time.Now(), - } -} - -// elapsedMs returns the milliseconds elapsed since the stream state was -// initialized. Used to attach time-since-stream-start to fire-once diagnostic -// logs so a single grep can reveal the temporal layout of a single stream -// (TTFC / TTFT / first-tool-call / natural-stop confirmation, etc). -func (s *streamState) elapsedMs() int64 { - if s.streamStartedAt.IsZero() { - return 0 - } - return time.Since(s.streamStartedAt).Milliseconds() -} - -func (s *streamState) buildOrderedToolCalls() []types.LLMToolCall { - if len(s.toolCallMap) == 0 { - return nil - } - result := make([]types.LLMToolCall, 0, len(s.toolCallMap)) - for i := 0; i < len(s.toolCallMap); i++ { - if tc, ok := s.toolCallMap[i]; ok && tc != nil { - result = append(result, *tc) - } - } - if len(result) == 0 { - return nil - } - return result -} - -// processStreamDelta 处理流式响应的单个 delta -func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.ChatCompletionStreamChoice, state *streamState, streamChan chan types.StreamResponse, reasoningContent string) { - delta := choice.Delta - isDone := string(choice.FinishReason) != "" - - // Track finish_reason for EOF handler fallback - if isDone { - state.lastFinishReason = string(choice.FinishReason) - } - - // 处理 tool calls - if len(delta.ToolCalls) > 0 { - c.processToolCallsDelta(ctx, delta.ToolCalls, state, streamChan) - } - - // Earliest reliable "no tool_calls" signal at the OpenAI-protocol level: - // finish_reason=stop arrived AND we never observed a tool_calls field on - // any prior delta. Logged once per stream so callers can grep for the - // natural-stop entry point without waiting for the higher-level summary. - if isDone && - string(choice.FinishReason) == "stop" && - !state.firstToolCallSeen && - !state.noToolCallStopLogged { - logger.Infof(ctx, "[LLM Stream] Natural-stop at OpenAI layer "+ - "(finish=stop, tool_calls field never observed, thinking_seen=%t, "+ - "first_content_seen=%t, elapsed_ms=%d)", - state.hasThinking, state.firstContentSeen, state.elapsedMs()) - state.noToolCallStopLogged = true - } - - // 发送思考内容(ReasoningContent,支持 DeepSeek 等模型) - if reasoningContent != "" { - // Earliest reasoning_content signal at the OpenAI-protocol level. Fired - // once per stream so we can distinguish "model emitted thinking before - // answer" vs "model never produced thinking" when triaging logs. - if !state.firstReasoningSeen { - state.firstReasoningSeen = true - logger.Infof(ctx, "[LLM Stream] First reasoning_content at OpenAI layer "+ - "(len=%d, preview=%q, elapsed_ms=%d)", - len(reasoningContent), truncateForDebug(reasoningContent, 80), state.elapsedMs()) - } - state.hasThinking = true - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Content: reasoningContent, - Done: false, - } - } - - // 发送回答内容 - if delta.Content != "" { - // Earliest delta.Content signal at the OpenAI-protocol level. Fired once - // per stream so we can measure TTFC (time-to-first-content) and tell - // "answer started before any tool_call" from "tool_call came first". - if !state.firstContentSeen { - state.firstContentSeen = true - logger.Infof(ctx, "[LLM Stream] First delta.Content at OpenAI layer "+ - "(len=%d, preview=%q, tool_call_seen=%t, thinking_seen=%t, elapsed_ms=%d)", - len(delta.Content), truncateForDebug(delta.Content, 80), - state.firstToolCallSeen, state.firstReasoningSeen, state.elapsedMs()) - } - // If we had thinking content and this is the first answer chunk, - // send a thinking done event first - if state.hasThinking { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Content: "", - Done: true, - } - state.hasThinking = false // Only send once - } - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Content: delta.Content, - Done: isDone, - ToolCalls: state.buildOrderedToolCalls(), - FinishReason: string(choice.FinishReason), - } - } - - if isDone && len(state.toolCallMap) > 0 { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Content: "", - Done: true, - ToolCalls: state.buildOrderedToolCalls(), - FinishReason: string(choice.FinishReason), - } - } - - // Ensure thinking done is sent when stream finishes without any answer content - // (e.g., model only produced reasoning then hit finish_reason with empty content). - if isDone && state.hasThinking { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Content: "", - Done: true, - } - state.hasThinking = false - } - - // Catch-all: isDone but none of the above branches sent a response with - // FinishReason (empty content, no tool calls, no thinking). This prevents - // the finish_reason from being lost in the streaming pipeline. - if isDone && delta.Content == "" && len(state.toolCallMap) == 0 && !state.hasThinking { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeAnswer, - Done: true, - FinishReason: string(choice.FinishReason), - } - } -} - -// processToolCallsDelta 处理 tool calls 的增量更新 -func (c *RemoteAPIChat) processToolCallsDelta(ctx context.Context, toolCalls []openai.ToolCall, state *streamState, streamChan chan types.StreamResponse) { - // Earliest signal at the OpenAI-protocol level that this stream will - // produce at least one tool call. Fires *before* the function name has - // stabilized, i.e. earlier than the higher-level ResponseTypeToolCall - // marker downstream consumers see. Useful for distinguishing - // "tool_calls field arrived but marker not yet emitted" from - // "tool_calls field truly absent" when triaging stream behavior. - if !state.firstToolCallSeen && len(toolCalls) > 0 { - state.firstToolCallSeen = true - var firstID, firstName string - for _, tc := range toolCalls { - if tc.ID != "" { - firstID = tc.ID - } - if tc.Function.Name != "" { - firstName = tc.Function.Name - } - if firstID != "" || firstName != "" { - break - } - } - logger.Infof(ctx, "[LLM Stream] First tool_calls delta at OpenAI layer "+ - "(count=%d, first_id=%q, first_name=%q, "+ - "first_content_seen=%t, thinking_seen=%t, elapsed_ms=%d)", - len(toolCalls), firstID, firstName, - state.firstContentSeen, state.firstReasoningSeen, state.elapsedMs()) - } - - for _, tc := range toolCalls { - var toolCallIndex int - if tc.Index != nil { - toolCallIndex = *tc.Index - } - toolCallEntry, exists := state.toolCallMap[toolCallIndex] - if !exists || toolCallEntry == nil { - toolCallEntry = &types.LLMToolCall{ - Type: string(tc.Type), - Function: types.FunctionCall{ - Name: "", - Arguments: "", - }, - } - state.toolCallMap[toolCallIndex] = toolCallEntry - } - - if tc.ID != "" { - toolCallEntry.ID = tc.ID - } - if tc.Type != "" { - toolCallEntry.Type = string(tc.Type) - } - if tc.Function.Name != "" { - // 防御性校验:解决部分供应商(如vLLM Ascend等)在每个流 Chunk 中重复发送完整工具名的问题。 - // 如果当前已存名字与新收到名字一致,则视为冗余重复,不进行叠加。 - if toolCallEntry.Function.Name != tc.Function.Name { - toolCallEntry.Function.Name += tc.Function.Name - } - } - - argsUpdated := false - if tc.Function.Arguments != "" { - toolCallEntry.Function.Arguments += tc.Function.Arguments - argsUpdated = true - } - - currName := toolCallEntry.Function.Name - if currName != "" && - currName == state.lastFunctionName[toolCallIndex] && - argsUpdated && - !state.nameNotified[toolCallIndex] && - toolCallEntry.ID != "" { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeToolCall, - Content: "", - Done: false, - Data: map[string]interface{}{ - "tool_name": currName, - "tool_call_id": toolCallEntry.ID, - }, - } - state.nameNotified[toolCallIndex] = true - } - - state.lastFunctionName[toolCallIndex] = currName - - // Stream thinking tool's thought field as thinking-type chunks - if toolCallEntry.Function.Name == "thinking" && argsUpdated { - extractor, exists := state.fieldExtractors[toolCallIndex] - if !exists { - extractor = newJSONFieldExtractor("thought") - state.fieldExtractors[toolCallIndex] = extractor - } - thoughtChunk := extractor.Feed(tc.Function.Arguments) - if thoughtChunk != "" { - streamChan <- types.StreamResponse{ - ResponseType: types.ResponseTypeThinking, - Content: thoughtChunk, - Done: false, - Data: map[string]interface{}{ - "source": "thinking_tool", - "tool_call_id": toolCallEntry.ID, - }, - } - } - } - } -} - // GetModelName 获取模型名称 func (c *RemoteAPIChat) GetModelName() string { return c.modelName @@ -1174,23 +395,3 @@ func (c *RemoteAPIChat) GetBaseURL() string { func (c *RemoteAPIChat) GetAPIKey() string { return c.apiKey } - -// cachedTokens returns the cached prompt-token count from an OpenAI-compatible -// usage detail block, or zero when the provider did not report one. Some -// providers omit PromptTokensDetails entirely, so the nil guard is required. -// -// Note on provider semantics: -// - Implicit-cache providers (OpenAI, Azure OpenAI, DeepSeek, …) populate -// `cached_tokens` automatically whenever the prompt prefix matches a -// previous request — no caller opt-in is required. -// - Explicit-cache providers (Qwen on Aliyun, Anthropic Claude, …) only -// populate `cached_tokens` after the caller attaches `cache_control: -// {"type": "ephemeral"}` to the relevant message / content block. This -// helper still returns zero for those providers until that opt-in is -// applied upstream of the request. -func cachedTokens(d *openai.PromptTokensDetails) int { - if d == nil { - return 0 - } - return d.CachedTokens -} diff --git a/internal/models/chat/remote_api_test.go b/internal/models/chat/remote_api_test.go index d0ce0048..a870f6be 100644 --- a/internal/models/chat/remote_api_test.go +++ b/internal/models/chat/remote_api_test.go @@ -178,7 +178,7 @@ func TestBuildChatCompletionRequest_GPT5MaxCompletionTokens(t *testing.T) { FrequencyPenalty: 0.1, PresencePenalty: 0.2, } - req := c.BuildChatCompletionRequest(messages, opts, false) + req := c.shapedRequest(messages, opts, false) if tc.shouldRewriteMaxT { assert.Equal(t, 0, req.MaxTokens, "MaxTokens must NOT be sent for GPT-5/o-series") @@ -201,7 +201,7 @@ func TestBuildChatCompletionRequest_GPT5MaxCompletionTokens(t *testing.T) { MaxTokens: 128, MaxCompletionTokens: 2048, } - req := c.BuildChatCompletionRequest(messages, opts, false) + req := c.shapedRequest(messages, opts, false) assert.Equal(t, 0, req.MaxTokens) assert.Equal(t, 2048, req.MaxCompletionTokens) }) diff --git a/internal/models/chat/stream_emit.go b/internal/models/chat/stream_emit.go new file mode 100644 index 00000000..b3c6c4cd --- /dev/null +++ b/internal/models/chat/stream_emit.go @@ -0,0 +1,36 @@ +package chat + +import "github.com/Tencent/WeKnora/internal/types" + +// thinkingEmitter owns the "reasoning then answer" hand-off that every +// streaming Chat implementation shares: thinking chunks are forwarded as they +// arrive, and exactly one thinking-done marker is emitted before the first +// answer token (or when the stream ends without one). Centralizing the +// bookkeeping keeps the OpenAI-compatible and Ollama stream loops in sync. +type thinkingEmitter struct { + active bool +} + +// emit forwards a reasoning chunk and records that a thinking-done marker is +// still owed. +func (e *thinkingEmitter) emit(ch chan types.StreamResponse, content string) { + e.active = true + ch <- types.StreamResponse{ + ResponseType: types.ResponseTypeThinking, + Content: content, + Done: false, + } +} + +// finish emits the single thinking-done marker if one is owed. Safe to call +// multiple times; only the first call after an emit sends anything. +func (e *thinkingEmitter) finish(ch chan types.StreamResponse) { + if !e.active { + return + } + e.active = false + ch <- types.StreamResponse{ + ResponseType: types.ResponseTypeThinking, + Done: true, + } +} diff --git a/internal/models/chat/stream_raw_dump.go b/internal/models/chat/stream_raw_dump.go index a64d51c0..7ed7e4e3 100644 --- a/internal/models/chat/stream_raw_dump.go +++ b/internal/models/chat/stream_raw_dump.go @@ -126,6 +126,54 @@ func (d *streamPacketDumper) WritePacketRaw(raw []byte) { _, _ = d.file.Write([]byte{'\n'}) } +// WriteError appends a terminal error record (stream read failure, API error, etc.). +func (d *streamPacketDumper) WriteError(message string) { + if d == nil || d.file == nil || strings.TrimSpace(message) == "" { + return + } + line, err := json.Marshal(map[string]any{ + "type": "error", + "model": d.model, + "timestamp": time.Now().UTC().Format(time.RFC3339Nano), + "message": message, + }) + if err != nil { + return + } + _ = d.writeLine(line) +} + +// WriteHTTPError appends a non-2xx HTTP response before any SSE packets. +func (d *streamPacketDumper) WriteHTTPError(statusCode int, body []byte) { + if d == nil || d.file == nil { + return + } + entry := map[string]any{ + "type": "http_error", + "model": d.model, + "timestamp": time.Now().UTC().Format(time.RFC3339Nano), + "status_code": statusCode, + } + if len(body) > 0 { + trimmed := bytesTrimSpace(body) + if json.Valid(trimmed) { + var parsed any + if json.Unmarshal(trimmed, &parsed) == nil { + entry["body"] = parsed + } else { + entry["body_raw"] = string(trimmed) + } + } else { + entry["body_raw"] = string(trimmed) + } + } + line, err := json.Marshal(entry) + if err != nil { + return + } + _ = d.writeLine(line) +} + // WritePacket marshals v as one JSON object per line (SDK stream Recv path). func (d *streamPacketDumper) WritePacket(v any) { if d == nil || v == nil { diff --git a/internal/models/chat/thinking.go b/internal/models/chat/thinking.go new file mode 100644 index 00000000..f676208d --- /dev/null +++ b/internal/models/chat/thinking.go @@ -0,0 +1,140 @@ +package chat + +import ( + "strings" + + "github.com/sashabaranov/go-openai" +) + +// ExtraConfigThinkingControl is the model parameters.extra_config key for +// selecting how ChatOptions.Thinking is translated to provider HTTP fields. +// The accepted values mirror the strings the frontend writes (see +// ModelEditorDialog.vue): "none", "enable_thinking", "thinking_type", +// "chat_template_kwargs". +const ExtraConfigThinkingControl = "thinking_control" + +// Wire-format request bodies used by providers that express extended-thinking +// through a non-standard top-level field. They embed the standard OpenAI +// request so all other fields are marshalled unchanged. + +// QwenChatCompletionRequest adds Aliyun Qwen's `enable_thinking` boolean. +type QwenChatCompletionRequest struct { + openai.ChatCompletionRequest + EnableThinking *bool `json:"enable_thinking,omitempty"` +} + +// ThinkingConfig is the `{ "type": "enabled"|"disabled" }` block used by +// LKEAP / Volcengine style providers. +type ThinkingConfig struct { + Type string `json:"type"` +} + +// ThinkingChatCompletionRequest adds the `thinking` object for providers that +// use the `{ "thinking": { "type": ... } }` wire format. +type ThinkingChatCompletionRequest struct { + openai.ChatCompletionRequest + Thinking *ThinkingConfig `json:"thinking,omitempty"` +} + +// ThinkingStrategy encodes how ChatOptions.Thinking is mapped onto a provider's +// HTTP request. Apply returns (customBody, useRawHTTP): +// - (nil, false) means "send the standard OpenAI request unchanged" (the +// caller keeps using the SDK path). +// - a non-nil customBody must be sent verbatim over raw HTTP because it +// carries fields the OpenAI SDK would strip. +// +// When opts.Thinking is nil most strategies emit nothing, deferring to the +// model's own default; the exception is enableThinking{alwaysSend: true} +// (Aliyun Qwen), which must always pin the field. +type ThinkingStrategy interface { + Apply(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (customBody any, useRawHTTP bool) +} + +// noThinking sends no thinking-related fields at all. +type noThinking struct{} + +func (noThinking) Apply(*openai.ChatCompletionRequest, *ChatOptions, bool) (any, bool) { + return nil, false +} + +// enableThinking encodes thinking via Qwen's `enable_thinking` boolean. +// +// - alwaysSend: pin the field even when opts.Thinking is nil (Aliyun Qwen +// thinking models require it on every request; default value is false). +// - disableOnNonStream: force enable_thinking=false for non-stream requests +// (Qwen3 rejects thinking in non-stream mode). +type enableThinking struct { + alwaysSend bool + disableOnNonStream bool +} + +func (s enableThinking) Apply(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) { + thinking := false + switch { + case opts != nil && opts.Thinking != nil: + thinking = *opts.Thinking + case !s.alwaysSend: + return nil, false + } + if s.disableOnNonStream && !isStream { + thinking = false + } + qwenReq := QwenChatCompletionRequest{ChatCompletionRequest: *req} + qwenReq.EnableThinking = &thinking + return qwenReq, true +} + +// thinkingTypeField encodes thinking via the `{ "thinking": { "type": ... } }` +// object (LKEAP / Volcengine). Emits nothing when opts.Thinking is unset. +type thinkingTypeField struct{} + +func (thinkingTypeField) Apply(req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool) (any, bool) { + if opts == nil || opts.Thinking == nil { + return nil, false + } + r := ThinkingChatCompletionRequest{ChatCompletionRequest: *req} + thinkingType := "disabled" + if *opts.Thinking { + thinkingType = "enabled" + } + r.Thinking = &ThinkingConfig{Type: thinkingType} + return r, true +} + +// chatTemplateKwargs encodes thinking via the standard request's +// `chat_template_kwargs.enable_thinking` (vLLM / NVIDIA / generic local +// deployments). Emits nothing when opts.Thinking is unset. +type chatTemplateKwargs struct{} + +func (chatTemplateKwargs) Apply(req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool) (any, bool) { + if opts == nil || opts.Thinking == nil { + return nil, false + } + req.ChatTemplateKwargs = map[string]interface{}{ + "enable_thinking": *opts.Thinking, + } + return req, true +} + +// parseThinkingOverride reads extra_config.thinking_control and returns the +// strategy it selects, or nil when unset (the provider adapter's default +// strategy then applies). An unrecognized non-empty value falls back to +// chat_template_kwargs, preserving the legacy default-mode behavior. +func parseThinkingOverride(extraConfig map[string]string) ThinkingStrategy { + if extraConfig == nil { + return nil + } + switch strings.ToLower(strings.TrimSpace(extraConfig[ExtraConfigThinkingControl])) { + case "": + return nil + case "none": + return noThinking{} + case "enable_thinking": + return enableThinking{} + case "thinking_type": + return thinkingTypeField{} + default: + // "chat_template_kwargs" and any unknown non-empty value. + return chatTemplateKwargs{} + } +} diff --git a/internal/models/chat/thinking_test.go b/internal/models/chat/thinking_test.go new file mode 100644 index 00000000..d55ad0a1 --- /dev/null +++ b/internal/models/chat/thinking_test.go @@ -0,0 +1,127 @@ +package chat + +import ( + "encoding/json" + "testing" + + "github.com/sashabaranov/go-openai" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func ptrBool(b bool) *bool { return &b } + +// TestThinkingStrategy_NilThinking verifies the strategies that defer to the +// model default emit nothing when ChatOptions.Thinking is unset. +func TestThinkingStrategy_NilThinking(t *testing.T) { + req := openai.ChatCompletionRequest{Model: "test"} + strategies := []ThinkingStrategy{ + noThinking{}, + enableThinking{}, // not alwaysSend + thinkingTypeField{}, + chatTemplateKwargs{}, + } + for _, s := range strategies { + custom, raw := s.Apply(&req, nil, true) + assert.Nil(t, custom, "%T", s) + assert.False(t, raw, "%T", s) + } +} + +// TestEnableThinking_QwenSemantics pins the Aliyun Qwen behavior: thinking is +// always sent, defaults to false, and is forced off on non-stream requests. +func TestEnableThinking_QwenSemantics(t *testing.T) { + s := enableThinking{alwaysSend: true, disableOnNonStream: true} + req := openai.ChatCompletionRequest{Model: "qwen3-32b"} + + t.Run("non-stream forces false even when requested true", func(t *testing.T) { + custom, raw := s.Apply(&req, &ChatOptions{Thinking: ptrBool(true)}, false) + require.True(t, raw) + qwen, ok := custom.(QwenChatCompletionRequest) + require.True(t, ok) + require.NotNil(t, qwen.EnableThinking) + assert.False(t, *qwen.EnableThinking) + }) + + t.Run("stream honors requested true", func(t *testing.T) { + custom, raw := s.Apply(&req, &ChatOptions{Thinking: ptrBool(true)}, true) + require.True(t, raw) + qwen := custom.(QwenChatCompletionRequest) + require.NotNil(t, qwen.EnableThinking) + assert.True(t, *qwen.EnableThinking) + }) + + t.Run("stream defaults to false when unset", func(t *testing.T) { + custom, raw := s.Apply(&req, nil, true) + require.True(t, raw) + qwen := custom.(QwenChatCompletionRequest) + require.NotNil(t, qwen.EnableThinking) + assert.False(t, *qwen.EnableThinking) + }) +} + +// TestEnableThinking_ExtraConfigSemantics pins the extra_config "enable_thinking" +// override: only sent when explicitly requested. +func TestEnableThinking_ExtraConfigSemantics(t *testing.T) { + s := enableThinking{} + req := openai.ChatCompletionRequest{Model: "qwen3"} + + custom, raw := s.Apply(&req, &ChatOptions{Thinking: ptrBool(true)}, true) + require.True(t, raw) + qwen := custom.(QwenChatCompletionRequest) + require.NotNil(t, qwen.EnableThinking) + assert.True(t, *qwen.EnableThinking) + + custom, raw = s.Apply(&req, nil, true) + assert.Nil(t, custom) + assert.False(t, raw) +} + +func TestThinkingTypeField(t *testing.T) { + s := thinkingTypeField{} + req := openai.ChatCompletionRequest{Model: "ds-v3"} + + custom, raw := s.Apply(&req, &ChatOptions{Thinking: ptrBool(false)}, true) + require.True(t, raw) + typed, ok := custom.(ThinkingChatCompletionRequest) + require.True(t, ok) + require.NotNil(t, typed.Thinking) + assert.Equal(t, "disabled", typed.Thinking.Type) + + custom, raw = s.Apply(&req, &ChatOptions{Thinking: ptrBool(true)}, true) + require.True(t, raw) + assert.Equal(t, "enabled", custom.(ThinkingChatCompletionRequest).Thinking.Type) +} + +func TestChatTemplateKwargs(t *testing.T) { + s := chatTemplateKwargs{} + req := openai.ChatCompletionRequest{Model: "vllm"} + + custom, raw := s.Apply(&req, &ChatOptions{Thinking: ptrBool(true)}, true) + require.True(t, raw) + out, ok := custom.(*openai.ChatCompletionRequest) + require.True(t, ok) + assert.Equal(t, true, out.ChatTemplateKwargs["enable_thinking"]) + + body, err := json.Marshal(custom) + require.NoError(t, err) + assert.Contains(t, string(body), "chat_template_kwargs") +} + +func TestParseThinkingOverride(t *testing.T) { + cases := map[string]ThinkingStrategy{ + "none": noThinking{}, + "enable_thinking": enableThinking{}, + "thinking_type": thinkingTypeField{}, + "chat_template_kwargs": chatTemplateKwargs{}, + "something-unknown": chatTemplateKwargs{}, // legacy default-mode fallback + } + for value, want := range cases { + got := parseThinkingOverride(map[string]string{ExtraConfigThinkingControl: value}) + assert.IsType(t, want, got, "value=%q", value) + } + + assert.Nil(t, parseThinkingOverride(nil)) + assert.Nil(t, parseThinkingOverride(map[string]string{})) + assert.Nil(t, parseThinkingOverride(map[string]string{ExtraConfigThinkingControl: ""})) +} diff --git a/internal/models/chat/transport.go b/internal/models/chat/transport.go new file mode 100644 index 00000000..8496bd1f --- /dev/null +++ b/internal/models/chat/transport.go @@ -0,0 +1,59 @@ +package chat + +import ( + "context" + "net/http" + "os" + "strconv" + "strings" + "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// LLM 调用超时配置。仅作为"上层未设置 deadline 时"的兜底,避免 hung 请求 +// 永久阻塞 worker。如果上层 ctx 已经设置了 deadline(无论比默认更短还是更长), +// 都会原样尊重,不再叠加默认超时。可通过环境变量覆盖: +// - WEKNORA_LLM_CHAT_TIMEOUT_SECONDS 非流式调用兜底超时(默认 600s) +// - WEKNORA_LLM_STREAM_TIMEOUT_SECONDS 流式调用兜底超时(默认 1800s) +var ( + defaultChatTimeout = envDurationSeconds("WEKNORA_LLM_CHAT_TIMEOUT_SECONDS", 300*time.Second) + defaultStreamTimeout = envDurationSeconds("WEKNORA_LLM_STREAM_TIMEOUT_SECONDS", 600*time.Second) +) + +// envDurationSeconds 读取以"秒"为单位的环境变量,解析失败或非正值时回退到 fallback。 +func envDurationSeconds(key string, fallback time.Duration) time.Duration { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return fallback + } + return time.Duration(n) * time.Second +} + +// withLLMTimeout 仅在上层 ctx 没有 deadline 时附加一个兜底超时; +// 如果上层已显式设置 deadline(无论更短或更长),则原样返回, +// 让调用方对自己的超时策略拥有最终决定权。 +func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return ctx, func() {} + } + return context.WithTimeout(ctx, d) +} + +// rawHTTPClient is a shared HTTP client for raw HTTP LLM calls with connection-level timeouts. +// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout) +// rather than http.Client.Timeout, so streaming calls are not prematurely terminated. +// Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer. +var rawHTTPClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: secutils.SSRFSafeDialContext, + TLSHandshakeTimeout: 10 * time.Second, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 5, + }, +} diff --git a/internal/models/chat/usage.go b/internal/models/chat/usage.go new file mode 100644 index 00000000..d08e5798 --- /dev/null +++ b/internal/models/chat/usage.go @@ -0,0 +1,20 @@ +package chat + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// logUsage emits the standard "[LLM Usage]" line shared by every Chat +// implementation. It is a no-op when usage is nil so callers can pass through +// optional usage blocks without guarding at each call site. +func logUsage(ctx context.Context, model string, u *types.TokenUsage) { + if u == nil { + return + } + logger.Infof(ctx, + "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + model, u.PromptTokens, u.CompletionTokens, u.TotalTokens, u.CachedTokens) +}