fix(chat): 为 LLM 非流式/流式调用添加超时保护,防止 worker 被 hung 请求永久阻塞
IM 渠道(企微等)的消息处理走 KnowledgeQA 管道,该管道的 LLM 调用此前 未设置任何超时,依赖 context cancellation 控制。然而 QA 管道传入的 context 本身无 deadline,一旦上游 LLM API(如 mimo-v2.5-pro)的网络请求 hang 在 TCP 层面(连接未断但无数据返回),worker goroutine 会永久阻塞在 CreateChatCompletion / CreateChatCompletionStream 上。 随着时间推移,多个 worker 陆续卡死,队列中所有 worker 耗尽后机器人对任何 消息都不再回复。 本提交在 RemoteAPIChat.Chat 和 RemoteAPIChat.ChatStream 两个入口方法中 统一添加 context.WithTimeout: - 非流式调用(Chat):120s 超时 - 流式调用(ChatStream):300s 超时 如果上层调用方已设置更短的 deadline,WithTimeout 自动遵循更短的那个, 不会覆盖已有的合理超时设置。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -17,8 +17,19 @@ import (
|
||||
"github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultChatTimeout is the per-call timeout for non-streaming LLM requests.
|
||||
// Prevents a single hung API call from blocking a worker indefinitely.
|
||||
defaultChatTimeout = 120 * time.Second
|
||||
// defaultStreamTimeout is the per-call timeout for streaming LLM requests.
|
||||
// Longer than defaultChatTimeout because streaming responses (especially with
|
||||
// thinking/reasoning) can take minutes to produce the first token and complete.
|
||||
defaultStreamTimeout = 300 * time.Second
|
||||
)
|
||||
|
||||
// rawHTTPClient is a shared HTTP client for raw HTTP LLM calls with connection-level timeouts.
|
||||
// No overall Timeout is set so streaming calls are controlled by context cancellation only.
|
||||
// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout)
|
||||
// rather than http.Client.Timeout, so streaming calls are not prematurely terminated.
|
||||
// Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer.
|
||||
var rawHTTPClient = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
@@ -308,6 +319,12 @@ func (c *RemoteAPIChat) logRequest(ctx context.Context, req any, isStream bool)
|
||||
|
||||
// Chat 进行非流式聊天
|
||||
func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
// Enforce a per-call timeout to prevent a hung API request from blocking the
|
||||
// worker indefinitely. If the caller has already set a shorter deadline,
|
||||
// context.WithTimeout respects it.
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, defaultChatTimeout)
|
||||
defer cancel()
|
||||
|
||||
req := c.BuildChatCompletionRequest(messages, opts, false)
|
||||
var customEndpoint string
|
||||
if c.endpointCustomizer != nil {
|
||||
@@ -317,23 +334,23 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
if c.requestCustomizer != nil {
|
||||
customReq, useRawHTTP := c.requestCustomizer(&req, opts, false)
|
||||
if useRawHTTP && customReq != nil {
|
||||
return c.chatWithRawHTTP(ctx, customEndpoint, customReq)
|
||||
return c.chatWithRawHTTP(timeoutCtx, customEndpoint, customReq)
|
||||
}
|
||||
}
|
||||
|
||||
// 使用自定义请求地址
|
||||
if customEndpoint != "" {
|
||||
return c.chatWithRawHTTP(ctx, customEndpoint, &req)
|
||||
return c.chatWithRawHTTP(timeoutCtx, customEndpoint, &req)
|
||||
}
|
||||
|
||||
c.logRequest(ctx, req, false)
|
||||
resp, err := c.client.CreateChatCompletion(ctx, req)
|
||||
c.logRequest(timeoutCtx, req, false)
|
||||
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
|
||||
if err != nil {
|
||||
if isMultimodalNotSupportedError(err) {
|
||||
logger.Warnf(ctx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
cleaned := stripImagesFromMessages(messages)
|
||||
req = c.BuildChatCompletionRequest(cleaned, opts, false)
|
||||
resp, err = c.client.CreateChatCompletion(ctx, req)
|
||||
resp, err = c.client.CreateChatCompletion(timeoutCtx, req)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create chat completion: %w", err)
|
||||
@@ -344,7 +361,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
|
||||
return result, nil
|
||||
}
|
||||
@@ -476,6 +493,12 @@ func removeThinkingContent(content string) string {
|
||||
|
||||
// ChatStream 进行流式聊天
|
||||
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
// Enforce a per-call timeout to prevent a hung stream from blocking the worker.
|
||||
// Uses a longer timeout than Chat because streaming responses (especially with
|
||||
// thinking/reasoning) can take minutes to complete.
|
||||
// If the caller has already set a shorter deadline, WithTimeout respects it.
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, defaultStreamTimeout)
|
||||
|
||||
req := c.BuildChatCompletionRequest(messages, opts, true)
|
||||
|
||||
var customEndpoint string
|
||||
@@ -487,32 +510,38 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
if c.requestCustomizer != nil {
|
||||
customReq, useRawHTTP := c.requestCustomizer(&req, opts, true)
|
||||
if useRawHTTP && customReq != nil {
|
||||
return c.chatStreamWithRawHTTP(ctx, customEndpoint, customReq)
|
||||
// chatStreamWithRawHTTP starts its own goroutine; the timeout context
|
||||
// will be cancelled when the deadline expires, providing the safety net.
|
||||
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, customReq)
|
||||
}
|
||||
}
|
||||
// 使用自定义请求地址
|
||||
if customEndpoint != "" {
|
||||
return c.chatStreamWithRawHTTP(ctx, customEndpoint, &req)
|
||||
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, &req)
|
||||
}
|
||||
c.logRequest(ctx, req, true)
|
||||
c.logRequest(timeoutCtx, req, true)
|
||||
|
||||
streamChan := make(chan types.StreamResponse)
|
||||
|
||||
stream, err := c.client.CreateChatCompletionStream(ctx, req)
|
||||
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||
if err != nil {
|
||||
if isMultimodalNotSupportedError(err) {
|
||||
logger.Warnf(ctx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||
cleaned := stripImagesFromMessages(messages)
|
||||
req = c.BuildChatCompletionRequest(cleaned, opts, true)
|
||||
stream, err = c.client.CreateChatCompletionStream(ctx, req)
|
||||
stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||
}
|
||||
if err != nil {
|
||||
cancel()
|
||||
close(streamChan)
|
||||
return nil, fmt.Errorf("create chat completion stream: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
go c.processStream(ctx, stream, streamChan)
|
||||
go func() {
|
||||
defer cancel()
|
||||
c.processStream(timeoutCtx, stream, streamChan)
|
||||
}()
|
||||
|
||||
return streamChan, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user