fix(chat): 为 LLM 非流式/流式调用添加超时保护,防止 worker 被 hung 请求永久阻塞

IM 渠道(企微等)的消息处理走 KnowledgeQA 管道,该管道的 LLM 调用此前
未设置任何超时,依赖 context cancellation 控制。然而 QA 管道传入的 context
本身无 deadline,一旦上游 LLM API(如 mimo-v2.5-pro)的网络请求 hang 在
TCP 层面(连接未断但无数据返回),worker goroutine 会永久阻塞在
CreateChatCompletion / CreateChatCompletionStream 上。

随着时间推移,多个 worker 陆续卡死,队列中所有 worker 耗尽后机器人对任何
消息都不再回复。

本提交在 RemoteAPIChat.Chat 和 RemoteAPIChat.ChatStream 两个入口方法中
统一添加 context.WithTimeout:
- 非流式调用(Chat):120s 超时
- 流式调用(ChatStream):300s 超时

如果上层调用方已设置更短的 deadline,WithTimeout 自动遵循更短的那个,
不会覆盖已有的合理超时设置。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
杨明康
2026-05-09 16:23:07 +08:00
committed by lyingbug
parent 02be05c095
commit 6b8dbaa25b
+44 -15
View File
@@ -17,8 +17,19 @@ import (
"github.com/sashabaranov/go-openai"
)
const (
// defaultChatTimeout is the per-call timeout for non-streaming LLM requests.
// Prevents a single hung API call from blocking a worker indefinitely.
defaultChatTimeout = 120 * time.Second
// defaultStreamTimeout is the per-call timeout for streaming LLM requests.
// Longer than defaultChatTimeout because streaming responses (especially with
// thinking/reasoning) can take minutes to produce the first token and complete.
defaultStreamTimeout = 300 * time.Second
)
// rawHTTPClient is a shared HTTP client for raw HTTP LLM calls with connection-level timeouts.
// No overall Timeout is set so streaming calls are controlled by context cancellation only.
// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout)
// rather than http.Client.Timeout, so streaming calls are not prematurely terminated.
// Uses SSRFSafeDialContext to prevent DNS rebinding attacks at the connection layer.
var rawHTTPClient = &http.Client{
Transport: &http.Transport{
@@ -308,6 +319,12 @@ func (c *RemoteAPIChat) logRequest(ctx context.Context, req any, isStream bool)
// Chat 进行非流式聊天
func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
// Enforce a per-call timeout to prevent a hung API request from blocking the
// worker indefinitely. If the caller has already set a shorter deadline,
// context.WithTimeout respects it.
timeoutCtx, cancel := context.WithTimeout(ctx, defaultChatTimeout)
defer cancel()
req := c.BuildChatCompletionRequest(messages, opts, false)
var customEndpoint string
if c.endpointCustomizer != nil {
@@ -317,23 +334,23 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
if c.requestCustomizer != nil {
customReq, useRawHTTP := c.requestCustomizer(&req, opts, false)
if useRawHTTP && customReq != nil {
return c.chatWithRawHTTP(ctx, customEndpoint, customReq)
return c.chatWithRawHTTP(timeoutCtx, customEndpoint, customReq)
}
}
// 使用自定义请求地址
if customEndpoint != "" {
return c.chatWithRawHTTP(ctx, customEndpoint, &req)
return c.chatWithRawHTTP(timeoutCtx, customEndpoint, &req)
}
c.logRequest(ctx, req, false)
resp, err := c.client.CreateChatCompletion(ctx, req)
c.logRequest(timeoutCtx, req, false)
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
if err != nil {
if isMultimodalNotSupportedError(err) {
logger.Warnf(ctx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
cleaned := stripImagesFromMessages(messages)
req = c.BuildChatCompletionRequest(cleaned, opts, false)
resp, err = c.client.CreateChatCompletion(ctx, req)
resp, err = c.client.CreateChatCompletion(timeoutCtx, req)
}
if err != nil {
return nil, fmt.Errorf("create chat completion: %w", err)
@@ -344,7 +361,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
if err != nil {
return nil, err
}
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
return result, nil
}
@@ -476,6 +493,12 @@ func removeThinkingContent(content string) string {
// ChatStream 进行流式聊天
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
// Enforce a per-call timeout to prevent a hung stream from blocking the worker.
// Uses a longer timeout than Chat because streaming responses (especially with
// thinking/reasoning) can take minutes to complete.
// If the caller has already set a shorter deadline, WithTimeout respects it.
timeoutCtx, cancel := context.WithTimeout(ctx, defaultStreamTimeout)
req := c.BuildChatCompletionRequest(messages, opts, true)
var customEndpoint string
@@ -487,32 +510,38 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
if c.requestCustomizer != nil {
customReq, useRawHTTP := c.requestCustomizer(&req, opts, true)
if useRawHTTP && customReq != nil {
return c.chatStreamWithRawHTTP(ctx, customEndpoint, customReq)
// chatStreamWithRawHTTP starts its own goroutine; the timeout context
// will be cancelled when the deadline expires, providing the safety net.
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, customReq)
}
}
// 使用自定义请求地址
if customEndpoint != "" {
return c.chatStreamWithRawHTTP(ctx, customEndpoint, &req)
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, &req)
}
c.logRequest(ctx, req, true)
c.logRequest(timeoutCtx, req, true)
streamChan := make(chan types.StreamResponse)
stream, err := c.client.CreateChatCompletionStream(ctx, req)
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
if err != nil {
if isMultimodalNotSupportedError(err) {
logger.Warnf(ctx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
cleaned := stripImagesFromMessages(messages)
req = c.BuildChatCompletionRequest(cleaned, opts, true)
stream, err = c.client.CreateChatCompletionStream(ctx, req)
stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req)
}
if err != nil {
cancel()
close(streamChan)
return nil, fmt.Errorf("create chat completion stream: %w", err)
}
}
go c.processStream(ctx, stream, streamChan)
go func() {
defer cancel()
c.processStream(timeoutCtx, stream, streamChan)
}()
return streamChan, nil
}