fix(chat): LLM 兜底超时仅在上层无 deadline 时生效,并修复 ChatStream raw HTTP 路径的 cancel 泄漏
PR #1238 引入的 context.WithTimeout 会把上层 ctx 的 deadline 强制截断到 默认 120s/300s,导致调用方无法设置更长的合理超时(例如长推理模型场景)。 同时 ChatStream 在走 chatStreamWithRawHTTP 早返回路径时丢弃了 cancel 函数,timeoutCtx 要等 deadline 到期才释放,go vet/lostcancel 也会告警。 本次修复: - 新增 withLLMTimeout 辅助函数:仅在上层 ctx 没有 deadline 时附加默认 超时;上层若已显式设置 deadline(无论比默认更短还是更长),都原样 尊重,把超时的最终决定权交还给调用方。 - 默认值通过环境变量可覆盖: - WEKNORA_LLM_CHAT_TIMEOUT_SECONDS (默认 120s) - WEKNORA_LLM_STREAM_TIMEOUT_SECONDS (默认 300s) - 修复 ChatStream raw HTTP 早返回路径未调用 cancel 的泄漏:新增 wrapStreamCancel,在底层 channel 关闭后统一执行 cancel。 - 补充单测覆盖三种 deadline 场景与环境变量解析。
This commit is contained in:
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -17,16 +19,39 @@ import (
|
||||
"github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultChatTimeout is the per-call timeout for non-streaming LLM requests.
|
||||
// Prevents a single hung API call from blocking a worker indefinitely.
|
||||
defaultChatTimeout = 120 * time.Second
|
||||
// defaultStreamTimeout is the per-call timeout for streaming LLM requests.
|
||||
// Longer than defaultChatTimeout because streaming responses (especially with
|
||||
// thinking/reasoning) can take minutes to produce the first token and complete.
|
||||
defaultStreamTimeout = 300 * time.Second
|
||||
// LLM 调用超时配置。仅作为"上层未设置 deadline 时"的兜底,避免 hung 请求
|
||||
// 永久阻塞 worker。如果上层 ctx 已经设置了 deadline(无论比默认更短还是更长),
|
||||
// 都会原样尊重,不再叠加默认超时。可通过环境变量覆盖:
|
||||
// - WEKNORA_LLM_CHAT_TIMEOUT_SECONDS 非流式调用兜底超时(默认 120s)
|
||||
// - WEKNORA_LLM_STREAM_TIMEOUT_SECONDS 流式调用兜底超时(默认 300s)
|
||||
var (
|
||||
defaultChatTimeout = envDurationSeconds("WEKNORA_LLM_CHAT_TIMEOUT_SECONDS", 120*time.Second)
|
||||
defaultStreamTimeout = envDurationSeconds("WEKNORA_LLM_STREAM_TIMEOUT_SECONDS", 300*time.Second)
|
||||
)
|
||||
|
||||
// envDurationSeconds 读取以"秒"为单位的环境变量,解析失败或非正值时回退到 fallback。
|
||||
func envDurationSeconds(key string, fallback time.Duration) time.Duration {
|
||||
v := strings.TrimSpace(os.Getenv(key))
|
||||
if v == "" {
|
||||
return fallback
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return time.Duration(n) * time.Second
|
||||
}
|
||||
|
||||
// withLLMTimeout 仅在上层 ctx 没有 deadline 时附加一个兜底超时;
|
||||
// 如果上层已显式设置 deadline(无论更短或更长),则原样返回,
|
||||
// 让调用方对自己的超时策略拥有最终决定权。
|
||||
func withLLMTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return ctx, func() {}
|
||||
}
|
||||
return context.WithTimeout(ctx, d)
|
||||
}
|
||||
|
||||
// rawHTTPClient is a shared HTTP client for raw HTTP LLM calls with connection-level timeouts.
|
||||
// Per-request timeout is enforced via context deadline (see defaultChatTimeout / defaultStreamTimeout)
|
||||
// rather than http.Client.Timeout, so streaming calls are not prematurely terminated.
|
||||
@@ -319,10 +344,9 @@ func (c *RemoteAPIChat) logRequest(ctx context.Context, req any, isStream bool)
|
||||
|
||||
// Chat 进行非流式聊天
|
||||
func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
// Enforce a per-call timeout to prevent a hung API request from blocking the
|
||||
// worker indefinitely. If the caller has already set a shorter deadline,
|
||||
// context.WithTimeout respects it.
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, defaultChatTimeout)
|
||||
// 仅在调用方未设置 deadline 时附加一个兜底超时,防止 hung 请求永久阻塞 worker;
|
||||
// 调用方若显式设置了更短或更长的 deadline,都会被原样尊重。
|
||||
timeoutCtx, cancel := withLLMTimeout(ctx, defaultChatTimeout)
|
||||
defer cancel()
|
||||
|
||||
req := c.BuildChatCompletionRequest(messages, opts, false)
|
||||
@@ -493,11 +517,9 @@ func removeThinkingContent(content string) string {
|
||||
|
||||
// ChatStream 进行流式聊天
|
||||
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
// Enforce a per-call timeout to prevent a hung stream from blocking the worker.
|
||||
// Uses a longer timeout than Chat because streaming responses (especially with
|
||||
// thinking/reasoning) can take minutes to complete.
|
||||
// If the caller has already set a shorter deadline, WithTimeout respects it.
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, defaultStreamTimeout)
|
||||
// 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长,
|
||||
// 因为带思考/推理的模型可能数十秒甚至几分钟才产出首 token。
|
||||
timeoutCtx, cancel := withLLMTimeout(ctx, defaultStreamTimeout)
|
||||
|
||||
req := c.BuildChatCompletionRequest(messages, opts, true)
|
||||
|
||||
@@ -510,14 +532,14 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
if c.requestCustomizer != nil {
|
||||
customReq, useRawHTTP := c.requestCustomizer(&req, opts, true)
|
||||
if useRawHTTP && customReq != nil {
|
||||
// chatStreamWithRawHTTP starts its own goroutine; the timeout context
|
||||
// will be cancelled when the deadline expires, providing the safety net.
|
||||
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, customReq)
|
||||
ch, err := c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, customReq)
|
||||
return wrapStreamCancel(ch, err, cancel)
|
||||
}
|
||||
}
|
||||
// 使用自定义请求地址
|
||||
if customEndpoint != "" {
|
||||
return c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, &req)
|
||||
ch, err := c.chatStreamWithRawHTTP(timeoutCtx, customEndpoint, &req)
|
||||
return wrapStreamCancel(ch, err, cancel)
|
||||
}
|
||||
c.logRequest(timeoutCtx, req, true)
|
||||
|
||||
@@ -546,6 +568,24 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
|
||||
return streamChan, nil
|
||||
}
|
||||
|
||||
// wrapStreamCancel 在子 channel 关闭后执行 cancel,避免 timeout context 泄漏。
|
||||
// 当底层调用直接返回 error 时,立即调用 cancel 并将 error 透出。
|
||||
func wrapStreamCancel(in <-chan types.StreamResponse, err error, cancel context.CancelFunc) (<-chan types.StreamResponse, error) {
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer cancel()
|
||||
defer close(out)
|
||||
for v := range in {
|
||||
out <- v
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// chatStreamWithRawHTTP 使用原始 HTTP 请求进行流式聊天
|
||||
func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint string, customReq any) (<-chan types.StreamResponse, error) {
|
||||
jsonData, err := json.Marshal(customReq)
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWithLLMTimeout_NoParentDeadline_AppliesDefault(t *testing.T) {
|
||||
ctx, cancel := withLLMTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
dl, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatalf("expected deadline to be set when parent has none")
|
||||
}
|
||||
if remaining := time.Until(dl); remaining <= 0 || remaining > 50*time.Millisecond {
|
||||
t.Fatalf("unexpected remaining duration: %v", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithLLMTimeout_ShorterParentDeadline_Respected(t *testing.T) {
|
||||
parent, parentCancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer parentCancel()
|
||||
|
||||
ctx, cancel := withLLMTimeout(parent, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
dl, _ := ctx.Deadline()
|
||||
if remaining := time.Until(dl); remaining > 30*time.Millisecond {
|
||||
t.Fatalf("parent shorter deadline should be respected, got remaining=%v", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithLLMTimeout_LongerParentDeadline_NotTruncated(t *testing.T) {
|
||||
parent, parentCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer parentCancel()
|
||||
|
||||
ctx, cancel := withLLMTimeout(parent, 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
dl, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatalf("deadline must be set")
|
||||
}
|
||||
if remaining := time.Until(dl); remaining < 5*time.Second {
|
||||
t.Fatalf("parent longer deadline should NOT be truncated by default, got remaining=%v", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvDurationSeconds(t *testing.T) {
|
||||
const key = "WEKNORA_TEST_TIMEOUT_SECONDS"
|
||||
|
||||
t.Run("unset returns fallback", func(t *testing.T) {
|
||||
os.Unsetenv(key)
|
||||
if got := envDurationSeconds(key, 7*time.Second); got != 7*time.Second {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid value parsed", func(t *testing.T) {
|
||||
t.Setenv(key, "42")
|
||||
if got := envDurationSeconds(key, 1*time.Second); got != 42*time.Second {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid falls back", func(t *testing.T) {
|
||||
t.Setenv(key, "not-a-number")
|
||||
if got := envDurationSeconds(key, 9*time.Second); got != 9*time.Second {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-positive falls back", func(t *testing.T) {
|
||||
t.Setenv(key, "0")
|
||||
if got := envDurationSeconds(key, 9*time.Second); got != 9*time.Second {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
t.Setenv(key, "-5")
|
||||
if got := envDurationSeconds(key, 9*time.Second); got != 9*time.Second {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user