63198c5d9f
Background ingestion/enrichment stages (summary, question, graph, multimodal) all target the same chat model, and asynq queue weights are scheduling priority rather than throttling. A large upload could therefore burst up to the entire worker pool of concurrent chat calls against a single provider, unbounded across replicas, with no model-side rate limit. Govern concurrency at the chat-client layer (the one place that sees every task type), keyed by model ID: - internal/models/limiter: Redis ZSET + Lua distributed semaphore with a lease heartbeat, expired-lease reclaim for crashed holders, and fail-open on any backend error or context cancellation. - chat: concurrencyChat decorator (outermost in NewChat) acquires a per-model slot only for background tasks; interactive chat is never gated. - router/task: asynq middleware tags every worker execution as background. - container: wire the Redis limiter with WEKNORA_MODEL_MAX_CONCURRENCY (model.max_concurrency), default 8; enabled only when Redis is present. Tested with miniredis: concurrency cap, expired-lease reclaim, per-key isolation, and all fail-open paths (nil client, backend down, cancelled ctx).
106 lines
3.6 KiB
Go
106 lines
3.6 KiB
Go
package chat
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/models/limiter"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// Model provider budgets are the real bottleneck shared by every LLM-backed
|
|
// background stage (summary / question / graph / multimodal enrichment), which
|
|
// all target the same model. This governor caps concurrent calls per model at
|
|
// the client layer — the one place that sees all task types — instead of at the
|
|
// asynq queue layer, whose weights are scheduling priority rather than
|
|
// throttling.
|
|
//
|
|
// Only background (asynq worker) calls are throttled; interactive chat is left
|
|
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
|
|
// exhaust the provider yet user-facing latency is never gated behind the
|
|
// semaphore.
|
|
var (
|
|
concurrencyMu sync.RWMutex
|
|
concurrencyLimiter limiter.ModelConcurrencyLimiter
|
|
concurrencyLimit int
|
|
)
|
|
|
|
// SetConcurrencyLimiter installs the process-wide background chat concurrency
|
|
// governor and the default per-model limit. Called once at startup (see
|
|
// container wiring). Passing a nil limiter or a non-positive limit disables
|
|
// governance (all calls pass through).
|
|
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
|
|
concurrencyMu.Lock()
|
|
defer concurrencyMu.Unlock()
|
|
concurrencyLimiter = l
|
|
concurrencyLimit = defaultLimit
|
|
}
|
|
|
|
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
|
|
concurrencyMu.RLock()
|
|
defer concurrencyMu.RUnlock()
|
|
return concurrencyLimiter, concurrencyLimit
|
|
}
|
|
|
|
// concurrencyChat throttles background LLM calls through a per-model
|
|
// distributed semaphore. It is the outermost wrapper so the slot is held only
|
|
// around the actual provider round-trip and the wait time is excluded from the
|
|
// inner debug/langfuse timing.
|
|
type concurrencyChat struct {
|
|
inner Chat
|
|
}
|
|
|
|
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
|
|
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
|
|
|
// gate acquires a concurrency slot when the call is a background task and a
|
|
// limiter is installed. Returns a release func (always safe to call) and
|
|
// whether the inner call may proceed — it always may; the gate never blocks a
|
|
// call permanently and fails open on any limiter error.
|
|
func (w *concurrencyChat) gate(ctx context.Context) func() {
|
|
lim, limit := getConcurrencyLimiter()
|
|
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
|
return func() {}
|
|
}
|
|
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
|
|
if err != nil || release == nil {
|
|
return func() {}
|
|
}
|
|
return release
|
|
}
|
|
|
|
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
|
release := w.gate(ctx)
|
|
defer release()
|
|
return w.inner.Chat(ctx, messages, opts)
|
|
}
|
|
|
|
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
|
release := w.gate(ctx)
|
|
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
|
if err != nil || ch == nil {
|
|
release()
|
|
return ch, err
|
|
}
|
|
// Hold the slot until the stream fully drains, then release.
|
|
out := make(chan types.StreamResponse)
|
|
go func() {
|
|
defer close(out)
|
|
defer release()
|
|
for resp := range ch {
|
|
out <- resp
|
|
}
|
|
}()
|
|
return out, nil
|
|
}
|
|
|
|
// wrapChatConcurrency installs the background concurrency governor as the
|
|
// outermost Chat decorator. It is always applied; when no limiter is installed
|
|
// or the call is interactive, the wrapper is a cheap passthrough.
|
|
func wrapChatConcurrency(c Chat, err error) (Chat, error) {
|
|
if err != nil || c == nil {
|
|
return c, err
|
|
}
|
|
return &concurrencyChat{inner: c}, nil
|
|
}
|