fix(models): harden background concurrency governor

Follow-up hardening on the per-model background concurrency governor:

- limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the
  lease score, so a semaphore that stays saturated with no slot turnover
  can't let the whole key expire (which would drop every lease and admit
  over the limit); reuse one poll timer instead of allocating time.After
  each iteration under contention.
- governor: extract the process-wide governor + Gate helper into the
  limiter package so chat and vlm share one limiter/limit without
  importing each other; add an in-process localLimiter for Lite mode.
- vlm: gate background multimodal (VLM) calls through the same governor;
  previously only chat was throttled.
- chat: ChatStream releases its slot when the consumer abandons the
  stream and cancels ctx (select on ctx.Done, drain inner), fixing a
  slot/goroutine leak.
- container/sync_task: install the local governor in Lite mode and tag
  SyncTaskExecutor runs as background; honour model.max_concurrency=0 as
  an explicit disable instead of falling back to the default.
- tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
wizardchen
2026-07-09 18:30:44 +08:00
committed by lyingbug
parent 63198c5d9f
commit 98ad41cda4
9 changed files with 440 additions and 61 deletions
+42 -13
View File
@@ -287,6 +287,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
// dispatches inline goroutines that the checkpoint-based abort
// already handles.
must(container.Provide(router.NewNoopTaskInspector))
// Even without Redis, background ingestion/enrichment can burst the
// worker pool against one provider, so install an in-process governor.
must(container.Invoke(registerLiteModelConcurrencyLimiter))
}
// Chat pipeline components for processing chat requests
@@ -448,21 +451,47 @@ func initLangfuse() (*langfuse.Manager, error) {
// across every replica. Interactive chat is never gated.
const defaultModelMaxConcurrency = 8
// registerModelConcurrencyLimiter builds the Redis-backed per-model chat
// concurrency governor and installs it into the chat package. The limit is
// resolved from system settings / env, defaulting to defaultModelMaxConcurrency.
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
limit := defaultModelMaxConcurrency
if ss != nil {
n := ss.GetInt(context.Background(), "model.max_concurrency",
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency))
if n > 0 {
limit = int(n)
}
// resolveModelMaxConcurrency reads the per-model background concurrency limit
// from system settings / env, defaulting to defaultModelMaxConcurrency when
// unset. A configured value of 0 (or negative) is honoured and disables the
// governor — that is the supported way to turn throttling off via config/env.
func resolveModelMaxConcurrency(ss interfaces.SystemSettingService) int {
if ss == nil {
return defaultModelMaxConcurrency
}
return int(ss.GetInt(context.Background(), "model.max_concurrency",
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency)))
}
// registerModelConcurrencyLimiter builds the Redis-backed per-model background
// concurrency governor (chat + vlm) and installs it. Only available with Redis
// (the shared semaphore backend); Lite mode uses registerLiteModelConcurrencyLimiter.
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
limit := resolveModelMaxConcurrency(ss)
limiter.SetGovernor(limiter.NewRedisLimiter(rdb), limit)
if limit <= 0 {
logger.Infof(context.Background(),
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
return
}
chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit)
logger.Infof(context.Background(),
"[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit)
"[ModelLimiter] background model concurrency governed per-model, limit=%d (distributed via redis)", limit)
}
// registerLiteModelConcurrencyLimiter installs an in-process per-model governor
// for Lite mode (no Redis). Lite runs a single process, so an in-process
// semaphore is sufficient to keep a background ingestion storm from bursting
// the whole worker pool against one provider.
func registerLiteModelConcurrencyLimiter(ss interfaces.SystemSettingService) {
limit := resolveModelMaxConcurrency(ss)
limiter.SetGovernor(limiter.NewLocalLimiter(), limit)
if limit <= 0 {
logger.Infof(context.Background(),
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
return
}
logger.Infof(context.Background(),
"[ModelLimiter] background model concurrency governed per-model, limit=%d (in-process, lite mode)", limit)
}
func initRedisClient() (*redis.Client, error) {
+18 -44
View File
@@ -2,7 +2,6 @@ package chat
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/types"
@@ -18,29 +17,8 @@ import (
// Only background (asynq worker) calls are throttled; interactive chat is left
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
// exhaust the provider yet user-facing latency is never gated behind the
// semaphore.
var (
concurrencyMu sync.RWMutex
concurrencyLimiter limiter.ModelConcurrencyLimiter
concurrencyLimit int
)
// SetConcurrencyLimiter installs the process-wide background chat concurrency
// governor and the default per-model limit. Called once at startup (see
// container wiring). Passing a nil limiter or a non-positive limit disables
// governance (all calls pass through).
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
concurrencyMu.Lock()
defer concurrencyMu.Unlock()
concurrencyLimiter = l
concurrencyLimit = defaultLimit
}
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
concurrencyMu.RLock()
defer concurrencyMu.RUnlock()
return concurrencyLimiter, concurrencyLimit
}
// semaphore. The governor singleton itself lives in the limiter package so chat
// and vlm share the same limiter and per-model budget.
// concurrencyChat throttles background LLM calls through a per-model
// distributed semaphore. It is the outermost wrapper so the slot is held only
@@ -53,42 +31,38 @@ type concurrencyChat struct {
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
// gate acquires a concurrency slot when the call is a background task and a
// limiter is installed. Returns a release func (always safe to call) and
// whether the inner call may proceed — it always may; the gate never blocks a
// call permanently and fails open on any limiter error.
func (w *concurrencyChat) gate(ctx context.Context) func() {
lim, limit := getConcurrencyLimiter()
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
return func() {}
}
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
if err != nil || release == nil {
return func() {}
}
return release
}
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
release := w.gate(ctx)
release := limiter.Gate(ctx, w.inner.GetModelID())
defer release()
return w.inner.Chat(ctx, messages, opts)
}
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
release := w.gate(ctx)
release := limiter.Gate(ctx, w.inner.GetModelID())
ch, err := w.inner.ChatStream(ctx, messages, opts)
if err != nil || ch == nil {
release()
return ch, err
}
// Hold the slot until the stream fully drains, then release.
// Hold the slot until the stream fully drains, then release. If the
// consumer abandons the stream (stops reading out) we would otherwise
// block forever on the send and never release the slot; select on
// ctx.Done() so a cancelled call frees its slot promptly, and drain the
// inner channel in the background so the upstream producer can exit.
out := make(chan types.StreamResponse)
go func() {
defer close(out)
defer release()
for resp := range ch {
out <- resp
select {
case out <- resp:
case <-ctx.Done():
go func() {
for range ch {
}
}()
return
}
}
}()
return out, nil
@@ -0,0 +1,93 @@
package chat
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/types"
)
// fakeChat is a minimal Chat whose stream emits continuously until ctx is done,
// so we can exercise the concurrency wrapper's slot lifecycle.
type fakeChat struct{ id string }
func (f *fakeChat) GetModelName() string { return f.id }
func (f *fakeChat) GetModelID() string { return f.id }
func (f *fakeChat) Chat(ctx context.Context, _ []Message, _ *ChatOptions) (*types.ChatResponse, error) {
return &types.ChatResponse{}, nil
}
func (f *fakeChat) ChatStream(ctx context.Context, _ []Message, _ *ChatOptions) (<-chan types.StreamResponse, error) {
ch := make(chan types.StreamResponse)
go func() {
defer close(ch)
for {
select {
case <-ctx.Done():
return
case ch <- types.StreamResponse{}:
}
}
}()
return ch, nil
}
// TestConcurrencyChatInteractiveNotGated verifies interactive calls bypass the
// governor entirely even at limit 1.
func TestConcurrencyChatInteractiveNotGated(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
w := &concurrencyChat{inner: &fakeChat{id: "model-x"}}
// No background marker: neither call should be throttled.
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
t.Fatalf("interactive chat: %v", err)
}
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
t.Fatalf("interactive chat 2: %v", err)
}
}
// TestConcurrencyChatStreamReleasesOnAbandon verifies that when the consumer
// stops reading and cancels the context, the held slot is released rather than
// leaked (the #9 fix).
func TestConcurrencyChatStreamReleasesOnAbandon(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
const id = "model-y"
w := &concurrencyChat{inner: &fakeChat{id: id}}
streamCtx, cancel := context.WithCancel(types.WithBackgroundTask(context.Background()))
out, err := w.ChatStream(streamCtx, nil, nil)
if err != nil {
t.Fatalf("stream: %v", err)
}
// Consume one item so the relay goroutine is running and holding the slot.
<-out
// The single slot is now held: a background acquire must block.
blocked := make(chan struct{})
go func() {
rel := limiter.Gate(types.WithBackgroundTask(context.Background()), id)
rel()
close(blocked)
}()
select {
case <-blocked:
t.Fatal("slot should be held by the live stream")
case <-time.After(50 * time.Millisecond):
}
// Abandon the stream: stop reading and cancel. The relay must release.
cancel()
select {
case <-blocked:
// released and reacquired successfully
case <-time.After(2 * time.Second):
t.Fatal("stream slot leaked: not released after consumer abandoned + cancelled")
}
}
+89
View File
@@ -0,0 +1,89 @@
package limiter
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/types"
)
// The concurrency governor is process-wide, shared by every model-client layer
// that fronts a provider (chat, vlm). Keeping the singleton here — rather than
// inside one client package — lets all of them gate against the same limiter
// and per-model limit without importing each other. Wired once at startup (see
// container.registerModelConcurrencyLimiter) via SetGovernor.
var (
governorMu sync.RWMutex
governor ModelConcurrencyLimiter
governorN int
)
// SetGovernor installs the process-wide background concurrency governor and the
// default per-model limit. Passing a nil limiter or a non-positive limit
// disables governance (all calls pass through). Safe to call at startup.
func SetGovernor(l ModelConcurrencyLimiter, limit int) {
governorMu.Lock()
defer governorMu.Unlock()
governor = l
governorN = limit
}
// Gate acquires a per-model concurrency slot when the call is a background task
// (see types.IsBackgroundTask) and a governor is installed. It returns a
// release func that is ALWAYS safe to call: on the passthrough / fail-open
// paths it is a cheap no-op. The gate never blocks a call permanently — a
// limiter/Redis outage or a cancelled context fails open.
func Gate(ctx context.Context, modelID string) func() {
governorMu.RLock()
l, limit := governor, governorN
governorMu.RUnlock()
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
return noop
}
release, err := l.Acquire(ctx, modelID, limit)
if err != nil || release == nil {
return noop
}
return release
}
// localLimiter is an in-process (single-node) counting semaphore keyed by
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
// single process with no Redis, so a shared distributed semaphore is neither
// available nor needed — but background ingestion can still burst the whole
// worker pool against one provider, so we still cap concurrency locally.
type localLimiter struct {
mu sync.Mutex
sems map[string]chan struct{}
}
// NewLocalLimiter builds an in-process per-key concurrency limiter.
func NewLocalLimiter() ModelConcurrencyLimiter {
return &localLimiter{sems: make(map[string]chan struct{})}
}
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
if l == nil || limit <= 0 || key == "" {
return noop, nil
}
l.mu.Lock()
sem, ok := l.sems[key]
if !ok {
// Capacity is fixed at first use for a key; the limit is a
// process-wide constant, so it never changes across acquires.
sem = make(chan struct{}, limit)
l.sems[key] = sem
}
l.mu.Unlock()
select {
case sem <- struct{}{}:
var once sync.Once
return func() { once.Do(func() { <-sem }) }, nil
case <-ctx.Done():
// Fail open on cancellation, mirroring the Redis limiter.
return noop, nil
}
}
+133
View File
@@ -0,0 +1,133 @@
package limiter
import (
"context"
"sync"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
func withBackground() context.Context {
return types.WithBackgroundTask(context.Background())
}
// TestGateOnlyGovernsBackground verifies interactive calls always pass through
// (no-op release) and only background calls consult the installed limiter.
func TestGateOnlyGovernsBackground(t *testing.T) {
t.Cleanup(func() { SetGovernor(nil, 0) })
SetGovernor(NewLocalLimiter(), 1)
// Interactive (non-background) ctx: never gated, even at limit 1.
rel1 := Gate(context.Background(), "m")
rel2 := Gate(context.Background(), "m")
rel1()
rel2()
// Background ctx: gated. First slot taken; second would block, so a
// cancelled second acquire must fail open (return a usable release).
held := Gate(withBackground(), "m")
defer held()
ctx, cancel := context.WithTimeout(withBackground(), 30*time.Millisecond)
defer cancel()
rel := Gate(ctx, "m")
if rel == nil {
t.Fatal("cancelled background gate must fail open with a usable release")
}
rel()
}
// TestGateDisabledWhenNoGovernor verifies Gate is a passthrough when no
// governor is installed.
func TestGateDisabledWhenNoGovernor(t *testing.T) {
SetGovernor(nil, 0)
if rel := Gate(withBackground(), "m"); rel == nil {
t.Fatal("gate with no governor must return a usable release")
} else {
rel()
}
}
// TestLocalLimiterCaps verifies the in-process limiter enforces the cap and
// releases slots.
func TestLocalLimiterCaps(t *testing.T) {
l := NewLocalLimiter()
ctx := context.Background()
r1, _ := l.Acquire(ctx, "k", 2)
r2, _ := l.Acquire(ctx, "k", 2)
// Third acquire must block while full.
done := make(chan func(), 1)
go func() {
r3, _ := l.Acquire(ctx, "k", 2)
done <- r3
}()
select {
case <-done:
t.Fatal("third acquire should block while at limit")
case <-time.After(50 * time.Millisecond):
}
r1() // free a slot; the waiter proceeds
select {
case r3 := <-done:
r3()
case <-time.After(time.Second):
t.Fatal("waiter should proceed after a slot frees")
}
r2()
}
// TestLocalLimiterIndependentKeys verifies per-key budgets are independent.
func TestLocalLimiterIndependentKeys(t *testing.T) {
l := NewLocalLimiter()
ctx := context.Background()
ra, _ := l.Acquire(ctx, "a", 1)
rb, _ := l.Acquire(ctx, "b", 1) // different key: must not block
ra()
rb()
}
// TestLocalLimiterFailOpen verifies degraded inputs pass through.
func TestLocalLimiterFailOpen(t *testing.T) {
l := NewLocalLimiter()
ctx := context.Background()
if r, err := l.Acquire(ctx, "", 4); err != nil || r == nil {
t.Fatal("empty key should fail open")
} else {
r()
}
if r, err := l.Acquire(ctx, "k", 0); err != nil || r == nil {
t.Fatal("limit<=0 should fail open")
} else {
r()
}
}
// TestLocalLimiterReleaseIdempotent verifies double release doesn't
// over-free a slot.
func TestLocalLimiterReleaseIdempotent(t *testing.T) {
l := NewLocalLimiter()
ctx := context.Background()
r1, _ := l.Acquire(ctx, "k", 1)
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); r1() }()
go func() { defer wg.Done(); r1() }()
wg.Wait()
// After idempotent release the single slot is free again.
r2, _ := l.Acquire(ctx, "k", 1)
done := make(chan struct{})
go func() { r3, _ := l.Acquire(ctx, "k", 1); r3(); close(done) }()
select {
case <-done:
t.Fatal("a double-release must not leave the slot permanently free")
case <-time.After(50 * time.Millisecond):
}
r2()
<-done
}
+22 -2
View File
@@ -96,6 +96,16 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
token := uuid.NewString()
ttlMs := l.ttl.Milliseconds()
// Reuse a single timer across poll iterations rather than allocating a new
// one via time.After each loop: under sustained contention a waiter can
// spin thousands of times, and every time.After timer lives until it fires.
// Start it stopped so the first Reset below arms it cleanly.
timer := time.NewTimer(0)
if !timer.Stop() {
<-timer.C
}
defer timer.Stop()
for {
now := time.Now().UnixMilli()
res, err := acquireScript.Run(ctx, l.rdb, []string{zkey},
@@ -109,13 +119,14 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
return l.hold(zkey, token), nil
}
timer.Reset(l.pollInterval)
select {
case <-ctx.Done():
// Fail open on cancellation too: let the inner call observe the
// cancelled context and return its own error, rather than us
// synthesising one here.
return noop, nil
case <-time.After(l.pollInterval):
case <-timer.C:
}
}
}
@@ -136,10 +147,19 @@ func (l *redisLimiter) hold(zkey, token string) func() {
// Detached ctx: the heartbeat must outlive request ctx up to
// release. Best-effort; a failed refresh just risks early
// reclamation, which the limit already tolerates.
_ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{
//
// Refresh BOTH the member lease score AND the ZSET key's own
// TTL. The acquire script only PEXPIREs the key on admission,
// so a semaphore that stays saturated with no slot turnover
// would otherwise let the whole key expire after ttl*2 —
// dropping every live lease and admitting over the limit. The
// heartbeat pushes the key TTL out in lockstep with the lease.
bg := context.Background()
_ = l.rdb.ZAdd(bg, zkey, redis.Z{
Score: float64(now + l.ttl.Milliseconds()),
Member: token,
}).Err()
_ = l.rdb.PExpire(bg, zkey, l.ttl*2).Err()
}
}
}()
@@ -0,0 +1,35 @@
package vlm
import (
"context"
"github.com/Tencent/WeKnora/internal/models/limiter"
)
// Multimodal enrichment (image OCR / caption) is a high-volume, slow background
// stage that hits the same provider budget as chat. Like chat, it must be
// governed at the client layer so an image-heavy ingestion storm can't burst
// the whole worker pool against one VLM provider. Only background (asynq
// worker) calls are throttled — see limiter.Gate / types.IsBackgroundTask.
type concurrencyVLM struct {
inner VLM
}
func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
release := limiter.Gate(ctx, w.inner.GetModelID())
defer release()
return w.inner.Predict(ctx, imgBytes, prompt)
}
// wrapVLMConcurrency installs the background concurrency governor as the
// outermost VLM decorator. Always applied; a cheap passthrough when no limiter
// is installed or the call is interactive.
func wrapVLMConcurrency(v VLM, err error) (VLM, error) {
if err != nil || v == nil {
return v, err
}
return &concurrencyVLM{inner: v}, nil
}
+4 -1
View File
@@ -87,7 +87,10 @@ func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
if logger.LLMDebugEnabled() {
v = &debugVLM{inner: v}
}
return wrapVLMLangfuse(v, nil)
v, err = wrapVLMLangfuse(v, nil)
// Outermost: hold the per-model concurrency slot only around the real
// provider round-trip, so the wait is excluded from debug/langfuse timing.
return wrapVLMConcurrency(v, err)
}
func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
+4 -1
View File
@@ -80,7 +80,10 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asy
time.Sleep(delay)
}
ctx := context.Background()
// Tag as a background worker execution so the per-model concurrency
// governor throttles Lite-mode ingestion/enrichment LLM calls, mirroring
// the asynq backgroundTaskMiddleware in the Redis path.
ctx := types.WithBackgroundTask(context.Background())
start := time.Now()
logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID)