fix(models): harden background concurrency governor

Follow-up hardening on the per-model background concurrency governor:

- limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the
  lease score, so a semaphore that stays saturated with no slot turnover
  can't let the whole key expire (which would drop every lease and admit
  over the limit); reuse one poll timer instead of allocating time.After
  each iteration under contention.
- governor: extract the process-wide governor + Gate helper into the
  limiter package so chat and vlm share one limiter/limit without
  importing each other; add an in-process localLimiter for Lite mode.
- vlm: gate background multimodal (VLM) calls through the same governor;
  previously only chat was throttled.
- chat: ChatStream releases its slot when the consumer abandons the
  stream and cancels ctx (select on ctx.Done, drain inner), fixing a
  slot/goroutine leak.
- container/sync_task: install the local governor in Lite mode and tag
  SyncTaskExecutor runs as background; honour model.max_concurrency=0 as
  an explicit disable instead of falling back to the default.
- tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
wizardchen
2026-07-09 18:30:44 +08:00
committed by lyingbug
parent 63198c5d9f
commit 98ad41cda4
9 changed files with 440 additions and 61 deletions
+18 -44
View File
@@ -2,7 +2,6 @@ package chat
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/types"
@@ -18,29 +17,8 @@ import (
// Only background (asynq worker) calls are throttled; interactive chat is left
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
// exhaust the provider yet user-facing latency is never gated behind the
// semaphore.
var (
concurrencyMu sync.RWMutex
concurrencyLimiter limiter.ModelConcurrencyLimiter
concurrencyLimit int
)
// SetConcurrencyLimiter installs the process-wide background chat concurrency
// governor and the default per-model limit. Called once at startup (see
// container wiring). Passing a nil limiter or a non-positive limit disables
// governance (all calls pass through).
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
concurrencyMu.Lock()
defer concurrencyMu.Unlock()
concurrencyLimiter = l
concurrencyLimit = defaultLimit
}
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
concurrencyMu.RLock()
defer concurrencyMu.RUnlock()
return concurrencyLimiter, concurrencyLimit
}
// semaphore. The governor singleton itself lives in the limiter package so chat
// and vlm share the same limiter and per-model budget.
// concurrencyChat throttles background LLM calls through a per-model
// distributed semaphore. It is the outermost wrapper so the slot is held only
@@ -53,42 +31,38 @@ type concurrencyChat struct {
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
// gate acquires a concurrency slot when the call is a background task and a
// limiter is installed. Returns a release func (always safe to call) and
// whether the inner call may proceed — it always may; the gate never blocks a
// call permanently and fails open on any limiter error.
func (w *concurrencyChat) gate(ctx context.Context) func() {
lim, limit := getConcurrencyLimiter()
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
return func() {}
}
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
if err != nil || release == nil {
return func() {}
}
return release
}
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
release := w.gate(ctx)
release := limiter.Gate(ctx, w.inner.GetModelID())
defer release()
return w.inner.Chat(ctx, messages, opts)
}
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
release := w.gate(ctx)
release := limiter.Gate(ctx, w.inner.GetModelID())
ch, err := w.inner.ChatStream(ctx, messages, opts)
if err != nil || ch == nil {
release()
return ch, err
}
// Hold the slot until the stream fully drains, then release.
// Hold the slot until the stream fully drains, then release. If the
// consumer abandons the stream (stops reading out) we would otherwise
// block forever on the send and never release the slot; select on
// ctx.Done() so a cancelled call frees its slot promptly, and drain the
// inner channel in the background so the upstream producer can exit.
out := make(chan types.StreamResponse)
go func() {
defer close(out)
defer release()
for resp := range ch {
out <- resp
select {
case out <- resp:
case <-ctx.Done():
go func() {
for range ch {
}
}()
return
}
}
}()
return out, nil
@@ -0,0 +1,93 @@
package chat
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/types"
)
// fakeChat is a minimal Chat whose stream emits continuously until ctx is done,
// so we can exercise the concurrency wrapper's slot lifecycle.
type fakeChat struct{ id string }
func (f *fakeChat) GetModelName() string { return f.id }
func (f *fakeChat) GetModelID() string { return f.id }
func (f *fakeChat) Chat(ctx context.Context, _ []Message, _ *ChatOptions) (*types.ChatResponse, error) {
return &types.ChatResponse{}, nil
}
func (f *fakeChat) ChatStream(ctx context.Context, _ []Message, _ *ChatOptions) (<-chan types.StreamResponse, error) {
ch := make(chan types.StreamResponse)
go func() {
defer close(ch)
for {
select {
case <-ctx.Done():
return
case ch <- types.StreamResponse{}:
}
}
}()
return ch, nil
}
// TestConcurrencyChatInteractiveNotGated verifies interactive calls bypass the
// governor entirely even at limit 1.
func TestConcurrencyChatInteractiveNotGated(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
w := &concurrencyChat{inner: &fakeChat{id: "model-x"}}
// No background marker: neither call should be throttled.
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
t.Fatalf("interactive chat: %v", err)
}
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
t.Fatalf("interactive chat 2: %v", err)
}
}
// TestConcurrencyChatStreamReleasesOnAbandon verifies that when the consumer
// stops reading and cancels the context, the held slot is released rather than
// leaked (the #9 fix).
func TestConcurrencyChatStreamReleasesOnAbandon(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
const id = "model-y"
w := &concurrencyChat{inner: &fakeChat{id: id}}
streamCtx, cancel := context.WithCancel(types.WithBackgroundTask(context.Background()))
out, err := w.ChatStream(streamCtx, nil, nil)
if err != nil {
t.Fatalf("stream: %v", err)
}
// Consume one item so the relay goroutine is running and holding the slot.
<-out
// The single slot is now held: a background acquire must block.
blocked := make(chan struct{})
go func() {
rel := limiter.Gate(types.WithBackgroundTask(context.Background()), id)
rel()
close(blocked)
}()
select {
case <-blocked:
t.Fatal("slot should be held by the live stream")
case <-time.After(50 * time.Millisecond):
}
// Abandon the stream: stop reading and cancel. The relay must release.
cancel()
select {
case <-blocked:
// released and reacquired successfully
case <-time.After(2 * time.Second):
t.Fatal("stream slot leaked: not released after consumer abandoned + cancelled")
}
}