fix(models): harden background concurrency governor
Follow-up hardening on the per-model background concurrency governor: - limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the lease score, so a semaphore that stays saturated with no slot turnover can't let the whole key expire (which would drop every lease and admit over the limit); reuse one poll timer instead of allocating time.After each iteration under contention. - governor: extract the process-wide governor + Gate helper into the limiter package so chat and vlm share one limiter/limit without importing each other; add an in-process localLimiter for Lite mode. - vlm: gate background multimodal (VLM) calls through the same governor; previously only chat was throttled. - chat: ChatStream releases its slot when the consumer abandons the stream and cancels ctx (select on ctx.Done, drain inner), fixing a slot/goroutine leak. - container/sync_task: install the local governor in Lite mode and tag SyncTaskExecutor runs as background; honour model.max_concurrency=0 as an explicit disable instead of falling back to the default. - tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
@@ -2,7 +2,6 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -18,29 +17,8 @@ import (
|
||||
// Only background (asynq worker) calls are throttled; interactive chat is left
|
||||
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
|
||||
// exhaust the provider yet user-facing latency is never gated behind the
|
||||
// semaphore.
|
||||
var (
|
||||
concurrencyMu sync.RWMutex
|
||||
concurrencyLimiter limiter.ModelConcurrencyLimiter
|
||||
concurrencyLimit int
|
||||
)
|
||||
|
||||
// SetConcurrencyLimiter installs the process-wide background chat concurrency
|
||||
// governor and the default per-model limit. Called once at startup (see
|
||||
// container wiring). Passing a nil limiter or a non-positive limit disables
|
||||
// governance (all calls pass through).
|
||||
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
|
||||
concurrencyMu.Lock()
|
||||
defer concurrencyMu.Unlock()
|
||||
concurrencyLimiter = l
|
||||
concurrencyLimit = defaultLimit
|
||||
}
|
||||
|
||||
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
|
||||
concurrencyMu.RLock()
|
||||
defer concurrencyMu.RUnlock()
|
||||
return concurrencyLimiter, concurrencyLimit
|
||||
}
|
||||
// semaphore. The governor singleton itself lives in the limiter package so chat
|
||||
// and vlm share the same limiter and per-model budget.
|
||||
|
||||
// concurrencyChat throttles background LLM calls through a per-model
|
||||
// distributed semaphore. It is the outermost wrapper so the slot is held only
|
||||
@@ -53,42 +31,38 @@ type concurrencyChat struct {
|
||||
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
// gate acquires a concurrency slot when the call is a background task and a
|
||||
// limiter is installed. Returns a release func (always safe to call) and
|
||||
// whether the inner call may proceed — it always may; the gate never blocks a
|
||||
// call permanently and fails open on any limiter error.
|
||||
func (w *concurrencyChat) gate(ctx context.Context) func() {
|
||||
lim, limit := getConcurrencyLimiter()
|
||||
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return func() {}
|
||||
}
|
||||
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
|
||||
if err != nil || release == nil {
|
||||
return func() {}
|
||||
}
|
||||
return release
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
defer release()
|
||||
return w.inner.Chat(ctx, messages, opts)
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
||||
if err != nil || ch == nil {
|
||||
release()
|
||||
return ch, err
|
||||
}
|
||||
// Hold the slot until the stream fully drains, then release.
|
||||
// Hold the slot until the stream fully drains, then release. If the
|
||||
// consumer abandons the stream (stops reading out) we would otherwise
|
||||
// block forever on the send and never release the slot; select on
|
||||
// ctx.Done() so a cancelled call frees its slot promptly, and drain the
|
||||
// inner channel in the background so the upstream producer can exit.
|
||||
out := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer release()
|
||||
for resp := range ch {
|
||||
out <- resp
|
||||
select {
|
||||
case out <- resp:
|
||||
case <-ctx.Done():
|
||||
go func() {
|
||||
for range ch {
|
||||
}
|
||||
}()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// fakeChat is a minimal Chat whose stream emits continuously until ctx is done,
|
||||
// so we can exercise the concurrency wrapper's slot lifecycle.
|
||||
type fakeChat struct{ id string }
|
||||
|
||||
func (f *fakeChat) GetModelName() string { return f.id }
|
||||
func (f *fakeChat) GetModelID() string { return f.id }
|
||||
|
||||
func (f *fakeChat) Chat(ctx context.Context, _ []Message, _ *ChatOptions) (*types.ChatResponse, error) {
|
||||
return &types.ChatResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeChat) ChatStream(ctx context.Context, _ []Message, _ *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
ch := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- types.StreamResponse{}:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// TestConcurrencyChatInteractiveNotGated verifies interactive calls bypass the
|
||||
// governor entirely even at limit 1.
|
||||
func TestConcurrencyChatInteractiveNotGated(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
w := &concurrencyChat{inner: &fakeChat{id: "model-x"}}
|
||||
// No background marker: neither call should be throttled.
|
||||
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
|
||||
t.Fatalf("interactive chat: %v", err)
|
||||
}
|
||||
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
|
||||
t.Fatalf("interactive chat 2: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyChatStreamReleasesOnAbandon verifies that when the consumer
|
||||
// stops reading and cancels the context, the held slot is released rather than
|
||||
// leaked (the #9 fix).
|
||||
func TestConcurrencyChatStreamReleasesOnAbandon(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
const id = "model-y"
|
||||
w := &concurrencyChat{inner: &fakeChat{id: id}}
|
||||
|
||||
streamCtx, cancel := context.WithCancel(types.WithBackgroundTask(context.Background()))
|
||||
out, err := w.ChatStream(streamCtx, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("stream: %v", err)
|
||||
}
|
||||
// Consume one item so the relay goroutine is running and holding the slot.
|
||||
<-out
|
||||
|
||||
// The single slot is now held: a background acquire must block.
|
||||
blocked := make(chan struct{})
|
||||
go func() {
|
||||
rel := limiter.Gate(types.WithBackgroundTask(context.Background()), id)
|
||||
rel()
|
||||
close(blocked)
|
||||
}()
|
||||
select {
|
||||
case <-blocked:
|
||||
t.Fatal("slot should be held by the live stream")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Abandon the stream: stop reading and cancel. The relay must release.
|
||||
cancel()
|
||||
select {
|
||||
case <-blocked:
|
||||
// released and reacquired successfully
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("stream slot leaked: not released after consumer abandoned + cancelled")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user