fix(models): harden background concurrency governor
Follow-up hardening on the per-model background concurrency governor: - limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the lease score, so a semaphore that stays saturated with no slot turnover can't let the whole key expire (which would drop every lease and admit over the limit); reuse one poll timer instead of allocating time.After each iteration under contention. - governor: extract the process-wide governor + Gate helper into the limiter package so chat and vlm share one limiter/limit without importing each other; add an in-process localLimiter for Lite mode. - vlm: gate background multimodal (VLM) calls through the same governor; previously only chat was throttled. - chat: ChatStream releases its slot when the consumer abandons the stream and cancels ctx (select on ctx.Done, drain inner), fixing a slot/goroutine leak. - container/sync_task: install the local governor in Lite mode and tag SyncTaskExecutor runs as background; honour model.max_concurrency=0 as an explicit disable instead of falling back to the default. - tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// The concurrency governor is process-wide, shared by every model-client layer
|
||||
// that fronts a provider (chat, vlm). Keeping the singleton here — rather than
|
||||
// inside one client package — lets all of them gate against the same limiter
|
||||
// and per-model limit without importing each other. Wired once at startup (see
|
||||
// container.registerModelConcurrencyLimiter) via SetGovernor.
|
||||
var (
|
||||
governorMu sync.RWMutex
|
||||
governor ModelConcurrencyLimiter
|
||||
governorN int
|
||||
)
|
||||
|
||||
// SetGovernor installs the process-wide background concurrency governor and the
|
||||
// default per-model limit. Passing a nil limiter or a non-positive limit
|
||||
// disables governance (all calls pass through). Safe to call at startup.
|
||||
func SetGovernor(l ModelConcurrencyLimiter, limit int) {
|
||||
governorMu.Lock()
|
||||
defer governorMu.Unlock()
|
||||
governor = l
|
||||
governorN = limit
|
||||
}
|
||||
|
||||
// Gate acquires a per-model concurrency slot when the call is a background task
|
||||
// (see types.IsBackgroundTask) and a governor is installed. It returns a
|
||||
// release func that is ALWAYS safe to call: on the passthrough / fail-open
|
||||
// paths it is a cheap no-op. The gate never blocks a call permanently — a
|
||||
// limiter/Redis outage or a cancelled context fails open.
|
||||
func Gate(ctx context.Context, modelID string) func() {
|
||||
governorMu.RLock()
|
||||
l, limit := governor, governorN
|
||||
governorMu.RUnlock()
|
||||
|
||||
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return noop
|
||||
}
|
||||
release, err := l.Acquire(ctx, modelID, limit)
|
||||
if err != nil || release == nil {
|
||||
return noop
|
||||
}
|
||||
return release
|
||||
}
|
||||
|
||||
// localLimiter is an in-process (single-node) counting semaphore keyed by
|
||||
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
|
||||
// single process with no Redis, so a shared distributed semaphore is neither
|
||||
// available nor needed — but background ingestion can still burst the whole
|
||||
// worker pool against one provider, so we still cap concurrency locally.
|
||||
type localLimiter struct {
|
||||
mu sync.Mutex
|
||||
sems map[string]chan struct{}
|
||||
}
|
||||
|
||||
// NewLocalLimiter builds an in-process per-key concurrency limiter.
|
||||
func NewLocalLimiter() ModelConcurrencyLimiter {
|
||||
return &localLimiter{sems: make(map[string]chan struct{})}
|
||||
}
|
||||
|
||||
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
|
||||
if l == nil || limit <= 0 || key == "" {
|
||||
return noop, nil
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
sem, ok := l.sems[key]
|
||||
if !ok {
|
||||
// Capacity is fixed at first use for a key; the limit is a
|
||||
// process-wide constant, so it never changes across acquires.
|
||||
sem = make(chan struct{}, limit)
|
||||
l.sems[key] = sem
|
||||
}
|
||||
l.mu.Unlock()
|
||||
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-sem }) }, nil
|
||||
case <-ctx.Done():
|
||||
// Fail open on cancellation, mirroring the Redis limiter.
|
||||
return noop, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func withBackground() context.Context {
|
||||
return types.WithBackgroundTask(context.Background())
|
||||
}
|
||||
|
||||
// TestGateOnlyGovernsBackground verifies interactive calls always pass through
|
||||
// (no-op release) and only background calls consult the installed limiter.
|
||||
func TestGateOnlyGovernsBackground(t *testing.T) {
|
||||
t.Cleanup(func() { SetGovernor(nil, 0) })
|
||||
SetGovernor(NewLocalLimiter(), 1)
|
||||
|
||||
// Interactive (non-background) ctx: never gated, even at limit 1.
|
||||
rel1 := Gate(context.Background(), "m")
|
||||
rel2 := Gate(context.Background(), "m")
|
||||
rel1()
|
||||
rel2()
|
||||
|
||||
// Background ctx: gated. First slot taken; second would block, so a
|
||||
// cancelled second acquire must fail open (return a usable release).
|
||||
held := Gate(withBackground(), "m")
|
||||
defer held()
|
||||
|
||||
ctx, cancel := context.WithTimeout(withBackground(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
rel := Gate(ctx, "m")
|
||||
if rel == nil {
|
||||
t.Fatal("cancelled background gate must fail open with a usable release")
|
||||
}
|
||||
rel()
|
||||
}
|
||||
|
||||
// TestGateDisabledWhenNoGovernor verifies Gate is a passthrough when no
|
||||
// governor is installed.
|
||||
func TestGateDisabledWhenNoGovernor(t *testing.T) {
|
||||
SetGovernor(nil, 0)
|
||||
if rel := Gate(withBackground(), "m"); rel == nil {
|
||||
t.Fatal("gate with no governor must return a usable release")
|
||||
} else {
|
||||
rel()
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLimiterCaps verifies the in-process limiter enforces the cap and
|
||||
// releases slots.
|
||||
func TestLocalLimiterCaps(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
|
||||
r1, _ := l.Acquire(ctx, "k", 2)
|
||||
r2, _ := l.Acquire(ctx, "k", 2)
|
||||
|
||||
// Third acquire must block while full.
|
||||
done := make(chan func(), 1)
|
||||
go func() {
|
||||
r3, _ := l.Acquire(ctx, "k", 2)
|
||||
done <- r3
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("third acquire should block while at limit")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
r1() // free a slot; the waiter proceeds
|
||||
select {
|
||||
case r3 := <-done:
|
||||
r3()
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("waiter should proceed after a slot frees")
|
||||
}
|
||||
r2()
|
||||
}
|
||||
|
||||
// TestLocalLimiterIndependentKeys verifies per-key budgets are independent.
|
||||
func TestLocalLimiterIndependentKeys(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
ra, _ := l.Acquire(ctx, "a", 1)
|
||||
rb, _ := l.Acquire(ctx, "b", 1) // different key: must not block
|
||||
ra()
|
||||
rb()
|
||||
}
|
||||
|
||||
// TestLocalLimiterFailOpen verifies degraded inputs pass through.
|
||||
func TestLocalLimiterFailOpen(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
if r, err := l.Acquire(ctx, "", 4); err != nil || r == nil {
|
||||
t.Fatal("empty key should fail open")
|
||||
} else {
|
||||
r()
|
||||
}
|
||||
if r, err := l.Acquire(ctx, "k", 0); err != nil || r == nil {
|
||||
t.Fatal("limit<=0 should fail open")
|
||||
} else {
|
||||
r()
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLimiterReleaseIdempotent verifies double release doesn't
|
||||
// over-free a slot.
|
||||
func TestLocalLimiterReleaseIdempotent(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
r1, _ := l.Acquire(ctx, "k", 1)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); r1() }()
|
||||
go func() { defer wg.Done(); r1() }()
|
||||
wg.Wait()
|
||||
|
||||
// After idempotent release the single slot is free again.
|
||||
r2, _ := l.Acquire(ctx, "k", 1)
|
||||
done := make(chan struct{})
|
||||
go func() { r3, _ := l.Acquire(ctx, "k", 1); r3(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("a double-release must not leave the slot permanently free")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
r2()
|
||||
<-done
|
||||
}
|
||||
@@ -96,6 +96,16 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
token := uuid.NewString()
|
||||
ttlMs := l.ttl.Milliseconds()
|
||||
|
||||
// Reuse a single timer across poll iterations rather than allocating a new
|
||||
// one via time.After each loop: under sustained contention a waiter can
|
||||
// spin thousands of times, and every time.After timer lives until it fires.
|
||||
// Start it stopped so the first Reset below arms it cleanly.
|
||||
timer := time.NewTimer(0)
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
now := time.Now().UnixMilli()
|
||||
res, err := acquireScript.Run(ctx, l.rdb, []string{zkey},
|
||||
@@ -109,13 +119,14 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
return l.hold(zkey, token), nil
|
||||
}
|
||||
|
||||
timer.Reset(l.pollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Fail open on cancellation too: let the inner call observe the
|
||||
// cancelled context and return its own error, rather than us
|
||||
// synthesising one here.
|
||||
return noop, nil
|
||||
case <-time.After(l.pollInterval):
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,10 +147,19 @@ func (l *redisLimiter) hold(zkey, token string) func() {
|
||||
// Detached ctx: the heartbeat must outlive request ctx up to
|
||||
// release. Best-effort; a failed refresh just risks early
|
||||
// reclamation, which the limit already tolerates.
|
||||
_ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{
|
||||
//
|
||||
// Refresh BOTH the member lease score AND the ZSET key's own
|
||||
// TTL. The acquire script only PEXPIREs the key on admission,
|
||||
// so a semaphore that stays saturated with no slot turnover
|
||||
// would otherwise let the whole key expire after ttl*2 —
|
||||
// dropping every live lease and admitting over the limit. The
|
||||
// heartbeat pushes the key TTL out in lockstep with the lease.
|
||||
bg := context.Background()
|
||||
_ = l.rdb.ZAdd(bg, zkey, redis.Z{
|
||||
Score: float64(now + l.ttl.Milliseconds()),
|
||||
Member: token,
|
||||
}).Err()
|
||||
_ = l.rdb.PExpire(bg, zkey, l.ttl*2).Err()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
Reference in New Issue
Block a user