Files
Enterprise-Knowledge-Base/internal/models/limiter/governor.go
T
wizardchen 98ad41cda4 fix(models): harden background concurrency governor
Follow-up hardening on the per-model background concurrency governor:

- limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the
  lease score, so a semaphore that stays saturated with no slot turnover
  can't let the whole key expire (which would drop every lease and admit
  over the limit); reuse one poll timer instead of allocating time.After
  each iteration under contention.
- governor: extract the process-wide governor + Gate helper into the
  limiter package so chat and vlm share one limiter/limit without
  importing each other; add an in-process localLimiter for Lite mode.
- vlm: gate background multimodal (VLM) calls through the same governor;
  previously only chat was throttled.
- chat: ChatStream releases its slot when the consumer abandons the
  stream and cancels ctx (select on ctx.Done, drain inner), fixing a
  slot/goroutine leak.
- container/sync_task: install the local governor in Lite mode and tag
  SyncTaskExecutor runs as background; honour model.max_concurrency=0 as
  an explicit disable instead of falling back to the default.
- tests: cover the governor gating, local limiter, and stream release.
2026-07-09 20:40:51 +08:00

90 lines
2.8 KiB
Go

package limiter
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/types"
)
// The concurrency governor is process-wide, shared by every model-client layer
// that fronts a provider (chat, vlm). Keeping the singleton here — rather than
// inside one client package — lets all of them gate against the same limiter
// and per-model limit without importing each other. Wired once at startup (see
// container.registerModelConcurrencyLimiter) via SetGovernor.
var (
governorMu sync.RWMutex
governor ModelConcurrencyLimiter
governorN int
)
// SetGovernor installs the process-wide background concurrency governor and the
// default per-model limit. Passing a nil limiter or a non-positive limit
// disables governance (all calls pass through). Safe to call at startup.
func SetGovernor(l ModelConcurrencyLimiter, limit int) {
governorMu.Lock()
defer governorMu.Unlock()
governor = l
governorN = limit
}
// Gate acquires a per-model concurrency slot when the call is a background task
// (see types.IsBackgroundTask) and a governor is installed. It returns a
// release func that is ALWAYS safe to call: on the passthrough / fail-open
// paths it is a cheap no-op. The gate never blocks a call permanently — a
// limiter/Redis outage or a cancelled context fails open.
func Gate(ctx context.Context, modelID string) func() {
governorMu.RLock()
l, limit := governor, governorN
governorMu.RUnlock()
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
return noop
}
release, err := l.Acquire(ctx, modelID, limit)
if err != nil || release == nil {
return noop
}
return release
}
// localLimiter is an in-process (single-node) counting semaphore keyed by
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
// single process with no Redis, so a shared distributed semaphore is neither
// available nor needed — but background ingestion can still burst the whole
// worker pool against one provider, so we still cap concurrency locally.
type localLimiter struct {
mu sync.Mutex
sems map[string]chan struct{}
}
// NewLocalLimiter builds an in-process per-key concurrency limiter.
func NewLocalLimiter() ModelConcurrencyLimiter {
return &localLimiter{sems: make(map[string]chan struct{})}
}
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
if l == nil || limit <= 0 || key == "" {
return noop, nil
}
l.mu.Lock()
sem, ok := l.sems[key]
if !ok {
// Capacity is fixed at first use for a key; the limit is a
// process-wide constant, so it never changes across acquires.
sem = make(chan struct{}, limit)
l.sems[key] = sem
}
l.mu.Unlock()
select {
case sem <- struct{}{}:
var once sync.Once
return func() { once.Do(func() { <-sem }) }, nil
case <-ctx.Done():
// Fail open on cancellation, mirroring the Redis limiter.
return noop, nil
}
}