fix(models): harden background concurrency governor

Follow-up hardening on the per-model background concurrency governor:

- limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the
  lease score, so a semaphore that stays saturated with no slot turnover
  can't let the whole key expire (which would drop every lease and admit
  over the limit); reuse one poll timer instead of allocating time.After
  each iteration under contention.
- governor: extract the process-wide governor + Gate helper into the
  limiter package so chat and vlm share one limiter/limit without
  importing each other; add an in-process localLimiter for Lite mode.
- vlm: gate background multimodal (VLM) calls through the same governor;
  previously only chat was throttled.
- chat: ChatStream releases its slot when the consumer abandons the
  stream and cancels ctx (select on ctx.Done, drain inner), fixing a
  slot/goroutine leak.
- container/sync_task: install the local governor in Lite mode and tag
  SyncTaskExecutor runs as background; honour model.max_concurrency=0 as
  an explicit disable instead of falling back to the default.
- tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
wizardchen
2026-07-09 18:30:44 +08:00
committed by lyingbug
parent 63198c5d9f
commit 98ad41cda4
9 changed files with 440 additions and 61 deletions
+42 -13
View File
@@ -287,6 +287,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
// dispatches inline goroutines that the checkpoint-based abort
// already handles.
must(container.Provide(router.NewNoopTaskInspector))
// Even without Redis, background ingestion/enrichment can burst the
// worker pool against one provider, so install an in-process governor.
must(container.Invoke(registerLiteModelConcurrencyLimiter))
}
// Chat pipeline components for processing chat requests
@@ -448,21 +451,47 @@ func initLangfuse() (*langfuse.Manager, error) {
// across every replica. Interactive chat is never gated.
const defaultModelMaxConcurrency = 8
// registerModelConcurrencyLimiter builds the Redis-backed per-model chat
// concurrency governor and installs it into the chat package. The limit is
// resolved from system settings / env, defaulting to defaultModelMaxConcurrency.
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
limit := defaultModelMaxConcurrency
if ss != nil {
n := ss.GetInt(context.Background(), "model.max_concurrency",
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency))
if n > 0 {
limit = int(n)
}
// resolveModelMaxConcurrency reads the per-model background concurrency limit
// from system settings / env, defaulting to defaultModelMaxConcurrency when
// unset. A configured value of 0 (or negative) is honoured and disables the
// governor — that is the supported way to turn throttling off via config/env.
func resolveModelMaxConcurrency(ss interfaces.SystemSettingService) int {
if ss == nil {
return defaultModelMaxConcurrency
}
return int(ss.GetInt(context.Background(), "model.max_concurrency",
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency)))
}
// registerModelConcurrencyLimiter builds the Redis-backed per-model background
// concurrency governor (chat + vlm) and installs it. Only available with Redis
// (the shared semaphore backend); Lite mode uses registerLiteModelConcurrencyLimiter.
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
limit := resolveModelMaxConcurrency(ss)
limiter.SetGovernor(limiter.NewRedisLimiter(rdb), limit)
if limit <= 0 {
logger.Infof(context.Background(),
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
return
}
chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit)
logger.Infof(context.Background(),
"[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit)
"[ModelLimiter] background model concurrency governed per-model, limit=%d (distributed via redis)", limit)
}
// registerLiteModelConcurrencyLimiter installs an in-process per-model governor
// for Lite mode (no Redis). Lite runs a single process, so an in-process
// semaphore is sufficient to keep a background ingestion storm from bursting
// the whole worker pool against one provider.
func registerLiteModelConcurrencyLimiter(ss interfaces.SystemSettingService) {
limit := resolveModelMaxConcurrency(ss)
limiter.SetGovernor(limiter.NewLocalLimiter(), limit)
if limit <= 0 {
logger.Infof(context.Background(),
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
return
}
logger.Infof(context.Background(),
"[ModelLimiter] background model concurrency governed per-model, limit=%d (in-process, lite mode)", limit)
}
func initRedisClient() (*redis.Client, error) {