feat(system): expose model concurrency stats on runtime queues dashboard

Surface per-model active/waiting/limit from the background task limiter so admins can distinguish task scheduling from model-service throttling, with full i18n coverage.
This commit is contained in:
wizardchen
2026-07-11 07:14:35 +08:00
committed by lyingbug
parent 80556fbaa7
commit b9cccc7bee
12 changed files with 286 additions and 22 deletions
+10
View File
@@ -509,6 +509,14 @@ export interface RuntimeWorkerPool {
queue_count: number
}
export interface ModelRuntimeStat {
model_id: string
name: string
active: number
waiting: number
limit: number
}
/**
* Runtime queue dashboard payload. `available` is false in Lite mode
* (no Redis/asynq) — render an "unavailable in this deployment" state
@@ -523,6 +531,8 @@ export interface RuntimeQueuesResponse {
wiki_concurrency: number
pools: RuntimeWorkerPool[]
queues: QueueStat[]
model_limiter_available: boolean
models: ModelRuntimeStat[]
timestamp: number
}
+10
View File
@@ -3506,6 +3506,16 @@ export default {
attention: "Attention",
paused: "Paused",
},
models: {
title: "Model concurrency",
description: "Actual model-service concurrency from background tasks. The section above schedules tasks; this section shows the separate model-service throttle stage.",
scope: "Active is cluster-wide · waiting is local",
disabled: "Background model concurrency governance is disabled. Configure the default model concurrency limit in Global settings.",
empty: "No model activity yet. A model appears after its first background call.",
backgroundOnly: "Background tasks only; interactive chat is excluded",
columns: { model: "Model ID", active: "In flight", waiting: "Throttled", usage: "Concurrency" },
status: { queued: "Throttling", full: "At limit" },
},
pools: {
core: "Core parsing",
enrichment: "Enrichment",
+10
View File
@@ -2511,6 +2511,16 @@ export default {
attention: "확인 필요",
paused: "일시중지됨",
},
models: {
title: "모델 동시성",
description: "백그라운드 작업이 실제로 모델 서비스에 진입할 때의 동시성입니다. 위 섹션은 작업 스케줄링이고, 이 섹션은 별도의 모델 서비스 스로틀 단계입니다.",
scope: "실행 중은 클러스터 전체 · 대기는 로컬",
disabled: "백그라운드 모델 동시성 제어가 비활성화되어 있습니다. 전역 설정에서 모델 기본 동시성 상한을 구성하세요.",
empty: "아직 모델 활동이 없습니다. 첫 백그라운드 호출 후 모델이 표시됩니다.",
backgroundOnly: "백그라운드 작업만; 대화형 채팅은 제외",
columns: { model: "모델 ID", active: "실행 중", waiting: "스로틀 대기", usage: "동시성" },
status: { queued: "스로틀 중", full: "한도 도달" },
},
pools: {
core: "핵심 파싱",
enrichment: "콘텐츠 보강",
+10
View File
@@ -2209,6 +2209,16 @@ export default {
attention: "Требует внимания",
paused: "Приостановлено",
},
models: {
title: "Параллелизм моделей",
description: "Фактический параллелизм вызовов модели из фоновых задач. Раздел выше планирует задачи; этот раздел показывает отдельный этап ограничения на стороне модели.",
scope: "Активные — по всему кластеру · ожидающие — локально",
disabled: "Управление фоновым параллелизмом моделей отключено. Настройте лимит по умолчанию в глобальных настройках.",
empty: "Пока нет данных о вызовах моделей. Модель появится после первого фонового вызова.",
backgroundOnly: "Только фоновые задачи; интерактивный чат не учитывается",
columns: { model: "ID модели", active: "В работе", waiting: "Ожидание ограничения", usage: "Параллелизм" },
status: { queued: "Ограничение", full: "На лимите" },
},
pools: {
core: "Основной разбор",
enrichment: "Обогащение",
+10
View File
@@ -2514,6 +2514,16 @@ export default {
attention: "需关注",
paused: "已暂停",
},
models: {
title: "模型并发占用",
description: "观察后台任务实际进入模型服务时的并发占用;上方是任务调度,这里是模型服务限流,两者处于不同处理阶段。",
scope: "占用为集群全局 · 等待为当前实例",
disabled: "模型后台并发治理未启用。可在全局设置中配置模型默认并发上限。",
empty: "暂无模型调用数据;模型首次执行后台任务后会出现在这里。",
backgroundOnly: "仅统计后台任务,不包含交互式对话",
columns: { model: "模型 ID", active: "调用中", waiting: "限流等待", usage: "并发用量" },
status: { queued: "限流中", full: "已满载" },
},
pools: {
core: "核心解析",
enrichment: "内容富化",
+80 -2
View File
@@ -59,7 +59,7 @@
</t-button>
</div>
<div v-else-if="!available" class="rq-state">
<div v-else-if="!available && !modelLimiterAvailable" class="rq-state">
<div class="rq-state-icon"><t-icon name="info-circle" size="24px" /></div>
<div class="rq-state-copy">
<strong>{{ t('system.globalSettings.runtime.unavailableTitle') }}</strong>
@@ -68,6 +68,7 @@
</div>
<template v-else>
<template v-if="available">
<section class="rq-overview" :aria-label="t('system.globalSettings.runtime.summary.title')">
<div class="rq-overview-title">
<span class="rq-overview-mark"><t-icon name="chart-line" /></span>
@@ -181,6 +182,46 @@
</t-table>
</div>
</section>
</template>
<section class="rq-details rq-models">
<div class="rq-details-header">
<div>
<h3>{{ t('system.globalSettings.runtime.models.title') }}</h3>
<p>{{ t('system.globalSettings.runtime.models.description') }}</p>
</div>
<span class="rq-pools-note">{{ t('system.globalSettings.runtime.models.scope') }}</span>
</div>
<div v-if="!modelLimiterAvailable" class="rq-empty">
<t-icon name="info-circle" size="28px" />
<span>{{ t('system.globalSettings.runtime.models.disabled') }}</span>
</div>
<div v-else-if="models.length === 0" class="rq-empty">
<t-icon name="server" size="28px" />
<span>{{ t('system.globalSettings.runtime.models.empty') }}</span>
</div>
<div v-else class="data-table-shell rq-table-shell">
<t-table row-key="model_id" :data="models" :columns="modelColumns" size="medium" hover>
<template #model_id="{ row }">
<div class="rq-queue-cell">
<span class="rq-queue-name">{{ row.name || row.model_id }}</span>
<span class="rq-queue-meta">{{ row.name ? row.model_id : t('system.globalSettings.runtime.models.backgroundOnly') }}</span>
</div>
</template>
<template #active="{ row }"><span class="rq-number" :class="{ 'rq-number--active': row.active > 0 }">{{ row.active }}</span></template>
<template #waiting="{ row }"><span class="rq-number" :class="{ 'rq-number--warning': row.waiting > 0 }">{{ row.waiting }}</span></template>
<template #usage="{ row }">
<div class="rq-model-usage">
<t-progress :percentage="modelUsage(row)" size="small" :label="false" />
<span>{{ row.active }} / {{ row.limit }}</span>
</div>
</template>
<template #status="{ row }">
<span class="rq-status" :class="`rq-status--${modelState(row).tone}`"><i />{{ modelState(row).label }}</span>
</template>
</t-table>
</div>
</section>
<p class="rq-footnote">{{ t('system.globalSettings.runtime.footnote') }}</p>
</template>
@@ -190,7 +231,7 @@
<script setup lang="ts">
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { getRuntimeQueues, type QueueStat, type RuntimeWorkerPool } from '@/api/system'
import { getRuntimeQueues, type QueueStat, type RuntimeWorkerPool, type ModelRuntimeStat } from '@/api/system'
const { t, te, locale } = useI18n()
@@ -198,6 +239,8 @@ const POLL_INTERVAL_MS = 5000
const queues = ref<QueueStat[]>([])
const pools = ref<RuntimeWorkerPool[]>([])
const models = ref<ModelRuntimeStat[]>([])
const modelLimiterAvailable = ref(false)
const available = ref(true)
const loading = ref(false)
const loadedOnce = ref(false)
@@ -216,6 +259,24 @@ const columns = computed(() => [
{ colKey: 'latency_ms', title: t('system.globalSettings.runtime.columns.latency'), width: 104, align: 'right' as const },
{ colKey: 'status', title: t('system.globalSettings.runtime.columns.status'), width: 96 },
])
const modelColumns = computed(() => [
{ colKey: 'model_id', title: t('system.globalSettings.runtime.models.columns.model'), minWidth: 240 },
{ colKey: 'active', title: t('system.globalSettings.runtime.models.columns.active'), width: 86, align: 'right' as const },
{ colKey: 'waiting', title: t('system.globalSettings.runtime.models.columns.waiting'), width: 86, align: 'right' as const },
{ colKey: 'usage', title: t('system.globalSettings.runtime.models.columns.usage'), width: 190 },
{ colKey: 'status', title: t('system.globalSettings.runtime.columns.status'), width: 96 },
])
function modelUsage(row: ModelRuntimeStat): number {
return row.limit > 0 ? Math.min(100, Math.round(row.active / row.limit * 100)) : 0
}
function modelState(row: ModelRuntimeStat): { label: string; tone: string } {
if (row.waiting > 0) return { label: t('system.globalSettings.runtime.models.status.queued'), tone: 'attention' }
if (row.active >= row.limit) return { label: t('system.globalSettings.runtime.models.status.full'), tone: 'waiting' }
if (row.active > 0) return { label: t('system.globalSettings.runtime.status.working'), tone: 'working' }
return { label: t('system.globalSettings.runtime.status.idle'), tone: 'idle' }
}
const totalActive = computed(() => queues.value.reduce((s, q) => s + q.active, 0))
const totalPending = computed(() => queues.value.reduce((s, q) => s + q.pending, 0))
@@ -282,6 +343,8 @@ async function load(showSpinner: boolean) {
available.value = resp.available
pools.value = resp.pools || []
queues.value = resp.queues || []
models.value = resp.models || []
modelLimiterAvailable.value = Boolean(resp.model_limiter_available)
updatedAt.value = new Date((resp.timestamp || Date.now() / 1000) * 1000)
.toLocaleTimeString(locale.value, { hour12: false })
error.value = ''
@@ -331,6 +394,21 @@ onUnmounted(() => stopPolling())
color: var(--td-text-color-primary);
}
.rq-models {
margin-top: 40px;
padding-top: 32px;
}
.rq-model-usage {
display: grid;
grid-template-columns: minmax(72px, 1fr) auto;
align-items: center;
gap: 10px;
color: var(--td-text-color-secondary);
font-variant-numeric: tabular-nums;
font-size: 12px;
}
.rq-header {
display: flex;
align-items: flex-start;
+16 -7
View File
@@ -21,6 +21,7 @@ import (
"github.com/Tencent/WeKnora/internal/database"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
modellimiter "github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -1394,13 +1395,15 @@ type RuntimeWorkerPool struct {
}
type RuntimeQueuesResponse struct {
Available bool `json:"available"`
UpstreamConcurrency int `json:"upstream_concurrency"`
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
Pools []RuntimeWorkerPool `json:"pools"`
Queues []types.QueueStat `json:"queues"`
Timestamp int64 `json:"timestamp"`
Available bool `json:"available"`
UpstreamConcurrency int `json:"upstream_concurrency"`
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
Pools []RuntimeWorkerPool `json:"pools"`
Queues []types.QueueStat `json:"queues"`
ModelLimiterAvailable bool `json:"model_limiter_available"`
Models []modellimiter.RuntimeStat `json:"models"`
Timestamp int64 `json:"timestamp"`
}
// GetRuntimeQueues godoc
@@ -1459,6 +1462,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
if resp.Queues == nil {
resp.Queues = []types.QueueStat{}
}
modelStats, modelSupported, modelErr := modellimiter.RuntimeStats(ctx)
if modelErr != nil {
logger.Errorf(ctx, "get model concurrency stats failed: %v", modelErr)
}
resp.ModelLimiterAvailable = modelSupported
resp.Models = modelStats
c.JSON(http.StatusOK, resp)
}
+2 -2
View File
@@ -35,13 +35,13 @@ func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName()
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
defer release()
return w.inner.Chat(ctx, messages, opts)
}
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
ch, err := w.inner.ChatStream(ctx, messages, opts)
if err != nil || ch == nil {
release()
@@ -30,13 +30,13 @@ type concurrencyEmbedder struct {
}
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
defer release()
return w.inner.Embed(ctx, text)
}
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
defer release()
return w.inner.BatchEmbed(ctx, texts)
}
+65 -3
View File
@@ -2,6 +2,7 @@ package limiter
import (
"context"
"sort"
"sync"
"github.com/Tencent/WeKnora/internal/types"
@@ -53,6 +54,10 @@ func Gate(ctx context.Context, modelID string) func() {
// blocks a call permanently — a limiter/Redis outage or a cancelled context
// fails open.
func GateN(ctx context.Context, modelID string, modelLimit int) func() {
return GateNamedN(ctx, modelID, "", modelLimit)
}
func GateNamedN(ctx context.Context, modelID, modelName string, modelLimit int) func() {
governorMu.RLock()
l, defaultLimit := governor, governorN
governorMu.RUnlock()
@@ -64,6 +69,9 @@ func GateN(ctx context.Context, modelID string, modelLimit int) func() {
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
return noop
}
if named, ok := l.(interface{ SetModelName(string, string) }); ok {
named.SetModelName(modelID, modelName)
}
release, err := l.Acquire(ctx, modelID, limit)
if err != nil || release == nil {
return noop
@@ -71,19 +79,38 @@ func GateN(ctx context.Context, modelID string, modelLimit int) func() {
return release
}
// RuntimeStats returns the semaphores observed by this process. Redis-backed
// limiters report cluster-wide active holders; waiting callers remain local to
// this instance by design.
func RuntimeStats(ctx context.Context) ([]RuntimeStat, bool, error) {
governorMu.RLock()
l := governor
governorMu.RUnlock()
inspector, ok := l.(runtimeInspectable)
if !ok || inspector == nil {
return []RuntimeStat{}, false, nil
}
stats, err := inspector.RuntimeStats(ctx)
if stats == nil {
stats = []RuntimeStat{}
}
return stats, true, err
}
// localLimiter is an in-process (single-node) counting semaphore keyed by
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
// single process with no Redis, so a shared distributed semaphore is neither
// available nor needed — but background ingestion can still burst the whole
// worker pool against one provider, so we still cap concurrency locally.
type localLimiter struct {
mu sync.Mutex
sems map[string]chan struct{}
mu sync.Mutex
sems map[string]chan struct{}
tracked map[string]*trackedSemaphore
}
// NewLocalLimiter builds an in-process per-key concurrency limiter.
func NewLocalLimiter() ModelConcurrencyLimiter {
return &localLimiter{sems: make(map[string]chan struct{})}
return &localLimiter{sems: make(map[string]chan struct{}), tracked: make(map[string]*trackedSemaphore)}
}
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
@@ -93,6 +120,12 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
l.mu.Lock()
sem, ok := l.sems[key]
tracked := l.tracked[key]
if tracked == nil {
tracked = &trackedSemaphore{}
l.tracked[key] = tracked
}
tracked.limit.Store(int64(limit))
if !ok {
// Capacity is fixed at first use for a key; the limit is a
// process-wide constant, so it never changes across acquires.
@@ -100,6 +133,8 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
l.sems[key] = sem
}
l.mu.Unlock()
tracked.waiting.Add(1)
defer tracked.waiting.Add(-1)
select {
case sem <- struct{}{}:
@@ -110,3 +145,30 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
return noop, nil
}
}
func (l *localLimiter) RuntimeStats(_ context.Context) ([]RuntimeStat, error) {
l.mu.Lock()
defer l.mu.Unlock()
stats := make([]RuntimeStat, 0, len(l.sems))
for modelID, sem := range l.sems {
tracked := l.tracked[modelID]
name, _ := tracked.name.Load().(string)
stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: int64(len(sem)), Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
}
sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
return stats, nil
}
func (l *localLimiter) SetModelName(modelID, name string) {
if modelID == "" || name == "" {
return
}
l.mu.Lock()
tracked := l.tracked[modelID]
if tracked == nil {
tracked = &trackedSemaphore{}
l.tracked[modelID] = tracked
}
l.mu.Unlock()
tracked.name.Store(name)
}
+70 -5
View File
@@ -15,7 +15,10 @@ package limiter
import (
"context"
"sort"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/Tencent/WeKnora/internal/logger"
@@ -34,15 +37,38 @@ type ModelConcurrencyLimiter interface {
Acquire(ctx context.Context, key string, limit int) (release func(), err error)
}
// RuntimeStat is a point-in-time view of a model semaphore. Active is
// cluster-wide for the Redis backend and process-local in Lite mode. Waiting is
// deliberately process-local: waiters block in application processes and are
// not represented in Redis.
type RuntimeStat struct {
ModelID string `json:"model_id"`
Name string `json:"name"`
Active int64 `json:"active"`
Waiting int64 `json:"waiting"`
Limit int `json:"limit"`
}
type runtimeInspectable interface {
RuntimeStats(context.Context) ([]RuntimeStat, error)
}
type trackedSemaphore struct {
limit atomic.Int64
waiting atomic.Int64
name atomic.Value // string
}
// noop is the release returned on the fail-open / passthrough paths.
func noop() {}
const (
// defaultLeaseTTL is how long a held slot survives without a heartbeat
// before another acquirer may reclaim it. Sized well above a typical
// enrichment LLM call so a slow provider never loses its slot mid-call;
// the heartbeat refreshes it for genuinely long calls.
defaultLeaseTTL = 5 * time.Minute
// defaultLeaseTTL is the crash-recovery window, not a request timeout.
// Live calls refresh their lease every ttl/3, so even a very long provider
// request keeps its slot. Keeping this short prevents an app/container
// restart from leaving an entire model budget apparently occupied for
// minutes while the replacement workers are blocked behind dead holders.
defaultLeaseTTL = 30 * time.Second
// defaultPollInterval is how often a waiting acquirer re-checks for a free
// slot. Small enough to stay responsive, large enough to avoid hammering
// Redis under contention.
@@ -75,6 +101,7 @@ type redisLimiter struct {
rdb *redis.Client
ttl time.Duration
pollInterval time.Duration
tracked sync.Map // model ID -> *trackedSemaphore
}
// NewRedisLimiter builds a distributed limiter backed by rdb. A nil client
@@ -93,6 +120,11 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
}
zkey := keyPrefix + key
entry, _ := l.tracked.LoadOrStore(key, &trackedSemaphore{})
tracked := entry.(*trackedSemaphore)
tracked.limit.Store(int64(limit))
tracked.waiting.Add(1)
defer tracked.waiting.Add(-1)
token := uuid.NewString()
ttlMs := l.ttl.Milliseconds()
@@ -131,6 +163,39 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
}
}
func (l *redisLimiter) RuntimeStats(ctx context.Context) ([]RuntimeStat, error) {
stats := make([]RuntimeStat, 0)
if l == nil || l.rdb == nil {
return stats, nil
}
var firstErr error
now := time.Now().UnixMilli()
l.tracked.Range(func(rawKey, rawValue any) bool {
modelID := rawKey.(string)
tracked := rawValue.(*trackedSemaphore)
active, err := l.rdb.ZCount(ctx, keyPrefix+modelID, strconv.FormatInt(now+1, 10), "+inf").Result()
if err != nil {
if firstErr == nil {
firstErr = err
}
return true
}
name, _ := tracked.name.Load().(string)
stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: active, Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
return true
})
sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
return stats, firstErr
}
func (l *redisLimiter) SetModelName(modelID, name string) {
if modelID == "" || name == "" {
return
}
entry, _ := l.tracked.LoadOrStore(modelID, &trackedSemaphore{})
entry.(*trackedSemaphore).name.Store(name)
}
// hold starts a heartbeat that refreshes the lease and returns an idempotent
// release that stops the heartbeat and drops the slot.
func (l *redisLimiter) hold(zkey, token string) func() {
+1 -1
View File
@@ -22,7 +22,7 @@ func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
defer release()
return w.inner.Predict(ctx, imgBytes, prompt)
}