feat(system): expose model concurrency stats on runtime queues dashboard
Surface per-model active/waiting/limit from the background task limiter so admins can distinguish task scheduling from model-service throttling, with full i18n coverage.
This commit is contained in:
@@ -509,6 +509,14 @@ export interface RuntimeWorkerPool {
|
||||
queue_count: number
|
||||
}
|
||||
|
||||
export interface ModelRuntimeStat {
|
||||
model_id: string
|
||||
name: string
|
||||
active: number
|
||||
waiting: number
|
||||
limit: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Runtime queue dashboard payload. `available` is false in Lite mode
|
||||
* (no Redis/asynq) — render an "unavailable in this deployment" state
|
||||
@@ -523,6 +531,8 @@ export interface RuntimeQueuesResponse {
|
||||
wiki_concurrency: number
|
||||
pools: RuntimeWorkerPool[]
|
||||
queues: QueueStat[]
|
||||
model_limiter_available: boolean
|
||||
models: ModelRuntimeStat[]
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
|
||||
@@ -3506,6 +3506,16 @@ export default {
|
||||
attention: "Attention",
|
||||
paused: "Paused",
|
||||
},
|
||||
models: {
|
||||
title: "Model concurrency",
|
||||
description: "Actual model-service concurrency from background tasks. The section above schedules tasks; this section shows the separate model-service throttle stage.",
|
||||
scope: "Active is cluster-wide · waiting is local",
|
||||
disabled: "Background model concurrency governance is disabled. Configure the default model concurrency limit in Global settings.",
|
||||
empty: "No model activity yet. A model appears after its first background call.",
|
||||
backgroundOnly: "Background tasks only; interactive chat is excluded",
|
||||
columns: { model: "Model ID", active: "In flight", waiting: "Throttled", usage: "Concurrency" },
|
||||
status: { queued: "Throttling", full: "At limit" },
|
||||
},
|
||||
pools: {
|
||||
core: "Core parsing",
|
||||
enrichment: "Enrichment",
|
||||
|
||||
@@ -2511,6 +2511,16 @@ export default {
|
||||
attention: "확인 필요",
|
||||
paused: "일시중지됨",
|
||||
},
|
||||
models: {
|
||||
title: "모델 동시성",
|
||||
description: "백그라운드 작업이 실제로 모델 서비스에 진입할 때의 동시성입니다. 위 섹션은 작업 스케줄링이고, 이 섹션은 별도의 모델 서비스 스로틀 단계입니다.",
|
||||
scope: "실행 중은 클러스터 전체 · 대기는 로컬",
|
||||
disabled: "백그라운드 모델 동시성 제어가 비활성화되어 있습니다. 전역 설정에서 모델 기본 동시성 상한을 구성하세요.",
|
||||
empty: "아직 모델 활동이 없습니다. 첫 백그라운드 호출 후 모델이 표시됩니다.",
|
||||
backgroundOnly: "백그라운드 작업만; 대화형 채팅은 제외",
|
||||
columns: { model: "모델 ID", active: "실행 중", waiting: "스로틀 대기", usage: "동시성" },
|
||||
status: { queued: "스로틀 중", full: "한도 도달" },
|
||||
},
|
||||
pools: {
|
||||
core: "핵심 파싱",
|
||||
enrichment: "콘텐츠 보강",
|
||||
|
||||
@@ -2209,6 +2209,16 @@ export default {
|
||||
attention: "Требует внимания",
|
||||
paused: "Приостановлено",
|
||||
},
|
||||
models: {
|
||||
title: "Параллелизм моделей",
|
||||
description: "Фактический параллелизм вызовов модели из фоновых задач. Раздел выше планирует задачи; этот раздел показывает отдельный этап ограничения на стороне модели.",
|
||||
scope: "Активные — по всему кластеру · ожидающие — локально",
|
||||
disabled: "Управление фоновым параллелизмом моделей отключено. Настройте лимит по умолчанию в глобальных настройках.",
|
||||
empty: "Пока нет данных о вызовах моделей. Модель появится после первого фонового вызова.",
|
||||
backgroundOnly: "Только фоновые задачи; интерактивный чат не учитывается",
|
||||
columns: { model: "ID модели", active: "В работе", waiting: "Ожидание ограничения", usage: "Параллелизм" },
|
||||
status: { queued: "Ограничение", full: "На лимите" },
|
||||
},
|
||||
pools: {
|
||||
core: "Основной разбор",
|
||||
enrichment: "Обогащение",
|
||||
|
||||
@@ -2514,6 +2514,16 @@ export default {
|
||||
attention: "需关注",
|
||||
paused: "已暂停",
|
||||
},
|
||||
models: {
|
||||
title: "模型并发占用",
|
||||
description: "观察后台任务实际进入模型服务时的并发占用;上方是任务调度,这里是模型服务限流,两者处于不同处理阶段。",
|
||||
scope: "占用为集群全局 · 等待为当前实例",
|
||||
disabled: "模型后台并发治理未启用。可在全局设置中配置模型默认并发上限。",
|
||||
empty: "暂无模型调用数据;模型首次执行后台任务后会出现在这里。",
|
||||
backgroundOnly: "仅统计后台任务,不包含交互式对话",
|
||||
columns: { model: "模型 ID", active: "调用中", waiting: "限流等待", usage: "并发用量" },
|
||||
status: { queued: "限流中", full: "已满载" },
|
||||
},
|
||||
pools: {
|
||||
core: "核心解析",
|
||||
enrichment: "内容富化",
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
</t-button>
|
||||
</div>
|
||||
|
||||
<div v-else-if="!available" class="rq-state">
|
||||
<div v-else-if="!available && !modelLimiterAvailable" class="rq-state">
|
||||
<div class="rq-state-icon"><t-icon name="info-circle" size="24px" /></div>
|
||||
<div class="rq-state-copy">
|
||||
<strong>{{ t('system.globalSettings.runtime.unavailableTitle') }}</strong>
|
||||
@@ -68,6 +68,7 @@
|
||||
</div>
|
||||
|
||||
<template v-else>
|
||||
<template v-if="available">
|
||||
<section class="rq-overview" :aria-label="t('system.globalSettings.runtime.summary.title')">
|
||||
<div class="rq-overview-title">
|
||||
<span class="rq-overview-mark"><t-icon name="chart-line" /></span>
|
||||
@@ -181,6 +182,46 @@
|
||||
</t-table>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<section class="rq-details rq-models">
|
||||
<div class="rq-details-header">
|
||||
<div>
|
||||
<h3>{{ t('system.globalSettings.runtime.models.title') }}</h3>
|
||||
<p>{{ t('system.globalSettings.runtime.models.description') }}</p>
|
||||
</div>
|
||||
<span class="rq-pools-note">{{ t('system.globalSettings.runtime.models.scope') }}</span>
|
||||
</div>
|
||||
<div v-if="!modelLimiterAvailable" class="rq-empty">
|
||||
<t-icon name="info-circle" size="28px" />
|
||||
<span>{{ t('system.globalSettings.runtime.models.disabled') }}</span>
|
||||
</div>
|
||||
<div v-else-if="models.length === 0" class="rq-empty">
|
||||
<t-icon name="server" size="28px" />
|
||||
<span>{{ t('system.globalSettings.runtime.models.empty') }}</span>
|
||||
</div>
|
||||
<div v-else class="data-table-shell rq-table-shell">
|
||||
<t-table row-key="model_id" :data="models" :columns="modelColumns" size="medium" hover>
|
||||
<template #model_id="{ row }">
|
||||
<div class="rq-queue-cell">
|
||||
<span class="rq-queue-name">{{ row.name || row.model_id }}</span>
|
||||
<span class="rq-queue-meta">{{ row.name ? row.model_id : t('system.globalSettings.runtime.models.backgroundOnly') }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<template #active="{ row }"><span class="rq-number" :class="{ 'rq-number--active': row.active > 0 }">{{ row.active }}</span></template>
|
||||
<template #waiting="{ row }"><span class="rq-number" :class="{ 'rq-number--warning': row.waiting > 0 }">{{ row.waiting }}</span></template>
|
||||
<template #usage="{ row }">
|
||||
<div class="rq-model-usage">
|
||||
<t-progress :percentage="modelUsage(row)" size="small" :label="false" />
|
||||
<span>{{ row.active }} / {{ row.limit }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<template #status="{ row }">
|
||||
<span class="rq-status" :class="`rq-status--${modelState(row).tone}`"><i />{{ modelState(row).label }}</span>
|
||||
</template>
|
||||
</t-table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<p class="rq-footnote">{{ t('system.globalSettings.runtime.footnote') }}</p>
|
||||
</template>
|
||||
@@ -190,7 +231,7 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { getRuntimeQueues, type QueueStat, type RuntimeWorkerPool } from '@/api/system'
|
||||
import { getRuntimeQueues, type QueueStat, type RuntimeWorkerPool, type ModelRuntimeStat } from '@/api/system'
|
||||
|
||||
const { t, te, locale } = useI18n()
|
||||
|
||||
@@ -198,6 +239,8 @@ const POLL_INTERVAL_MS = 5000
|
||||
|
||||
const queues = ref<QueueStat[]>([])
|
||||
const pools = ref<RuntimeWorkerPool[]>([])
|
||||
const models = ref<ModelRuntimeStat[]>([])
|
||||
const modelLimiterAvailable = ref(false)
|
||||
const available = ref(true)
|
||||
const loading = ref(false)
|
||||
const loadedOnce = ref(false)
|
||||
@@ -216,6 +259,24 @@ const columns = computed(() => [
|
||||
{ colKey: 'latency_ms', title: t('system.globalSettings.runtime.columns.latency'), width: 104, align: 'right' as const },
|
||||
{ colKey: 'status', title: t('system.globalSettings.runtime.columns.status'), width: 96 },
|
||||
])
|
||||
const modelColumns = computed(() => [
|
||||
{ colKey: 'model_id', title: t('system.globalSettings.runtime.models.columns.model'), minWidth: 240 },
|
||||
{ colKey: 'active', title: t('system.globalSettings.runtime.models.columns.active'), width: 86, align: 'right' as const },
|
||||
{ colKey: 'waiting', title: t('system.globalSettings.runtime.models.columns.waiting'), width: 86, align: 'right' as const },
|
||||
{ colKey: 'usage', title: t('system.globalSettings.runtime.models.columns.usage'), width: 190 },
|
||||
{ colKey: 'status', title: t('system.globalSettings.runtime.columns.status'), width: 96 },
|
||||
])
|
||||
|
||||
function modelUsage(row: ModelRuntimeStat): number {
|
||||
return row.limit > 0 ? Math.min(100, Math.round(row.active / row.limit * 100)) : 0
|
||||
}
|
||||
|
||||
function modelState(row: ModelRuntimeStat): { label: string; tone: string } {
|
||||
if (row.waiting > 0) return { label: t('system.globalSettings.runtime.models.status.queued'), tone: 'attention' }
|
||||
if (row.active >= row.limit) return { label: t('system.globalSettings.runtime.models.status.full'), tone: 'waiting' }
|
||||
if (row.active > 0) return { label: t('system.globalSettings.runtime.status.working'), tone: 'working' }
|
||||
return { label: t('system.globalSettings.runtime.status.idle'), tone: 'idle' }
|
||||
}
|
||||
|
||||
const totalActive = computed(() => queues.value.reduce((s, q) => s + q.active, 0))
|
||||
const totalPending = computed(() => queues.value.reduce((s, q) => s + q.pending, 0))
|
||||
@@ -282,6 +343,8 @@ async function load(showSpinner: boolean) {
|
||||
available.value = resp.available
|
||||
pools.value = resp.pools || []
|
||||
queues.value = resp.queues || []
|
||||
models.value = resp.models || []
|
||||
modelLimiterAvailable.value = Boolean(resp.model_limiter_available)
|
||||
updatedAt.value = new Date((resp.timestamp || Date.now() / 1000) * 1000)
|
||||
.toLocaleTimeString(locale.value, { hour12: false })
|
||||
error.value = ''
|
||||
@@ -331,6 +394,21 @@ onUnmounted(() => stopPolling())
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
|
||||
.rq-models {
|
||||
margin-top: 40px;
|
||||
padding-top: 32px;
|
||||
}
|
||||
|
||||
.rq-model-usage {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(72px, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
color: var(--td-text-color-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.rq-header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/database"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
modellimiter "github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -1394,13 +1395,15 @@ type RuntimeWorkerPool struct {
|
||||
}
|
||||
|
||||
type RuntimeQueuesResponse struct {
|
||||
Available bool `json:"available"`
|
||||
UpstreamConcurrency int `json:"upstream_concurrency"`
|
||||
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
|
||||
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
|
||||
Pools []RuntimeWorkerPool `json:"pools"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
Available bool `json:"available"`
|
||||
UpstreamConcurrency int `json:"upstream_concurrency"`
|
||||
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
|
||||
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
|
||||
Pools []RuntimeWorkerPool `json:"pools"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
ModelLimiterAvailable bool `json:"model_limiter_available"`
|
||||
Models []modellimiter.RuntimeStat `json:"models"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// GetRuntimeQueues godoc
|
||||
@@ -1459,6 +1462,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
if resp.Queues == nil {
|
||||
resp.Queues = []types.QueueStat{}
|
||||
}
|
||||
modelStats, modelSupported, modelErr := modellimiter.RuntimeStats(ctx)
|
||||
if modelErr != nil {
|
||||
logger.Errorf(ctx, "get model concurrency stats failed: %v", modelErr)
|
||||
}
|
||||
resp.ModelLimiterAvailable = modelSupported
|
||||
resp.Models = modelStats
|
||||
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
@@ -35,13 +35,13 @@ func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName()
|
||||
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Chat(ctx, messages, opts)
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
||||
if err != nil || ch == nil {
|
||||
release()
|
||||
|
||||
@@ -30,13 +30,13 @@ type concurrencyEmbedder struct {
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Embed(ctx, text)
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
defer release()
|
||||
return w.inner.BatchEmbed(ctx, texts)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -53,6 +54,10 @@ func Gate(ctx context.Context, modelID string) func() {
|
||||
// blocks a call permanently — a limiter/Redis outage or a cancelled context
|
||||
// fails open.
|
||||
func GateN(ctx context.Context, modelID string, modelLimit int) func() {
|
||||
return GateNamedN(ctx, modelID, "", modelLimit)
|
||||
}
|
||||
|
||||
func GateNamedN(ctx context.Context, modelID, modelName string, modelLimit int) func() {
|
||||
governorMu.RLock()
|
||||
l, defaultLimit := governor, governorN
|
||||
governorMu.RUnlock()
|
||||
@@ -64,6 +69,9 @@ func GateN(ctx context.Context, modelID string, modelLimit int) func() {
|
||||
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return noop
|
||||
}
|
||||
if named, ok := l.(interface{ SetModelName(string, string) }); ok {
|
||||
named.SetModelName(modelID, modelName)
|
||||
}
|
||||
release, err := l.Acquire(ctx, modelID, limit)
|
||||
if err != nil || release == nil {
|
||||
return noop
|
||||
@@ -71,19 +79,38 @@ func GateN(ctx context.Context, modelID string, modelLimit int) func() {
|
||||
return release
|
||||
}
|
||||
|
||||
// RuntimeStats returns the semaphores observed by this process. Redis-backed
|
||||
// limiters report cluster-wide active holders; waiting callers remain local to
|
||||
// this instance by design.
|
||||
func RuntimeStats(ctx context.Context) ([]RuntimeStat, bool, error) {
|
||||
governorMu.RLock()
|
||||
l := governor
|
||||
governorMu.RUnlock()
|
||||
inspector, ok := l.(runtimeInspectable)
|
||||
if !ok || inspector == nil {
|
||||
return []RuntimeStat{}, false, nil
|
||||
}
|
||||
stats, err := inspector.RuntimeStats(ctx)
|
||||
if stats == nil {
|
||||
stats = []RuntimeStat{}
|
||||
}
|
||||
return stats, true, err
|
||||
}
|
||||
|
||||
// localLimiter is an in-process (single-node) counting semaphore keyed by
|
||||
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
|
||||
// single process with no Redis, so a shared distributed semaphore is neither
|
||||
// available nor needed — but background ingestion can still burst the whole
|
||||
// worker pool against one provider, so we still cap concurrency locally.
|
||||
type localLimiter struct {
|
||||
mu sync.Mutex
|
||||
sems map[string]chan struct{}
|
||||
mu sync.Mutex
|
||||
sems map[string]chan struct{}
|
||||
tracked map[string]*trackedSemaphore
|
||||
}
|
||||
|
||||
// NewLocalLimiter builds an in-process per-key concurrency limiter.
|
||||
func NewLocalLimiter() ModelConcurrencyLimiter {
|
||||
return &localLimiter{sems: make(map[string]chan struct{})}
|
||||
return &localLimiter{sems: make(map[string]chan struct{}), tracked: make(map[string]*trackedSemaphore)}
|
||||
}
|
||||
|
||||
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
|
||||
@@ -93,6 +120,12 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
|
||||
l.mu.Lock()
|
||||
sem, ok := l.sems[key]
|
||||
tracked := l.tracked[key]
|
||||
if tracked == nil {
|
||||
tracked = &trackedSemaphore{}
|
||||
l.tracked[key] = tracked
|
||||
}
|
||||
tracked.limit.Store(int64(limit))
|
||||
if !ok {
|
||||
// Capacity is fixed at first use for a key; the limit is a
|
||||
// process-wide constant, so it never changes across acquires.
|
||||
@@ -100,6 +133,8 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
l.sems[key] = sem
|
||||
}
|
||||
l.mu.Unlock()
|
||||
tracked.waiting.Add(1)
|
||||
defer tracked.waiting.Add(-1)
|
||||
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
@@ -110,3 +145,30 @@ func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
return noop, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *localLimiter) RuntimeStats(_ context.Context) ([]RuntimeStat, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
stats := make([]RuntimeStat, 0, len(l.sems))
|
||||
for modelID, sem := range l.sems {
|
||||
tracked := l.tracked[modelID]
|
||||
name, _ := tracked.name.Load().(string)
|
||||
stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: int64(len(sem)), Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
|
||||
}
|
||||
sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (l *localLimiter) SetModelName(modelID, name string) {
|
||||
if modelID == "" || name == "" {
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
tracked := l.tracked[modelID]
|
||||
if tracked == nil {
|
||||
tracked = &trackedSemaphore{}
|
||||
l.tracked[modelID] = tracked
|
||||
}
|
||||
l.mu.Unlock()
|
||||
tracked.name.Store(name)
|
||||
}
|
||||
|
||||
@@ -15,7 +15,10 @@ package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -34,15 +37,38 @@ type ModelConcurrencyLimiter interface {
|
||||
Acquire(ctx context.Context, key string, limit int) (release func(), err error)
|
||||
}
|
||||
|
||||
// RuntimeStat is a point-in-time view of a model semaphore. Active is
|
||||
// cluster-wide for the Redis backend and process-local in Lite mode. Waiting is
|
||||
// deliberately process-local: waiters block in application processes and are
|
||||
// not represented in Redis.
|
||||
type RuntimeStat struct {
|
||||
ModelID string `json:"model_id"`
|
||||
Name string `json:"name"`
|
||||
Active int64 `json:"active"`
|
||||
Waiting int64 `json:"waiting"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
type runtimeInspectable interface {
|
||||
RuntimeStats(context.Context) ([]RuntimeStat, error)
|
||||
}
|
||||
|
||||
type trackedSemaphore struct {
|
||||
limit atomic.Int64
|
||||
waiting atomic.Int64
|
||||
name atomic.Value // string
|
||||
}
|
||||
|
||||
// noop is the release returned on the fail-open / passthrough paths.
|
||||
func noop() {}
|
||||
|
||||
const (
|
||||
// defaultLeaseTTL is how long a held slot survives without a heartbeat
|
||||
// before another acquirer may reclaim it. Sized well above a typical
|
||||
// enrichment LLM call so a slow provider never loses its slot mid-call;
|
||||
// the heartbeat refreshes it for genuinely long calls.
|
||||
defaultLeaseTTL = 5 * time.Minute
|
||||
// defaultLeaseTTL is the crash-recovery window, not a request timeout.
|
||||
// Live calls refresh their lease every ttl/3, so even a very long provider
|
||||
// request keeps its slot. Keeping this short prevents an app/container
|
||||
// restart from leaving an entire model budget apparently occupied for
|
||||
// minutes while the replacement workers are blocked behind dead holders.
|
||||
defaultLeaseTTL = 30 * time.Second
|
||||
// defaultPollInterval is how often a waiting acquirer re-checks for a free
|
||||
// slot. Small enough to stay responsive, large enough to avoid hammering
|
||||
// Redis under contention.
|
||||
@@ -75,6 +101,7 @@ type redisLimiter struct {
|
||||
rdb *redis.Client
|
||||
ttl time.Duration
|
||||
pollInterval time.Duration
|
||||
tracked sync.Map // model ID -> *trackedSemaphore
|
||||
}
|
||||
|
||||
// NewRedisLimiter builds a distributed limiter backed by rdb. A nil client
|
||||
@@ -93,6 +120,11 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
}
|
||||
|
||||
zkey := keyPrefix + key
|
||||
entry, _ := l.tracked.LoadOrStore(key, &trackedSemaphore{})
|
||||
tracked := entry.(*trackedSemaphore)
|
||||
tracked.limit.Store(int64(limit))
|
||||
tracked.waiting.Add(1)
|
||||
defer tracked.waiting.Add(-1)
|
||||
token := uuid.NewString()
|
||||
ttlMs := l.ttl.Milliseconds()
|
||||
|
||||
@@ -131,6 +163,39 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
}
|
||||
}
|
||||
|
||||
func (l *redisLimiter) RuntimeStats(ctx context.Context) ([]RuntimeStat, error) {
|
||||
stats := make([]RuntimeStat, 0)
|
||||
if l == nil || l.rdb == nil {
|
||||
return stats, nil
|
||||
}
|
||||
var firstErr error
|
||||
now := time.Now().UnixMilli()
|
||||
l.tracked.Range(func(rawKey, rawValue any) bool {
|
||||
modelID := rawKey.(string)
|
||||
tracked := rawValue.(*trackedSemaphore)
|
||||
active, err := l.rdb.ZCount(ctx, keyPrefix+modelID, strconv.FormatInt(now+1, 10), "+inf").Result()
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
return true
|
||||
}
|
||||
name, _ := tracked.name.Load().(string)
|
||||
stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: active, Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
|
||||
return true
|
||||
})
|
||||
sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
|
||||
return stats, firstErr
|
||||
}
|
||||
|
||||
func (l *redisLimiter) SetModelName(modelID, name string) {
|
||||
if modelID == "" || name == "" {
|
||||
return
|
||||
}
|
||||
entry, _ := l.tracked.LoadOrStore(modelID, &trackedSemaphore{})
|
||||
entry.(*trackedSemaphore).name.Store(name)
|
||||
}
|
||||
|
||||
// hold starts a heartbeat that refreshes the lease and returns an idempotent
|
||||
// release that stops the heartbeat and drops the slot.
|
||||
func (l *redisLimiter) hold(zkey, token string) func() {
|
||||
|
||||
@@ -22,7 +22,7 @@ func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Predict(ctx, imgBytes, prompt)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user