feat(models): add max concurrency setting for background tasks
Introduce a new `max_concurrency` parameter to the model configuration, allowing control over the maximum number of concurrent background tasks (ingestion/enrichment) per model. This feature is applicable to chat, embedding, and VLM models, with a value of 0 or undefined falling back to the global default. Update UI components and localization files to support this new setting, ensuring a seamless user experience.
This commit is contained in:
@@ -28,6 +28,9 @@ export interface ModelConfig {
|
||||
// 会在调用远程模型 API 时附加到每个请求上。Authorization、Content-Type 等保留头会被忽略。
|
||||
custom_headers?: Record<string, string>;
|
||||
supports_vision?: boolean; // Whether the model accepts image/multimodal input
|
||||
// 后台任务(入库/富化)对该模型的并发上限,按模型 ID 全副本共享。
|
||||
// 0 或不填表示沿用全局默认(model.max_concurrency);仅对 chat/embedding/vllm 生效。
|
||||
max_concurrency?: number;
|
||||
app_id?: string;
|
||||
// Secret fields (api_key, app_secret) are never returned by the server in
|
||||
// this shape — they live behind the /credentials subresource. They are
|
||||
|
||||
@@ -313,7 +313,7 @@
|
||||
</template>
|
||||
|
||||
<!-- Section 3 — 高级选项(仅在有内容时渲染,避免空 section 出现底部分隔线) -->
|
||||
<section v-if="activeModelType === 'embedding' || activeModelType === 'chat'" class="setting-drawer__section">
|
||||
<section v-if="['embedding', 'chat', 'vllm'].includes(activeModelType)" class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">{{ $t('model.editor.sectionAdvanced') }}</h4>
|
||||
|
||||
<!-- Embedding 专用:维度 -->
|
||||
@@ -376,6 +376,18 @@
|
||||
</t-select>
|
||||
<p class="form-desc">{{ $t('model.editor.thinkingControlDesc') }}</p>
|
||||
</div>
|
||||
|
||||
<!--
|
||||
Background concurrency cap for this model. Only chat / embedding / vllm
|
||||
are gated by the governor (see internal/models/limiter), so we surface
|
||||
it just for those three. 0 = fall back to the global default.
|
||||
-->
|
||||
<div class="form-item">
|
||||
<label class="form-label">{{ $t('model.editor.maxConcurrencyLabel') }}</label>
|
||||
<t-input v-model.number="formData.maxConcurrency" type="number" :min="0" :max="4096"
|
||||
:placeholder="$t('model.editor.maxConcurrencyPlaceholder')" />
|
||||
<p class="form-desc">{{ $t('model.editor.maxConcurrencyDesc') }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
</t-form>
|
||||
@@ -425,6 +437,8 @@ interface ModelFormData {
|
||||
interfaceType?: 'ollama' | 'openai'
|
||||
isDefault: boolean
|
||||
supportsVision?: boolean
|
||||
/** 后台任务对该模型的并发上限;0/undefined 表示沿用全局默认。仅 chat/embedding/vllm 生效。 */
|
||||
maxConcurrency?: number
|
||||
/** extra_config.thinking_control — how agent thinking on/off maps to API fields. */
|
||||
thinkingControl?: string
|
||||
// 自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)
|
||||
@@ -819,6 +833,7 @@ const formData = ref<ModelFormData>({
|
||||
interfaceType: 'ollama',
|
||||
isDefault: false,
|
||||
supportsVision: false,
|
||||
maxConcurrency: undefined,
|
||||
thinkingControl: defaultThinkingControl('generic', ''),
|
||||
customHeaders: [],
|
||||
appSecret: '',
|
||||
@@ -1059,6 +1074,7 @@ const resetForm = () => {
|
||||
interfaceType: undefined,
|
||||
isDefault: false,
|
||||
supportsVision: false,
|
||||
maxConcurrency: undefined,
|
||||
thinkingControl: defaultThinkingControl('generic', ''),
|
||||
customHeaders: [],
|
||||
appSecret: '',
|
||||
|
||||
@@ -3773,6 +3773,9 @@ export default {
|
||||
dimensionOverrideDesc: 'Enable only if the provider documentation says this model accepts a dimensions parameter.',
|
||||
supportsVisionLabel: 'Supports Vision / Multimodal',
|
||||
supportsVisionDesc: 'Whether the model accepts image and multimodal input',
|
||||
maxConcurrencyLabel: 'Background concurrency limit',
|
||||
maxConcurrencyPlaceholder: '0 = use global default',
|
||||
maxConcurrencyDesc: 'Caps concurrent background (ingestion/enrichment) calls to this model, shared per model across all replicas. 0 or empty falls back to the global default; interactive chat is never affected.',
|
||||
thinkingControlLabel: 'Thinking mode request format',
|
||||
thinkingControlDesc:
|
||||
'Controls how the agent’s “Thinking mode” on/off switch is written to the API. We pre-select based on vendor/model when possible; change it to match your API docs. With “Do not send”, the agent Thinking mode switch has no effect.',
|
||||
|
||||
@@ -2765,6 +2765,9 @@ export default {
|
||||
dimensionOverrideDesc: "제공자 문서에서 이 모델이 dimensions 매개변수를 지원한다고 확인한 경우에만 켜세요.",
|
||||
supportsVisionLabel: "비전/멀티모달 지원",
|
||||
supportsVisionDesc: "모델의 이미지 등 멀티모달 입력 지원 여부",
|
||||
maxConcurrencyLabel: "백그라운드 동시 실행 상한",
|
||||
maxConcurrencyPlaceholder: "0이면 전역 기본값 사용",
|
||||
maxConcurrencyDesc: "문서 인덱싱/보강 등 백그라운드 작업이 이 모델을 호출하는 동시 실행 수를 제한합니다(모델별로 모든 복제본이 공유). 0 또는 비워 두면 전역 기본값을 사용하며, 대화형 채팅에는 영향을 주지 않습니다.",
|
||||
thinkingControlLabel: "사고 모드 매개변수 형식",
|
||||
thinkingControlDesc:
|
||||
"에이전트 「사고 모드」 켜기/끄기 시 API에 어떻게 기록할지 결정합니다. 벤더/모델에 따라 미리 선택되며, 실제 API와 다르면 문서에 맞게 수정하세요. 「전송 안 함」을 선택하면 에이전트 「사고 모드」 스위치가 효과가 없습니다.",
|
||||
|
||||
@@ -2463,6 +2463,9 @@ export default {
|
||||
dimensionOverrideDesc: 'Включайте только если документация провайдера подтверждает поддержку параметра dimensions.',
|
||||
supportsVisionLabel: 'Поддержка визуального / мультимодального ввода',
|
||||
supportsVisionDesc: 'Поддерживает ли модель изображения и другой мультимодальный ввод',
|
||||
maxConcurrencyLabel: 'Лимит фоновой параллельности',
|
||||
maxConcurrencyPlaceholder: '0 — использовать глобальное значение',
|
||||
maxConcurrencyDesc: 'Ограничивает число одновременных фоновых вызовов (индексация/обогащение) к этой модели, общее для модели по всем репликам. 0 или пусто — используется глобальное значение по умолчанию; интерактивный чат не затрагивается.',
|
||||
thinkingControlLabel: 'Формат параметров режима размышления',
|
||||
thinkingControlDesc:
|
||||
'Определяет, как переключатель «Режим размышления» агента записывается в API. При возможности выбирается по поставщику/модели; при несоответствии измените по документации API. При выборе «Не отправлять» переключатель «Режим размышления» агента не действует.',
|
||||
|
||||
@@ -2787,6 +2787,9 @@ export default {
|
||||
dimensionOverrideDesc: "仅在确认该模型支持 dimensions 参数时开启;默认只使用检测到的实际维度。",
|
||||
supportsVisionLabel: "支持视觉/多模态",
|
||||
supportsVisionDesc: "模型是否支持图片等多模态输入",
|
||||
maxConcurrencyLabel: "后台并发上限",
|
||||
maxConcurrencyPlaceholder: "0 表示使用全局默认",
|
||||
maxConcurrencyDesc: "限制文档入库/富化等后台任务对该模型的并发调用数(按模型全副本共享)。0 或留空表示沿用全局默认;不影响交互式对话。",
|
||||
thinkingControlLabel: "思考模式参数格式",
|
||||
thinkingControlDesc:
|
||||
"决定智能体「思考模式」开/关时如何写入 API。已尝试按厂商/模型预选,若与实际情况不符请按 API 文档手动修改;选「不写入」时,智能体「思考模式」开关不生效。",
|
||||
|
||||
@@ -188,6 +188,7 @@ function convertToLegacyFormat(model: ModelConfig) {
|
||||
supportsDimensionOverride: model.parameters.embedding_parameters?.supports_dimension_override || false,
|
||||
isBuiltin: model.is_builtin || false,
|
||||
supportsVision: model.parameters.supports_vision || false,
|
||||
maxConcurrency: model.parameters.max_concurrency,
|
||||
customHeaders: model.parameters.custom_headers
|
||||
? Object.entries(model.parameters.custom_headers).map(([key, value]) => ({ key, value: String(value) }))
|
||||
: [],
|
||||
@@ -443,7 +444,12 @@ const handleModelSave = async (modelData: any) => {
|
||||
supports_vision: true
|
||||
} : saveType === 'chat' ? {
|
||||
supports_vision: modelData.supportsVision ?? false
|
||||
} : {})
|
||||
} : {}),
|
||||
// 后台并发上限:仅 chat/embedding/vllm 受治理,>0 才写入(0/空沿用全局默认)。
|
||||
...(['chat', 'embedding', 'vllm'].includes(saveType)
|
||||
&& Number(modelData.maxConcurrency) > 0
|
||||
? { max_concurrency: Number(modelData.maxConcurrency) }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ type ModelParametersDTO struct {
|
||||
ExtraConfig map[string]string `json:"extra_config,omitempty"`
|
||||
CustomHeaders map[string]string `json:"custom_headers,omitempty"`
|
||||
SupportsVision bool `json:"supports_vision"`
|
||||
MaxConcurrency int `json:"max_concurrency,omitempty"`
|
||||
AppID string `json:"app_id,omitempty"`
|
||||
}
|
||||
|
||||
@@ -67,6 +68,7 @@ func NewModelResponse(ctx context.Context, m *types.Model) *ModelResponse {
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
SupportsVision: m.Parameters.SupportsVision,
|
||||
MaxConcurrency: m.Parameters.MaxConcurrency,
|
||||
AppID: m.Parameters.AppID,
|
||||
}
|
||||
if !CanViewIntegrationSecrets(ctx) {
|
||||
|
||||
@@ -99,13 +99,16 @@ type Chat interface {
|
||||
}
|
||||
|
||||
type ChatConfig struct {
|
||||
Source types.ModelSource
|
||||
BaseURL string
|
||||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
Provider string
|
||||
ExtraConfig map[string]string
|
||||
Source types.ModelSource
|
||||
BaseURL string
|
||||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
Provider string
|
||||
// MaxConcurrency caps concurrent background calls to this model; 0 falls
|
||||
// back to the process-wide default (see limiter.GateN).
|
||||
MaxConcurrency int
|
||||
ExtraConfig map[string]string
|
||||
// CustomHeaders 允许在调用远程 OpenAI 兼容 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string
|
||||
AppID string
|
||||
@@ -121,16 +124,17 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) *ChatConfig {
|
||||
return nil
|
||||
}
|
||||
return &ChatConfig{
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
Provider: m.Parameters.Provider,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
Provider: m.Parameters.Provider,
|
||||
MaxConcurrency: m.Parameters.MaxConcurrency,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +154,7 @@ func NewChat(config *ChatConfig, ollamaService *ollama.OllamaService) (Chat, err
|
||||
c, err = wrapChatLangfuse(c, err)
|
||||
// Outermost: hold the per-model concurrency slot only around the real
|
||||
// provider round-trip, so the wait is excluded from debug/langfuse timing.
|
||||
return wrapChatConcurrency(c, err)
|
||||
return wrapChatConcurrency(c, config.MaxConcurrency, err)
|
||||
}
|
||||
|
||||
// NewRemoteChat 根据 provider 创建远程聊天实例。
|
||||
|
||||
@@ -26,19 +26,22 @@ import (
|
||||
// inner debug/langfuse timing.
|
||||
type concurrencyChat struct {
|
||||
inner Chat
|
||||
// limit is this model's configured per-model background cap; 0 falls back
|
||||
// to the process-wide default (see limiter.GateN).
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Chat(ctx, messages, opts)
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
||||
if err != nil || ch == nil {
|
||||
release()
|
||||
@@ -71,9 +74,9 @@ func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, op
|
||||
// wrapChatConcurrency installs the background concurrency governor as the
|
||||
// outermost Chat decorator. It is always applied; when no limiter is installed
|
||||
// or the call is interactive, the wrapper is a cheap passthrough.
|
||||
func wrapChatConcurrency(c Chat, err error) (Chat, error) {
|
||||
func wrapChatConcurrency(c Chat, limit int, err error) (Chat, error) {
|
||||
if err != nil || c == nil {
|
||||
return c, err
|
||||
}
|
||||
return &concurrencyChat{inner: c}, nil
|
||||
return &concurrencyChat{inner: c, limit: limit}, nil
|
||||
}
|
||||
|
||||
@@ -24,16 +24,19 @@ import (
|
||||
// is acceptable for background ingestion.
|
||||
type concurrencyEmbedder struct {
|
||||
inner Embedder
|
||||
// limit is this model's configured per-model background cap; 0 falls back
|
||||
// to the process-wide default (see limiter.GateN).
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Embed(ctx, text)
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.BatchEmbed(ctx, texts)
|
||||
}
|
||||
@@ -55,9 +58,9 @@ func (w *concurrencyEmbedder) GetModelID() string { return w.inner.GetModelID(
|
||||
// wrapEmbeddingConcurrency installs the background concurrency governor directly
|
||||
// around the real embedder. Always applied; a cheap passthrough when no limiter
|
||||
// is installed or the call is interactive.
|
||||
func wrapEmbeddingConcurrency(e Embedder) Embedder {
|
||||
func wrapEmbeddingConcurrency(e Embedder, limit int) Embedder {
|
||||
if e == nil {
|
||||
return e
|
||||
}
|
||||
return &concurrencyEmbedder{inner: e}
|
||||
return &concurrencyEmbedder{inner: e, limit: limit}
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
|
||||
|
||||
f := newFakeEmbedder("emb-bg")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
const n = 5
|
||||
@@ -110,6 +110,46 @@ func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderPerModelLimitOverridesDefault verifies a model's own
|
||||
// configured limit takes precedence over the process-wide default.
|
||||
func TestConcurrencyEmbedderPerModelLimitOverridesDefault(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
// Global default is generous (10), but this model is pinned to 1.
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 10)
|
||||
|
||||
f := newFakeEmbedder("emb-permodel")
|
||||
w := wrapEmbeddingConcurrency(f, 1)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
const n = 3
|
||||
var wg sync.WaitGroup
|
||||
for range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = w.BatchEmbed(ctx, []string{"x"})
|
||||
}()
|
||||
}
|
||||
|
||||
// Only one may be in flight because the per-model limit is 1.
|
||||
select {
|
||||
case <-f.enter:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("expected one call to enter, inFlight=%d", atomic.LoadInt32(&f.inFlight))
|
||||
}
|
||||
select {
|
||||
case <-f.enter:
|
||||
t.Fatal("a second call entered while per-model limit=1 slot was held")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(f.release)
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&f.maxSeen); got > 1 {
|
||||
t.Fatalf("max in-flight %d exceeded per-model limit 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderInteractiveNotGated verifies interactive calls bypass
|
||||
// the governor entirely, even at limit 1.
|
||||
func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
|
||||
@@ -117,7 +157,7 @@ func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
f := newFakeEmbedder("emb-interactive")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := context.Background() // no background marker
|
||||
const n = 3
|
||||
@@ -158,7 +198,7 @@ func TestConcurrencyEmbedderPoolFanOutGated(t *testing.T) {
|
||||
|
||||
f := newFakeEmbedder("emb-pool")
|
||||
f.pooler = NewBatchEmbedder(pool)
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
done := make(chan struct{})
|
||||
|
||||
@@ -50,7 +50,10 @@ type Config struct {
|
||||
SupportsDimensionOverride bool `json:"supports_dimension_override"`
|
||||
ModelID string `json:"model_id"`
|
||||
Provider string `json:"provider"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
// MaxConcurrency caps concurrent background calls to this model; 0 falls
|
||||
// back to the process-wide default (see limiter.GateN).
|
||||
MaxConcurrency int `json:"max_concurrency"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string `json:"custom_headers"`
|
||||
AppID string
|
||||
@@ -74,6 +77,7 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) Config {
|
||||
SupportsDimensionOverride: m.Parameters.EmbeddingParameters.SupportsDimensionOverride,
|
||||
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
|
||||
Provider: m.Parameters.Provider,
|
||||
MaxConcurrency: m.Parameters.MaxConcurrency,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
@@ -93,7 +97,7 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
|
||||
// Innermost: gate the real provider round-trips (including the per-sub-batch
|
||||
// pool callbacks) before debug/langfuse wrap for logging/tracing. See
|
||||
// concurrencyEmbedder for why this sits below the observability decorators.
|
||||
e = wrapEmbeddingConcurrency(e)
|
||||
e = wrapEmbeddingConcurrency(e, config.MaxConcurrency)
|
||||
if logger.LLMDebugEnabled() {
|
||||
e = &debugEmbedder{inner: e}
|
||||
}
|
||||
|
||||
@@ -28,16 +28,28 @@ func SetGovernor(l ModelConcurrencyLimiter, limit int) {
|
||||
governorN = limit
|
||||
}
|
||||
|
||||
// Gate acquires a per-model concurrency slot when the call is a background task
|
||||
// (see types.IsBackgroundTask) and a governor is installed. It returns a
|
||||
// release func that is ALWAYS safe to call: on the passthrough / fail-open
|
||||
// paths it is a cheap no-op. The gate never blocks a call permanently — a
|
||||
// limiter/Redis outage or a cancelled context fails open.
|
||||
// Gate acquires a per-model concurrency slot using the process-wide default
|
||||
// limit. Equivalent to GateN(ctx, modelID, 0).
|
||||
func Gate(ctx context.Context, modelID string) func() {
|
||||
return GateN(ctx, modelID, 0)
|
||||
}
|
||||
|
||||
// GateN acquires a per-model concurrency slot when the call is a background task
|
||||
// (see types.IsBackgroundTask) and a governor is installed. modelLimit is the
|
||||
// model's own configured cap; a value <= 0 means "fall back to the process-wide
|
||||
// default" (governorN). It returns a release func that is ALWAYS safe to call:
|
||||
// on the passthrough / fail-open paths it is a cheap no-op. The gate never
|
||||
// blocks a call permanently — a limiter/Redis outage or a cancelled context
|
||||
// fails open.
|
||||
func GateN(ctx context.Context, modelID string, modelLimit int) func() {
|
||||
governorMu.RLock()
|
||||
l, limit := governor, governorN
|
||||
l, defaultLimit := governor, governorN
|
||||
governorMu.RUnlock()
|
||||
|
||||
limit := modelLimit
|
||||
if limit <= 0 {
|
||||
limit = defaultLimit
|
||||
}
|
||||
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return noop
|
||||
}
|
||||
|
||||
@@ -13,13 +13,16 @@ import (
|
||||
// worker) calls are throttled — see limiter.Gate / types.IsBackgroundTask.
|
||||
type concurrencyVLM struct {
|
||||
inner VLM
|
||||
// limit is this model's configured per-model background cap; 0 falls back
|
||||
// to the process-wide default (see limiter.GateN).
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Predict(ctx, imgBytes, prompt)
|
||||
}
|
||||
@@ -27,9 +30,9 @@ func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt
|
||||
// wrapVLMConcurrency installs the background concurrency governor as the
|
||||
// outermost VLM decorator. Always applied; a cheap passthrough when no limiter
|
||||
// is installed or the call is interactive.
|
||||
func wrapVLMConcurrency(v VLM, err error) (VLM, error) {
|
||||
func wrapVLMConcurrency(v VLM, limit int, err error) (VLM, error) {
|
||||
if err != nil || v == nil {
|
||||
return v, err
|
||||
}
|
||||
return &concurrencyVLM{inner: v}, nil
|
||||
return &concurrencyVLM{inner: v, limit: limit}, nil
|
||||
}
|
||||
|
||||
+17
-13
@@ -29,7 +29,10 @@ type Config struct {
|
||||
ModelID string
|
||||
InterfaceType string // "ollama" or "openai" (default)
|
||||
Provider string
|
||||
Extra map[string]any
|
||||
// MaxConcurrency caps concurrent background calls to this model; 0 falls
|
||||
// back to the process-wide default (see limiter.GateN).
|
||||
MaxConcurrency int
|
||||
Extra map[string]any
|
||||
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string
|
||||
AppID string
|
||||
@@ -53,17 +56,18 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) *Config {
|
||||
}
|
||||
}
|
||||
return &Config{
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
InterfaceType: ifType,
|
||||
Provider: m.Parameters.Provider,
|
||||
Extra: stringMapToAnyMap(m.Parameters.ExtraConfig),
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
ModelID: m.ID,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
InterfaceType: ifType,
|
||||
Provider: m.Parameters.Provider,
|
||||
MaxConcurrency: m.Parameters.MaxConcurrency,
|
||||
Extra: stringMapToAnyMap(m.Parameters.ExtraConfig),
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +94,7 @@ func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
||||
v, err = wrapVLMLangfuse(v, nil)
|
||||
// Outermost: hold the per-model concurrency slot only around the real
|
||||
// provider round-trip, so the wait is excluded from debug/langfuse timing.
|
||||
return wrapVLMConcurrency(v, err)
|
||||
return wrapVLMConcurrency(v, config.MaxConcurrency, err)
|
||||
}
|
||||
|
||||
func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
||||
|
||||
@@ -75,6 +75,12 @@ type ModelParameters struct {
|
||||
// 保留字段(Authorization、api-key、Content-Type、Accept 等)会在运行期被忽略以避免破坏签名/鉴权流程。
|
||||
CustomHeaders map[string]string `yaml:"custom_headers,omitempty" json:"custom_headers,omitempty"`
|
||||
SupportsVision bool `yaml:"supports_vision" json:"supports_vision"` // Whether the model accepts image/multimodal input
|
||||
// MaxConcurrency caps concurrent in-flight BACKGROUND (ingestion /
|
||||
// enrichment) calls to THIS specific model, keyed by model ID and shared
|
||||
// across all replicas. 0 (the default) means "fall back to the
|
||||
// process-wide model.max_concurrency". Interactive user-facing calls are
|
||||
// never gated. Only chat / vlm / embedding honour this (see limiter.Gate).
|
||||
MaxConcurrency int `yaml:"max_concurrency,omitempty" json:"max_concurrency,omitempty"`
|
||||
// WeKnoraCloud 厂商专用凭证
|
||||
AppID string `yaml:"app_id,omitempty" json:"app_id,omitempty"`
|
||||
AppSecret string `yaml:"app_secret,omitempty" json:"app_secret,omitempty"` // AES-256 加密存储,实际承载上游 API Key
|
||||
|
||||
Reference in New Issue
Block a user