feat(models): add max concurrency setting for background tasks
Introduce a new `max_concurrency` parameter to the model configuration, allowing control over the maximum number of concurrent background tasks (ingestion/enrichment) per model. This feature is applicable to chat, embedding, and VLM models, with a value of 0 or undefined falling back to the global default. Update UI components and localization files to support this new setting, ensuring a seamless user experience.
This commit is contained in:
@@ -24,16 +24,19 @@ import (
|
||||
// is acceptable for background ingestion.
|
||||
type concurrencyEmbedder struct {
|
||||
inner Embedder
|
||||
// limit is this model's configured per-model background cap; 0 falls back
|
||||
// to the process-wide default (see limiter.GateN).
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Embed(ctx, text)
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
release := limiter.GateN(ctx, w.inner.GetModelID(), w.limit)
|
||||
defer release()
|
||||
return w.inner.BatchEmbed(ctx, texts)
|
||||
}
|
||||
@@ -55,9 +58,9 @@ func (w *concurrencyEmbedder) GetModelID() string { return w.inner.GetModelID(
|
||||
// wrapEmbeddingConcurrency installs the background concurrency governor directly
|
||||
// around the real embedder. Always applied; a cheap passthrough when no limiter
|
||||
// is installed or the call is interactive.
|
||||
func wrapEmbeddingConcurrency(e Embedder) Embedder {
|
||||
func wrapEmbeddingConcurrency(e Embedder, limit int) Embedder {
|
||||
if e == nil {
|
||||
return e
|
||||
}
|
||||
return &concurrencyEmbedder{inner: e}
|
||||
return &concurrencyEmbedder{inner: e, limit: limit}
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
|
||||
|
||||
f := newFakeEmbedder("emb-bg")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
const n = 5
|
||||
@@ -110,6 +110,46 @@ func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderPerModelLimitOverridesDefault verifies a model's own
|
||||
// configured limit takes precedence over the process-wide default.
|
||||
func TestConcurrencyEmbedderPerModelLimitOverridesDefault(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
// Global default is generous (10), but this model is pinned to 1.
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 10)
|
||||
|
||||
f := newFakeEmbedder("emb-permodel")
|
||||
w := wrapEmbeddingConcurrency(f, 1)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
const n = 3
|
||||
var wg sync.WaitGroup
|
||||
for range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = w.BatchEmbed(ctx, []string{"x"})
|
||||
}()
|
||||
}
|
||||
|
||||
// Only one may be in flight because the per-model limit is 1.
|
||||
select {
|
||||
case <-f.enter:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("expected one call to enter, inFlight=%d", atomic.LoadInt32(&f.inFlight))
|
||||
}
|
||||
select {
|
||||
case <-f.enter:
|
||||
t.Fatal("a second call entered while per-model limit=1 slot was held")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(f.release)
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&f.maxSeen); got > 1 {
|
||||
t.Fatalf("max in-flight %d exceeded per-model limit 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderInteractiveNotGated verifies interactive calls bypass
|
||||
// the governor entirely, even at limit 1.
|
||||
func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
|
||||
@@ -117,7 +157,7 @@ func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
f := newFakeEmbedder("emb-interactive")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := context.Background() // no background marker
|
||||
const n = 3
|
||||
@@ -158,7 +198,7 @@ func TestConcurrencyEmbedderPoolFanOutGated(t *testing.T) {
|
||||
|
||||
f := newFakeEmbedder("emb-pool")
|
||||
f.pooler = NewBatchEmbedder(pool)
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
w := wrapEmbeddingConcurrency(f, 0)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
done := make(chan struct{})
|
||||
|
||||
@@ -50,7 +50,10 @@ type Config struct {
|
||||
SupportsDimensionOverride bool `json:"supports_dimension_override"`
|
||||
ModelID string `json:"model_id"`
|
||||
Provider string `json:"provider"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
// MaxConcurrency caps concurrent background calls to this model; 0 falls
|
||||
// back to the process-wide default (see limiter.GateN).
|
||||
MaxConcurrency int `json:"max_concurrency"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string `json:"custom_headers"`
|
||||
AppID string
|
||||
@@ -74,6 +77,7 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) Config {
|
||||
SupportsDimensionOverride: m.Parameters.EmbeddingParameters.SupportsDimensionOverride,
|
||||
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
|
||||
Provider: m.Parameters.Provider,
|
||||
MaxConcurrency: m.Parameters.MaxConcurrency,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
@@ -93,7 +97,7 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
|
||||
// Innermost: gate the real provider round-trips (including the per-sub-batch
|
||||
// pool callbacks) before debug/langfuse wrap for logging/tracing. See
|
||||
// concurrencyEmbedder for why this sits below the observability decorators.
|
||||
e = wrapEmbeddingConcurrency(e)
|
||||
e = wrapEmbeddingConcurrency(e, config.MaxConcurrency)
|
||||
if logger.LLMDebugEnabled() {
|
||||
e = &debugEmbedder{inner: e}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user