feat(models): cap background embedding concurrency per model

Extend the per-model background concurrency governor to embedding, the
highest-volume background model call (document ingestion vectorises every
chunk, so a batch upload can burst the whole worker pool against one provider).

The wrapper sits innermost (below debug/langfuse) because BatchEmbedWithPool
fans a batch out into per-sub-batch BatchEmbed calls via the pooler, and the
pooler invokes BatchEmbed on whichever Embedder is threaded down as `model`;
sitting innermost routes those per-sub-batch provider round-trips through Gate
so the semaphore bounds real concurrent calls rather than one coarse per-doc
unit. Only background (asynq) calls are throttled; interactive query embedding
is never gated.
This commit is contained in:
wizardchen
2026-07-09 19:41:18 +08:00
committed by lyingbug
parent 98ad41cda4
commit 200c344065
3 changed files with 259 additions and 0 deletions
@@ -0,0 +1,63 @@
package embedding
import (
"context"
"github.com/Tencent/WeKnora/internal/models/limiter"
)
// Embedding is the highest-volume background model call: document ingestion
// vectorises every chunk, so a single batch upload can burst the whole worker
// pool against one embedding provider. Like chat and vlm, embedding is governed
// at the client layer via the shared per-model concurrency governor. Only
// background (asynq worker) calls are throttled — see limiter.Gate /
// types.IsBackgroundTask; interactive query embedding is never gated.
//
// Placement note: unlike chat/vlm (outermost), this wrapper sits INNERMOST —
// directly around the real embedder, BELOW the debug/langfuse decorators.
// BatchEmbedWithPool fans a batch out into per-sub-batch BatchEmbed calls
// through the pooler, and the pooler invokes BatchEmbed on whichever Embedder
// was threaded down as `model`. Sitting innermost is what routes those
// per-sub-batch provider round-trips back through Gate, so the semaphore bounds
// real concurrent provider calls rather than one coarse per-document unit. The
// trade-off is that Gate wait time is included in debug/langfuse timing, which
// is acceptable for background ingestion.
type concurrencyEmbedder struct {
inner Embedder
}
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
release := limiter.Gate(ctx, w.inner.GetModelID())
defer release()
return w.inner.Embed(ctx, text)
}
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
release := limiter.Gate(ctx, w.inner.GetModelID())
defer release()
return w.inner.BatchEmbed(ctx, texts)
}
// BatchEmbedWithPool threads THIS wrapper down as the model so the pooler's
// per-sub-batch callbacks land on our gated BatchEmbed above, rather than on
// the raw embedder. The wait for each slot is held only around the actual
// per-sub-batch provider round-trip.
func (w *concurrencyEmbedder) BatchEmbedWithPool(
ctx context.Context, model Embedder, texts []string,
) ([][]float32, error) {
return w.inner.BatchEmbedWithPool(ctx, w, texts)
}
func (w *concurrencyEmbedder) GetModelName() string { return w.inner.GetModelName() }
func (w *concurrencyEmbedder) GetDimensions() int { return w.inner.GetDimensions() }
func (w *concurrencyEmbedder) GetModelID() string { return w.inner.GetModelID() }
// wrapEmbeddingConcurrency installs the background concurrency governor directly
// around the real embedder. Always applied; a cheap passthrough when no limiter
// is installed or the call is interactive.
func wrapEmbeddingConcurrency(e Embedder) Embedder {
if e == nil {
return e
}
return &concurrencyEmbedder{inner: e}
}
@@ -0,0 +1,192 @@
package embedding
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/types"
"github.com/panjf2000/ants/v2"
)
// fakeEmbedder blocks every Embed/BatchEmbed call until release is closed and
// records the max number of calls it ever saw in flight simultaneously, so
// tests can assert the governor's per-model concurrency bound.
type fakeEmbedder struct {
id string
pooler EmbedderPooler // optional: exercises the BatchEmbedWithPool fan-out
inFlight int32
maxSeen int32
enter chan struct{} // one signal per call that reaches the provider
release chan struct{} // closed to unblock all (current + future) calls
}
func newFakeEmbedder(id string) *fakeEmbedder {
return &fakeEmbedder{
id: id,
enter: make(chan struct{}, 256),
release: make(chan struct{}),
}
}
func (f *fakeEmbedder) track() {
n := atomic.AddInt32(&f.inFlight, 1)
for {
old := atomic.LoadInt32(&f.maxSeen)
if n <= old || atomic.CompareAndSwapInt32(&f.maxSeen, old, n) {
break
}
}
f.enter <- struct{}{}
<-f.release
atomic.AddInt32(&f.inFlight, -1)
}
func (f *fakeEmbedder) Embed(ctx context.Context, _ string) ([]float32, error) {
f.track()
return []float32{1}, nil
}
func (f *fakeEmbedder) BatchEmbed(ctx context.Context, _ []string) ([][]float32, error) {
f.track()
return [][]float32{{1}}, nil
}
func (f *fakeEmbedder) BatchEmbedWithPool(
ctx context.Context, model Embedder, texts []string,
) ([][]float32, error) {
if f.pooler != nil {
return f.pooler.BatchEmbedWithPool(ctx, model, texts)
}
return model.BatchEmbed(ctx, texts)
}
func (f *fakeEmbedder) GetModelName() string { return f.id }
func (f *fakeEmbedder) GetDimensions() int { return 1 }
func (f *fakeEmbedder) GetModelID() string { return f.id }
// TestConcurrencyEmbedderBackgroundGated verifies background BatchEmbed calls
// are capped at the per-model limit.
func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
f := newFakeEmbedder("emb-bg")
w := wrapEmbeddingConcurrency(f)
ctx := types.WithBackgroundTask(context.Background())
const n = 5
var wg sync.WaitGroup
for range n {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = w.BatchEmbed(ctx, []string{"x"})
}()
}
// With limit=2 exactly two calls may reach the provider concurrently.
for i := range 2 {
select {
case <-f.enter:
case <-time.After(2 * time.Second):
t.Fatalf("expected call %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight))
}
}
// A third must not sneak in while both slots are held.
select {
case <-f.enter:
t.Fatal("a third call entered while limit=2 slots were held")
case <-time.After(150 * time.Millisecond):
}
close(f.release)
wg.Wait()
if got := atomic.LoadInt32(&f.maxSeen); got > 2 {
t.Fatalf("max in-flight %d exceeded limit 2", got)
}
}
// TestConcurrencyEmbedderInteractiveNotGated verifies interactive calls bypass
// the governor entirely, even at limit 1.
func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
f := newFakeEmbedder("emb-interactive")
w := wrapEmbeddingConcurrency(f)
ctx := context.Background() // no background marker
const n = 3
var wg sync.WaitGroup
for range n {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = w.Embed(ctx, "x")
}()
}
// All three must be able to run at once despite limit=1.
for i := range n {
select {
case <-f.enter:
case <-time.After(2 * time.Second):
t.Fatalf("interactive call %d did not enter (should be ungated), inFlight=%d",
i, atomic.LoadInt32(&f.inFlight))
}
}
close(f.release)
wg.Wait()
}
// TestConcurrencyEmbedderPoolFanOutGated verifies that BatchEmbedWithPool's
// per-sub-batch provider round-trips are individually gated — the reason the
// wrapper sits innermost.
func TestConcurrencyEmbedderPoolFanOutGated(t *testing.T) {
t.Setenv("BATCH_EMBED_SIZE", "1") // one provider round-trip per text
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
pool, err := ants.NewPool(16)
if err != nil {
t.Fatalf("new pool: %v", err)
}
defer pool.Release()
f := newFakeEmbedder("emb-pool")
f.pooler = NewBatchEmbedder(pool)
w := wrapEmbeddingConcurrency(f)
ctx := types.WithBackgroundTask(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
_, _ = w.BatchEmbedWithPool(ctx, w, []string{"a", "b", "c", "d", "e"})
}()
for i := range 2 {
select {
case <-f.enter:
case <-time.After(2 * time.Second):
t.Fatalf("expected sub-batch %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight))
}
}
select {
case <-f.enter:
t.Fatal("a third sub-batch entered while limit=2 slots were held")
case <-time.After(150 * time.Millisecond):
}
close(f.release)
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("BatchEmbedWithPool did not complete after release")
}
if got := atomic.LoadInt32(&f.maxSeen); got > 2 {
t.Fatalf("max in-flight sub-batches %d exceeded limit 2", got)
}
}
+4
View File
@@ -90,6 +90,10 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
if setter, ok := e.(interface{ SetSupportsDimensionOverride(bool) }); ok {
setter.SetSupportsDimensionOverride(config.SupportsDimensionOverride)
}
// Innermost: gate the real provider round-trips (including the per-sub-batch
// pool callbacks) before debug/langfuse wrap for logging/tracing. See
// concurrencyEmbedder for why this sits below the observability decorators.
e = wrapEmbeddingConcurrency(e)
if logger.LLMDebugEnabled() {
e = &debugEmbedder{inner: e}
}