feat(models): cap background embedding concurrency per model
Extend the per-model background concurrency governor to embedding, the highest-volume background model call (document ingestion vectorises every chunk, so a batch upload can burst the whole worker pool against one provider). The wrapper sits innermost (below debug/langfuse) because BatchEmbedWithPool fans a batch out into per-sub-batch BatchEmbed calls via the pooler, and the pooler invokes BatchEmbed on whichever Embedder is threaded down as `model`; sitting innermost routes those per-sub-batch provider round-trips through Gate so the semaphore bounds real concurrent calls rather than one coarse per-doc unit. Only background (asynq) calls are throttled; interactive query embedding is never gated.
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
)
|
||||
|
||||
// Embedding is the highest-volume background model call: document ingestion
|
||||
// vectorises every chunk, so a single batch upload can burst the whole worker
|
||||
// pool against one embedding provider. Like chat and vlm, embedding is governed
|
||||
// at the client layer via the shared per-model concurrency governor. Only
|
||||
// background (asynq worker) calls are throttled — see limiter.Gate /
|
||||
// types.IsBackgroundTask; interactive query embedding is never gated.
|
||||
//
|
||||
// Placement note: unlike chat/vlm (outermost), this wrapper sits INNERMOST —
|
||||
// directly around the real embedder, BELOW the debug/langfuse decorators.
|
||||
// BatchEmbedWithPool fans a batch out into per-sub-batch BatchEmbed calls
|
||||
// through the pooler, and the pooler invokes BatchEmbed on whichever Embedder
|
||||
// was threaded down as `model`. Sitting innermost is what routes those
|
||||
// per-sub-batch provider round-trips back through Gate, so the semaphore bounds
|
||||
// real concurrent provider calls rather than one coarse per-document unit. The
|
||||
// trade-off is that Gate wait time is included in debug/langfuse timing, which
|
||||
// is acceptable for background ingestion.
|
||||
type concurrencyEmbedder struct {
|
||||
inner Embedder
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
defer release()
|
||||
return w.inner.Embed(ctx, text)
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
defer release()
|
||||
return w.inner.BatchEmbed(ctx, texts)
|
||||
}
|
||||
|
||||
// BatchEmbedWithPool threads THIS wrapper down as the model so the pooler's
|
||||
// per-sub-batch callbacks land on our gated BatchEmbed above, rather than on
|
||||
// the raw embedder. The wait for each slot is held only around the actual
|
||||
// per-sub-batch provider round-trip.
|
||||
func (w *concurrencyEmbedder) BatchEmbedWithPool(
|
||||
ctx context.Context, model Embedder, texts []string,
|
||||
) ([][]float32, error) {
|
||||
return w.inner.BatchEmbedWithPool(ctx, w, texts)
|
||||
}
|
||||
|
||||
func (w *concurrencyEmbedder) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyEmbedder) GetDimensions() int { return w.inner.GetDimensions() }
|
||||
func (w *concurrencyEmbedder) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
// wrapEmbeddingConcurrency installs the background concurrency governor directly
|
||||
// around the real embedder. Always applied; a cheap passthrough when no limiter
|
||||
// is installed or the call is interactive.
|
||||
func wrapEmbeddingConcurrency(e Embedder) Embedder {
|
||||
if e == nil {
|
||||
return e
|
||||
}
|
||||
return &concurrencyEmbedder{inner: e}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
)
|
||||
|
||||
// fakeEmbedder blocks every Embed/BatchEmbed call until release is closed and
|
||||
// records the max number of calls it ever saw in flight simultaneously, so
|
||||
// tests can assert the governor's per-model concurrency bound.
|
||||
type fakeEmbedder struct {
|
||||
id string
|
||||
pooler EmbedderPooler // optional: exercises the BatchEmbedWithPool fan-out
|
||||
inFlight int32
|
||||
maxSeen int32
|
||||
enter chan struct{} // one signal per call that reaches the provider
|
||||
release chan struct{} // closed to unblock all (current + future) calls
|
||||
}
|
||||
|
||||
func newFakeEmbedder(id string) *fakeEmbedder {
|
||||
return &fakeEmbedder{
|
||||
id: id,
|
||||
enter: make(chan struct{}, 256),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeEmbedder) track() {
|
||||
n := atomic.AddInt32(&f.inFlight, 1)
|
||||
for {
|
||||
old := atomic.LoadInt32(&f.maxSeen)
|
||||
if n <= old || atomic.CompareAndSwapInt32(&f.maxSeen, old, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
f.enter <- struct{}{}
|
||||
<-f.release
|
||||
atomic.AddInt32(&f.inFlight, -1)
|
||||
}
|
||||
|
||||
func (f *fakeEmbedder) Embed(ctx context.Context, _ string) ([]float32, error) {
|
||||
f.track()
|
||||
return []float32{1}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEmbedder) BatchEmbed(ctx context.Context, _ []string) ([][]float32, error) {
|
||||
f.track()
|
||||
return [][]float32{{1}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEmbedder) BatchEmbedWithPool(
|
||||
ctx context.Context, model Embedder, texts []string,
|
||||
) ([][]float32, error) {
|
||||
if f.pooler != nil {
|
||||
return f.pooler.BatchEmbedWithPool(ctx, model, texts)
|
||||
}
|
||||
return model.BatchEmbed(ctx, texts)
|
||||
}
|
||||
|
||||
func (f *fakeEmbedder) GetModelName() string { return f.id }
|
||||
func (f *fakeEmbedder) GetDimensions() int { return 1 }
|
||||
func (f *fakeEmbedder) GetModelID() string { return f.id }
|
||||
|
||||
// TestConcurrencyEmbedderBackgroundGated verifies background BatchEmbed calls
|
||||
// are capped at the per-model limit.
|
||||
func TestConcurrencyEmbedderBackgroundGated(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
|
||||
|
||||
f := newFakeEmbedder("emb-bg")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
const n = 5
|
||||
var wg sync.WaitGroup
|
||||
for range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = w.BatchEmbed(ctx, []string{"x"})
|
||||
}()
|
||||
}
|
||||
|
||||
// With limit=2 exactly two calls may reach the provider concurrently.
|
||||
for i := range 2 {
|
||||
select {
|
||||
case <-f.enter:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("expected call %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight))
|
||||
}
|
||||
}
|
||||
// A third must not sneak in while both slots are held.
|
||||
select {
|
||||
case <-f.enter:
|
||||
t.Fatal("a third call entered while limit=2 slots were held")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(f.release)
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&f.maxSeen); got > 2 {
|
||||
t.Fatalf("max in-flight %d exceeded limit 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderInteractiveNotGated verifies interactive calls bypass
|
||||
// the governor entirely, even at limit 1.
|
||||
func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
f := newFakeEmbedder("emb-interactive")
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
|
||||
ctx := context.Background() // no background marker
|
||||
const n = 3
|
||||
var wg sync.WaitGroup
|
||||
for range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = w.Embed(ctx, "x")
|
||||
}()
|
||||
}
|
||||
// All three must be able to run at once despite limit=1.
|
||||
for i := range n {
|
||||
select {
|
||||
case <-f.enter:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("interactive call %d did not enter (should be ungated), inFlight=%d",
|
||||
i, atomic.LoadInt32(&f.inFlight))
|
||||
}
|
||||
}
|
||||
close(f.release)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// TestConcurrencyEmbedderPoolFanOutGated verifies that BatchEmbedWithPool's
|
||||
// per-sub-batch provider round-trips are individually gated — the reason the
|
||||
// wrapper sits innermost.
|
||||
func TestConcurrencyEmbedderPoolFanOutGated(t *testing.T) {
|
||||
t.Setenv("BATCH_EMBED_SIZE", "1") // one provider round-trip per text
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 2)
|
||||
|
||||
pool, err := ants.NewPool(16)
|
||||
if err != nil {
|
||||
t.Fatalf("new pool: %v", err)
|
||||
}
|
||||
defer pool.Release()
|
||||
|
||||
f := newFakeEmbedder("emb-pool")
|
||||
f.pooler = NewBatchEmbedder(pool)
|
||||
w := wrapEmbeddingConcurrency(f)
|
||||
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, _ = w.BatchEmbedWithPool(ctx, w, []string{"a", "b", "c", "d", "e"})
|
||||
}()
|
||||
|
||||
for i := range 2 {
|
||||
select {
|
||||
case <-f.enter:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("expected sub-batch %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight))
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-f.enter:
|
||||
t.Fatal("a third sub-batch entered while limit=2 slots were held")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(f.release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("BatchEmbedWithPool did not complete after release")
|
||||
}
|
||||
if got := atomic.LoadInt32(&f.maxSeen); got > 2 {
|
||||
t.Fatalf("max in-flight sub-batches %d exceeded limit 2", got)
|
||||
}
|
||||
}
|
||||
@@ -90,6 +90,10 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
|
||||
if setter, ok := e.(interface{ SetSupportsDimensionOverride(bool) }); ok {
|
||||
setter.SetSupportsDimensionOverride(config.SupportsDimensionOverride)
|
||||
}
|
||||
// Innermost: gate the real provider round-trips (including the per-sub-batch
|
||||
// pool callbacks) before debug/langfuse wrap for logging/tracing. See
|
||||
// concurrencyEmbedder for why this sits below the observability decorators.
|
||||
e = wrapEmbeddingConcurrency(e)
|
||||
if logger.LLMDebugEnabled() {
|
||||
e = &debugEmbedder{inner: e}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user