diff --git a/internal/models/embedding/concurrency_wrapper.go b/internal/models/embedding/concurrency_wrapper.go new file mode 100644 index 00000000..cbdf7768 --- /dev/null +++ b/internal/models/embedding/concurrency_wrapper.go @@ -0,0 +1,63 @@ +package embedding + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/models/limiter" +) + +// Embedding is the highest-volume background model call: document ingestion +// vectorises every chunk, so a single batch upload can burst the whole worker +// pool against one embedding provider. Like chat and vlm, embedding is governed +// at the client layer via the shared per-model concurrency governor. Only +// background (asynq worker) calls are throttled — see limiter.Gate / +// types.IsBackgroundTask; interactive query embedding is never gated. +// +// Placement note: unlike chat/vlm (outermost), this wrapper sits INNERMOST — +// directly around the real embedder, BELOW the debug/langfuse decorators. +// BatchEmbedWithPool fans a batch out into per-sub-batch BatchEmbed calls +// through the pooler, and the pooler invokes BatchEmbed on whichever Embedder +// was threaded down as `model`. Sitting innermost is what routes those +// per-sub-batch provider round-trips back through Gate, so the semaphore bounds +// real concurrent provider calls rather than one coarse per-document unit. The +// trade-off is that Gate wait time is included in debug/langfuse timing, which +// is acceptable for background ingestion. +type concurrencyEmbedder struct { + inner Embedder +} + +func (w *concurrencyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + release := limiter.Gate(ctx, w.inner.GetModelID()) + defer release() + return w.inner.Embed(ctx, text) +} + +func (w *concurrencyEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + release := limiter.Gate(ctx, w.inner.GetModelID()) + defer release() + return w.inner.BatchEmbed(ctx, texts) +} + +// BatchEmbedWithPool threads THIS wrapper down as the model so the pooler's +// per-sub-batch callbacks land on our gated BatchEmbed above, rather than on +// the raw embedder. The wait for each slot is held only around the actual +// per-sub-batch provider round-trip. +func (w *concurrencyEmbedder) BatchEmbedWithPool( + ctx context.Context, model Embedder, texts []string, +) ([][]float32, error) { + return w.inner.BatchEmbedWithPool(ctx, w, texts) +} + +func (w *concurrencyEmbedder) GetModelName() string { return w.inner.GetModelName() } +func (w *concurrencyEmbedder) GetDimensions() int { return w.inner.GetDimensions() } +func (w *concurrencyEmbedder) GetModelID() string { return w.inner.GetModelID() } + +// wrapEmbeddingConcurrency installs the background concurrency governor directly +// around the real embedder. Always applied; a cheap passthrough when no limiter +// is installed or the call is interactive. +func wrapEmbeddingConcurrency(e Embedder) Embedder { + if e == nil { + return e + } + return &concurrencyEmbedder{inner: e} +} diff --git a/internal/models/embedding/concurrency_wrapper_test.go b/internal/models/embedding/concurrency_wrapper_test.go new file mode 100644 index 00000000..d1a281d7 --- /dev/null +++ b/internal/models/embedding/concurrency_wrapper_test.go @@ -0,0 +1,192 @@ +package embedding + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/models/limiter" + "github.com/Tencent/WeKnora/internal/types" + "github.com/panjf2000/ants/v2" +) + +// fakeEmbedder blocks every Embed/BatchEmbed call until release is closed and +// records the max number of calls it ever saw in flight simultaneously, so +// tests can assert the governor's per-model concurrency bound. +type fakeEmbedder struct { + id string + pooler EmbedderPooler // optional: exercises the BatchEmbedWithPool fan-out + inFlight int32 + maxSeen int32 + enter chan struct{} // one signal per call that reaches the provider + release chan struct{} // closed to unblock all (current + future) calls +} + +func newFakeEmbedder(id string) *fakeEmbedder { + return &fakeEmbedder{ + id: id, + enter: make(chan struct{}, 256), + release: make(chan struct{}), + } +} + +func (f *fakeEmbedder) track() { + n := atomic.AddInt32(&f.inFlight, 1) + for { + old := atomic.LoadInt32(&f.maxSeen) + if n <= old || atomic.CompareAndSwapInt32(&f.maxSeen, old, n) { + break + } + } + f.enter <- struct{}{} + <-f.release + atomic.AddInt32(&f.inFlight, -1) +} + +func (f *fakeEmbedder) Embed(ctx context.Context, _ string) ([]float32, error) { + f.track() + return []float32{1}, nil +} + +func (f *fakeEmbedder) BatchEmbed(ctx context.Context, _ []string) ([][]float32, error) { + f.track() + return [][]float32{{1}}, nil +} + +func (f *fakeEmbedder) BatchEmbedWithPool( + ctx context.Context, model Embedder, texts []string, +) ([][]float32, error) { + if f.pooler != nil { + return f.pooler.BatchEmbedWithPool(ctx, model, texts) + } + return model.BatchEmbed(ctx, texts) +} + +func (f *fakeEmbedder) GetModelName() string { return f.id } +func (f *fakeEmbedder) GetDimensions() int { return 1 } +func (f *fakeEmbedder) GetModelID() string { return f.id } + +// TestConcurrencyEmbedderBackgroundGated verifies background BatchEmbed calls +// are capped at the per-model limit. +func TestConcurrencyEmbedderBackgroundGated(t *testing.T) { + t.Cleanup(func() { limiter.SetGovernor(nil, 0) }) + limiter.SetGovernor(limiter.NewLocalLimiter(), 2) + + f := newFakeEmbedder("emb-bg") + w := wrapEmbeddingConcurrency(f) + + ctx := types.WithBackgroundTask(context.Background()) + const n = 5 + var wg sync.WaitGroup + for range n { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = w.BatchEmbed(ctx, []string{"x"}) + }() + } + + // With limit=2 exactly two calls may reach the provider concurrently. + for i := range 2 { + select { + case <-f.enter: + case <-time.After(2 * time.Second): + t.Fatalf("expected call %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight)) + } + } + // A third must not sneak in while both slots are held. + select { + case <-f.enter: + t.Fatal("a third call entered while limit=2 slots were held") + case <-time.After(150 * time.Millisecond): + } + + close(f.release) + wg.Wait() + if got := atomic.LoadInt32(&f.maxSeen); got > 2 { + t.Fatalf("max in-flight %d exceeded limit 2", got) + } +} + +// TestConcurrencyEmbedderInteractiveNotGated verifies interactive calls bypass +// the governor entirely, even at limit 1. +func TestConcurrencyEmbedderInteractiveNotGated(t *testing.T) { + t.Cleanup(func() { limiter.SetGovernor(nil, 0) }) + limiter.SetGovernor(limiter.NewLocalLimiter(), 1) + + f := newFakeEmbedder("emb-interactive") + w := wrapEmbeddingConcurrency(f) + + ctx := context.Background() // no background marker + const n = 3 + var wg sync.WaitGroup + for range n { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = w.Embed(ctx, "x") + }() + } + // All three must be able to run at once despite limit=1. + for i := range n { + select { + case <-f.enter: + case <-time.After(2 * time.Second): + t.Fatalf("interactive call %d did not enter (should be ungated), inFlight=%d", + i, atomic.LoadInt32(&f.inFlight)) + } + } + close(f.release) + wg.Wait() +} + +// TestConcurrencyEmbedderPoolFanOutGated verifies that BatchEmbedWithPool's +// per-sub-batch provider round-trips are individually gated — the reason the +// wrapper sits innermost. +func TestConcurrencyEmbedderPoolFanOutGated(t *testing.T) { + t.Setenv("BATCH_EMBED_SIZE", "1") // one provider round-trip per text + t.Cleanup(func() { limiter.SetGovernor(nil, 0) }) + limiter.SetGovernor(limiter.NewLocalLimiter(), 2) + + pool, err := ants.NewPool(16) + if err != nil { + t.Fatalf("new pool: %v", err) + } + defer pool.Release() + + f := newFakeEmbedder("emb-pool") + f.pooler = NewBatchEmbedder(pool) + w := wrapEmbeddingConcurrency(f) + + ctx := types.WithBackgroundTask(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = w.BatchEmbedWithPool(ctx, w, []string{"a", "b", "c", "d", "e"}) + }() + + for i := range 2 { + select { + case <-f.enter: + case <-time.After(2 * time.Second): + t.Fatalf("expected sub-batch %d to enter, inFlight=%d", i, atomic.LoadInt32(&f.inFlight)) + } + } + select { + case <-f.enter: + t.Fatal("a third sub-batch entered while limit=2 slots were held") + case <-time.After(150 * time.Millisecond): + } + + close(f.release) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("BatchEmbedWithPool did not complete after release") + } + if got := atomic.LoadInt32(&f.maxSeen); got > 2 { + t.Fatalf("max in-flight sub-batches %d exceeded limit 2", got) + } +} diff --git a/internal/models/embedding/embedder.go b/internal/models/embedding/embedder.go index 1d377828..00765976 100644 --- a/internal/models/embedding/embedder.go +++ b/internal/models/embedding/embedder.go @@ -90,6 +90,10 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll if setter, ok := e.(interface{ SetSupportsDimensionOverride(bool) }); ok { setter.SetSupportsDimensionOverride(config.SupportsDimensionOverride) } + // Innermost: gate the real provider round-trips (including the per-sub-batch + // pool callbacks) before debug/langfuse wrap for logging/tracing. See + // concurrencyEmbedder for why this sits below the observability decorators. + e = wrapEmbeddingConcurrency(e) if logger.LLMDebugEnabled() { e = &debugEmbedder{inner: e} }