fix(models): harden background concurrency governor
Follow-up hardening on the per-model background concurrency governor: - limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the lease score, so a semaphore that stays saturated with no slot turnover can't let the whole key expire (which would drop every lease and admit over the limit); reuse one poll timer instead of allocating time.After each iteration under contention. - governor: extract the process-wide governor + Gate helper into the limiter package so chat and vlm share one limiter/limit without importing each other; add an in-process localLimiter for Lite mode. - vlm: gate background multimodal (VLM) calls through the same governor; previously only chat was throttled. - chat: ChatStream releases its slot when the consumer abandons the stream and cancels ctx (select on ctx.Done, drain inner), fixing a slot/goroutine leak. - container/sync_task: install the local governor in Lite mode and tag SyncTaskExecutor runs as background; honour model.max_concurrency=0 as an explicit disable instead of falling back to the default. - tests: cover the governor gating, local limiter, and stream release.
This commit is contained in:
@@ -287,6 +287,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// dispatches inline goroutines that the checkpoint-based abort
|
||||
// already handles.
|
||||
must(container.Provide(router.NewNoopTaskInspector))
|
||||
// Even without Redis, background ingestion/enrichment can burst the
|
||||
// worker pool against one provider, so install an in-process governor.
|
||||
must(container.Invoke(registerLiteModelConcurrencyLimiter))
|
||||
}
|
||||
|
||||
// Chat pipeline components for processing chat requests
|
||||
@@ -448,21 +451,47 @@ func initLangfuse() (*langfuse.Manager, error) {
|
||||
// across every replica. Interactive chat is never gated.
|
||||
const defaultModelMaxConcurrency = 8
|
||||
|
||||
// registerModelConcurrencyLimiter builds the Redis-backed per-model chat
|
||||
// concurrency governor and installs it into the chat package. The limit is
|
||||
// resolved from system settings / env, defaulting to defaultModelMaxConcurrency.
|
||||
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
|
||||
limit := defaultModelMaxConcurrency
|
||||
if ss != nil {
|
||||
n := ss.GetInt(context.Background(), "model.max_concurrency",
|
||||
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency))
|
||||
if n > 0 {
|
||||
limit = int(n)
|
||||
}
|
||||
// resolveModelMaxConcurrency reads the per-model background concurrency limit
|
||||
// from system settings / env, defaulting to defaultModelMaxConcurrency when
|
||||
// unset. A configured value of 0 (or negative) is honoured and disables the
|
||||
// governor — that is the supported way to turn throttling off via config/env.
|
||||
func resolveModelMaxConcurrency(ss interfaces.SystemSettingService) int {
|
||||
if ss == nil {
|
||||
return defaultModelMaxConcurrency
|
||||
}
|
||||
return int(ss.GetInt(context.Background(), "model.max_concurrency",
|
||||
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency)))
|
||||
}
|
||||
|
||||
// registerModelConcurrencyLimiter builds the Redis-backed per-model background
|
||||
// concurrency governor (chat + vlm) and installs it. Only available with Redis
|
||||
// (the shared semaphore backend); Lite mode uses registerLiteModelConcurrencyLimiter.
|
||||
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
|
||||
limit := resolveModelMaxConcurrency(ss)
|
||||
limiter.SetGovernor(limiter.NewRedisLimiter(rdb), limit)
|
||||
if limit <= 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
|
||||
return
|
||||
}
|
||||
chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit)
|
||||
logger.Infof(context.Background(),
|
||||
"[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit)
|
||||
"[ModelLimiter] background model concurrency governed per-model, limit=%d (distributed via redis)", limit)
|
||||
}
|
||||
|
||||
// registerLiteModelConcurrencyLimiter installs an in-process per-model governor
|
||||
// for Lite mode (no Redis). Lite runs a single process, so an in-process
|
||||
// semaphore is sufficient to keep a background ingestion storm from bursting
|
||||
// the whole worker pool against one provider.
|
||||
func registerLiteModelConcurrencyLimiter(ss interfaces.SystemSettingService) {
|
||||
limit := resolveModelMaxConcurrency(ss)
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), limit)
|
||||
if limit <= 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)")
|
||||
return
|
||||
}
|
||||
logger.Infof(context.Background(),
|
||||
"[ModelLimiter] background model concurrency governed per-model, limit=%d (in-process, lite mode)", limit)
|
||||
}
|
||||
|
||||
func initRedisClient() (*redis.Client, error) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -18,29 +17,8 @@ import (
|
||||
// Only background (asynq worker) calls are throttled; interactive chat is left
|
||||
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
|
||||
// exhaust the provider yet user-facing latency is never gated behind the
|
||||
// semaphore.
|
||||
var (
|
||||
concurrencyMu sync.RWMutex
|
||||
concurrencyLimiter limiter.ModelConcurrencyLimiter
|
||||
concurrencyLimit int
|
||||
)
|
||||
|
||||
// SetConcurrencyLimiter installs the process-wide background chat concurrency
|
||||
// governor and the default per-model limit. Called once at startup (see
|
||||
// container wiring). Passing a nil limiter or a non-positive limit disables
|
||||
// governance (all calls pass through).
|
||||
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
|
||||
concurrencyMu.Lock()
|
||||
defer concurrencyMu.Unlock()
|
||||
concurrencyLimiter = l
|
||||
concurrencyLimit = defaultLimit
|
||||
}
|
||||
|
||||
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
|
||||
concurrencyMu.RLock()
|
||||
defer concurrencyMu.RUnlock()
|
||||
return concurrencyLimiter, concurrencyLimit
|
||||
}
|
||||
// semaphore. The governor singleton itself lives in the limiter package so chat
|
||||
// and vlm share the same limiter and per-model budget.
|
||||
|
||||
// concurrencyChat throttles background LLM calls through a per-model
|
||||
// distributed semaphore. It is the outermost wrapper so the slot is held only
|
||||
@@ -53,42 +31,38 @@ type concurrencyChat struct {
|
||||
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
// gate acquires a concurrency slot when the call is a background task and a
|
||||
// limiter is installed. Returns a release func (always safe to call) and
|
||||
// whether the inner call may proceed — it always may; the gate never blocks a
|
||||
// call permanently and fails open on any limiter error.
|
||||
func (w *concurrencyChat) gate(ctx context.Context) func() {
|
||||
lim, limit := getConcurrencyLimiter()
|
||||
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return func() {}
|
||||
}
|
||||
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
|
||||
if err != nil || release == nil {
|
||||
return func() {}
|
||||
}
|
||||
return release
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
defer release()
|
||||
return w.inner.Chat(ctx, messages, opts)
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
||||
if err != nil || ch == nil {
|
||||
release()
|
||||
return ch, err
|
||||
}
|
||||
// Hold the slot until the stream fully drains, then release.
|
||||
// Hold the slot until the stream fully drains, then release. If the
|
||||
// consumer abandons the stream (stops reading out) we would otherwise
|
||||
// block forever on the send and never release the slot; select on
|
||||
// ctx.Done() so a cancelled call frees its slot promptly, and drain the
|
||||
// inner channel in the background so the upstream producer can exit.
|
||||
out := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer release()
|
||||
for resp := range ch {
|
||||
out <- resp
|
||||
select {
|
||||
case out <- resp:
|
||||
case <-ctx.Done():
|
||||
go func() {
|
||||
for range ch {
|
||||
}
|
||||
}()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// fakeChat is a minimal Chat whose stream emits continuously until ctx is done,
|
||||
// so we can exercise the concurrency wrapper's slot lifecycle.
|
||||
type fakeChat struct{ id string }
|
||||
|
||||
func (f *fakeChat) GetModelName() string { return f.id }
|
||||
func (f *fakeChat) GetModelID() string { return f.id }
|
||||
|
||||
func (f *fakeChat) Chat(ctx context.Context, _ []Message, _ *ChatOptions) (*types.ChatResponse, error) {
|
||||
return &types.ChatResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeChat) ChatStream(ctx context.Context, _ []Message, _ *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
ch := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- types.StreamResponse{}:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// TestConcurrencyChatInteractiveNotGated verifies interactive calls bypass the
|
||||
// governor entirely even at limit 1.
|
||||
func TestConcurrencyChatInteractiveNotGated(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
w := &concurrencyChat{inner: &fakeChat{id: "model-x"}}
|
||||
// No background marker: neither call should be throttled.
|
||||
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
|
||||
t.Fatalf("interactive chat: %v", err)
|
||||
}
|
||||
if _, err := w.Chat(context.Background(), nil, nil); err != nil {
|
||||
t.Fatalf("interactive chat 2: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrencyChatStreamReleasesOnAbandon verifies that when the consumer
|
||||
// stops reading and cancels the context, the held slot is released rather than
|
||||
// leaked (the #9 fix).
|
||||
func TestConcurrencyChatStreamReleasesOnAbandon(t *testing.T) {
|
||||
t.Cleanup(func() { limiter.SetGovernor(nil, 0) })
|
||||
limiter.SetGovernor(limiter.NewLocalLimiter(), 1)
|
||||
|
||||
const id = "model-y"
|
||||
w := &concurrencyChat{inner: &fakeChat{id: id}}
|
||||
|
||||
streamCtx, cancel := context.WithCancel(types.WithBackgroundTask(context.Background()))
|
||||
out, err := w.ChatStream(streamCtx, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("stream: %v", err)
|
||||
}
|
||||
// Consume one item so the relay goroutine is running and holding the slot.
|
||||
<-out
|
||||
|
||||
// The single slot is now held: a background acquire must block.
|
||||
blocked := make(chan struct{})
|
||||
go func() {
|
||||
rel := limiter.Gate(types.WithBackgroundTask(context.Background()), id)
|
||||
rel()
|
||||
close(blocked)
|
||||
}()
|
||||
select {
|
||||
case <-blocked:
|
||||
t.Fatal("slot should be held by the live stream")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Abandon the stream: stop reading and cancel. The relay must release.
|
||||
cancel()
|
||||
select {
|
||||
case <-blocked:
|
||||
// released and reacquired successfully
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("stream slot leaked: not released after consumer abandoned + cancelled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// The concurrency governor is process-wide, shared by every model-client layer
|
||||
// that fronts a provider (chat, vlm). Keeping the singleton here — rather than
|
||||
// inside one client package — lets all of them gate against the same limiter
|
||||
// and per-model limit without importing each other. Wired once at startup (see
|
||||
// container.registerModelConcurrencyLimiter) via SetGovernor.
|
||||
var (
|
||||
governorMu sync.RWMutex
|
||||
governor ModelConcurrencyLimiter
|
||||
governorN int
|
||||
)
|
||||
|
||||
// SetGovernor installs the process-wide background concurrency governor and the
|
||||
// default per-model limit. Passing a nil limiter or a non-positive limit
|
||||
// disables governance (all calls pass through). Safe to call at startup.
|
||||
func SetGovernor(l ModelConcurrencyLimiter, limit int) {
|
||||
governorMu.Lock()
|
||||
defer governorMu.Unlock()
|
||||
governor = l
|
||||
governorN = limit
|
||||
}
|
||||
|
||||
// Gate acquires a per-model concurrency slot when the call is a background task
|
||||
// (see types.IsBackgroundTask) and a governor is installed. It returns a
|
||||
// release func that is ALWAYS safe to call: on the passthrough / fail-open
|
||||
// paths it is a cheap no-op. The gate never blocks a call permanently — a
|
||||
// limiter/Redis outage or a cancelled context fails open.
|
||||
func Gate(ctx context.Context, modelID string) func() {
|
||||
governorMu.RLock()
|
||||
l, limit := governor, governorN
|
||||
governorMu.RUnlock()
|
||||
|
||||
if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return noop
|
||||
}
|
||||
release, err := l.Acquire(ctx, modelID, limit)
|
||||
if err != nil || release == nil {
|
||||
return noop
|
||||
}
|
||||
return release
|
||||
}
|
||||
|
||||
// localLimiter is an in-process (single-node) counting semaphore keyed by
|
||||
// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
|
||||
// single process with no Redis, so a shared distributed semaphore is neither
|
||||
// available nor needed — but background ingestion can still burst the whole
|
||||
// worker pool against one provider, so we still cap concurrency locally.
|
||||
type localLimiter struct {
|
||||
mu sync.Mutex
|
||||
sems map[string]chan struct{}
|
||||
}
|
||||
|
||||
// NewLocalLimiter builds an in-process per-key concurrency limiter.
|
||||
func NewLocalLimiter() ModelConcurrencyLimiter {
|
||||
return &localLimiter{sems: make(map[string]chan struct{})}
|
||||
}
|
||||
|
||||
func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
|
||||
if l == nil || limit <= 0 || key == "" {
|
||||
return noop, nil
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
sem, ok := l.sems[key]
|
||||
if !ok {
|
||||
// Capacity is fixed at first use for a key; the limit is a
|
||||
// process-wide constant, so it never changes across acquires.
|
||||
sem = make(chan struct{}, limit)
|
||||
l.sems[key] = sem
|
||||
}
|
||||
l.mu.Unlock()
|
||||
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-sem }) }, nil
|
||||
case <-ctx.Done():
|
||||
// Fail open on cancellation, mirroring the Redis limiter.
|
||||
return noop, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func withBackground() context.Context {
|
||||
return types.WithBackgroundTask(context.Background())
|
||||
}
|
||||
|
||||
// TestGateOnlyGovernsBackground verifies interactive calls always pass through
|
||||
// (no-op release) and only background calls consult the installed limiter.
|
||||
func TestGateOnlyGovernsBackground(t *testing.T) {
|
||||
t.Cleanup(func() { SetGovernor(nil, 0) })
|
||||
SetGovernor(NewLocalLimiter(), 1)
|
||||
|
||||
// Interactive (non-background) ctx: never gated, even at limit 1.
|
||||
rel1 := Gate(context.Background(), "m")
|
||||
rel2 := Gate(context.Background(), "m")
|
||||
rel1()
|
||||
rel2()
|
||||
|
||||
// Background ctx: gated. First slot taken; second would block, so a
|
||||
// cancelled second acquire must fail open (return a usable release).
|
||||
held := Gate(withBackground(), "m")
|
||||
defer held()
|
||||
|
||||
ctx, cancel := context.WithTimeout(withBackground(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
rel := Gate(ctx, "m")
|
||||
if rel == nil {
|
||||
t.Fatal("cancelled background gate must fail open with a usable release")
|
||||
}
|
||||
rel()
|
||||
}
|
||||
|
||||
// TestGateDisabledWhenNoGovernor verifies Gate is a passthrough when no
|
||||
// governor is installed.
|
||||
func TestGateDisabledWhenNoGovernor(t *testing.T) {
|
||||
SetGovernor(nil, 0)
|
||||
if rel := Gate(withBackground(), "m"); rel == nil {
|
||||
t.Fatal("gate with no governor must return a usable release")
|
||||
} else {
|
||||
rel()
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLimiterCaps verifies the in-process limiter enforces the cap and
|
||||
// releases slots.
|
||||
func TestLocalLimiterCaps(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
|
||||
r1, _ := l.Acquire(ctx, "k", 2)
|
||||
r2, _ := l.Acquire(ctx, "k", 2)
|
||||
|
||||
// Third acquire must block while full.
|
||||
done := make(chan func(), 1)
|
||||
go func() {
|
||||
r3, _ := l.Acquire(ctx, "k", 2)
|
||||
done <- r3
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("third acquire should block while at limit")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
r1() // free a slot; the waiter proceeds
|
||||
select {
|
||||
case r3 := <-done:
|
||||
r3()
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("waiter should proceed after a slot frees")
|
||||
}
|
||||
r2()
|
||||
}
|
||||
|
||||
// TestLocalLimiterIndependentKeys verifies per-key budgets are independent.
|
||||
func TestLocalLimiterIndependentKeys(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
ra, _ := l.Acquire(ctx, "a", 1)
|
||||
rb, _ := l.Acquire(ctx, "b", 1) // different key: must not block
|
||||
ra()
|
||||
rb()
|
||||
}
|
||||
|
||||
// TestLocalLimiterFailOpen verifies degraded inputs pass through.
|
||||
func TestLocalLimiterFailOpen(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
if r, err := l.Acquire(ctx, "", 4); err != nil || r == nil {
|
||||
t.Fatal("empty key should fail open")
|
||||
} else {
|
||||
r()
|
||||
}
|
||||
if r, err := l.Acquire(ctx, "k", 0); err != nil || r == nil {
|
||||
t.Fatal("limit<=0 should fail open")
|
||||
} else {
|
||||
r()
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLimiterReleaseIdempotent verifies double release doesn't
|
||||
// over-free a slot.
|
||||
func TestLocalLimiterReleaseIdempotent(t *testing.T) {
|
||||
l := NewLocalLimiter()
|
||||
ctx := context.Background()
|
||||
r1, _ := l.Acquire(ctx, "k", 1)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); r1() }()
|
||||
go func() { defer wg.Done(); r1() }()
|
||||
wg.Wait()
|
||||
|
||||
// After idempotent release the single slot is free again.
|
||||
r2, _ := l.Acquire(ctx, "k", 1)
|
||||
done := make(chan struct{})
|
||||
go func() { r3, _ := l.Acquire(ctx, "k", 1); r3(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("a double-release must not leave the slot permanently free")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
r2()
|
||||
<-done
|
||||
}
|
||||
@@ -96,6 +96,16 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
token := uuid.NewString()
|
||||
ttlMs := l.ttl.Milliseconds()
|
||||
|
||||
// Reuse a single timer across poll iterations rather than allocating a new
|
||||
// one via time.After each loop: under sustained contention a waiter can
|
||||
// spin thousands of times, and every time.After timer lives until it fires.
|
||||
// Start it stopped so the first Reset below arms it cleanly.
|
||||
timer := time.NewTimer(0)
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
now := time.Now().UnixMilli()
|
||||
res, err := acquireScript.Run(ctx, l.rdb, []string{zkey},
|
||||
@@ -109,13 +119,14 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func
|
||||
return l.hold(zkey, token), nil
|
||||
}
|
||||
|
||||
timer.Reset(l.pollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Fail open on cancellation too: let the inner call observe the
|
||||
// cancelled context and return its own error, rather than us
|
||||
// synthesising one here.
|
||||
return noop, nil
|
||||
case <-time.After(l.pollInterval):
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,10 +147,19 @@ func (l *redisLimiter) hold(zkey, token string) func() {
|
||||
// Detached ctx: the heartbeat must outlive request ctx up to
|
||||
// release. Best-effort; a failed refresh just risks early
|
||||
// reclamation, which the limit already tolerates.
|
||||
_ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{
|
||||
//
|
||||
// Refresh BOTH the member lease score AND the ZSET key's own
|
||||
// TTL. The acquire script only PEXPIREs the key on admission,
|
||||
// so a semaphore that stays saturated with no slot turnover
|
||||
// would otherwise let the whole key expire after ttl*2 —
|
||||
// dropping every live lease and admitting over the limit. The
|
||||
// heartbeat pushes the key TTL out in lockstep with the lease.
|
||||
bg := context.Background()
|
||||
_ = l.rdb.ZAdd(bg, zkey, redis.Z{
|
||||
Score: float64(now + l.ttl.Milliseconds()),
|
||||
Member: token,
|
||||
}).Err()
|
||||
_ = l.rdb.PExpire(bg, zkey, l.ttl*2).Err()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package vlm
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
)
|
||||
|
||||
// Multimodal enrichment (image OCR / caption) is a high-volume, slow background
|
||||
// stage that hits the same provider budget as chat. Like chat, it must be
|
||||
// governed at the client layer so an image-heavy ingestion storm can't burst
|
||||
// the whole worker pool against one VLM provider. Only background (asynq
|
||||
// worker) calls are throttled — see limiter.Gate / types.IsBackgroundTask.
|
||||
type concurrencyVLM struct {
|
||||
inner VLM
|
||||
}
|
||||
|
||||
func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
release := limiter.Gate(ctx, w.inner.GetModelID())
|
||||
defer release()
|
||||
return w.inner.Predict(ctx, imgBytes, prompt)
|
||||
}
|
||||
|
||||
// wrapVLMConcurrency installs the background concurrency governor as the
|
||||
// outermost VLM decorator. Always applied; a cheap passthrough when no limiter
|
||||
// is installed or the call is interactive.
|
||||
func wrapVLMConcurrency(v VLM, err error) (VLM, error) {
|
||||
if err != nil || v == nil {
|
||||
return v, err
|
||||
}
|
||||
return &concurrencyVLM{inner: v}, nil
|
||||
}
|
||||
@@ -87,7 +87,10 @@ func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
||||
if logger.LLMDebugEnabled() {
|
||||
v = &debugVLM{inner: v}
|
||||
}
|
||||
return wrapVLMLangfuse(v, nil)
|
||||
v, err = wrapVLMLangfuse(v, nil)
|
||||
// Outermost: hold the per-model concurrency slot only around the real
|
||||
// provider round-trip, so the wait is excluded from debug/langfuse timing.
|
||||
return wrapVLMConcurrency(v, err)
|
||||
}
|
||||
|
||||
func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
||||
|
||||
@@ -80,7 +80,10 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asy
|
||||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Tag as a background worker execution so the per-model concurrency
|
||||
// governor throttles Lite-mode ingestion/enrichment LLM calls, mirroring
|
||||
// the asynq backgroundTaskMiddleware in the Redis path.
|
||||
ctx := types.WithBackgroundTask(context.Background())
|
||||
start := time.Now()
|
||||
logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user