From 98ad41cda42cf2b0886e9c8dc1b66b08a2f05e73 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Jul 2026 18:30:44 +0800 Subject: [PATCH] fix(models): harden background concurrency governor Follow-up hardening on the per-model background concurrency governor: - limiter: heartbeat now refreshes the ZSET key's PEXPIRE alongside the lease score, so a semaphore that stays saturated with no slot turnover can't let the whole key expire (which would drop every lease and admit over the limit); reuse one poll timer instead of allocating time.After each iteration under contention. - governor: extract the process-wide governor + Gate helper into the limiter package so chat and vlm share one limiter/limit without importing each other; add an in-process localLimiter for Lite mode. - vlm: gate background multimodal (VLM) calls through the same governor; previously only chat was throttled. - chat: ChatStream releases its slot when the consumer abandons the stream and cancels ctx (select on ctx.Done, drain inner), fixing a slot/goroutine leak. - container/sync_task: install the local governor in Lite mode and tag SyncTaskExecutor runs as background; honour model.max_concurrency=0 as an explicit disable instead of falling back to the default. - tests: cover the governor gating, local limiter, and stream release. --- internal/container/container.go | 55 ++++++-- internal/models/chat/concurrency_wrapper.go | 62 +++----- .../models/chat/concurrency_wrapper_test.go | 93 ++++++++++++ internal/models/limiter/governor.go | 89 ++++++++++++ internal/models/limiter/governor_test.go | 133 ++++++++++++++++++ internal/models/limiter/limiter.go | 24 +++- internal/models/vlm/concurrency_wrapper.go | 35 +++++ internal/models/vlm/vlm.go | 5 +- internal/router/sync_task.go | 5 +- 9 files changed, 440 insertions(+), 61 deletions(-) create mode 100644 internal/models/chat/concurrency_wrapper_test.go create mode 100644 internal/models/limiter/governor.go create mode 100644 internal/models/limiter/governor_test.go create mode 100644 internal/models/vlm/concurrency_wrapper.go diff --git a/internal/container/container.go b/internal/container/container.go index f77d5843..0fdb8e27 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -287,6 +287,9 @@ func BuildContainer(container *dig.Container) *dig.Container { // dispatches inline goroutines that the checkpoint-based abort // already handles. must(container.Provide(router.NewNoopTaskInspector)) + // Even without Redis, background ingestion/enrichment can burst the + // worker pool against one provider, so install an in-process governor. + must(container.Invoke(registerLiteModelConcurrencyLimiter)) } // Chat pipeline components for processing chat requests @@ -448,21 +451,47 @@ func initLangfuse() (*langfuse.Manager, error) { // across every replica. Interactive chat is never gated. const defaultModelMaxConcurrency = 8 -// registerModelConcurrencyLimiter builds the Redis-backed per-model chat -// concurrency governor and installs it into the chat package. The limit is -// resolved from system settings / env, defaulting to defaultModelMaxConcurrency. -func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) { - limit := defaultModelMaxConcurrency - if ss != nil { - n := ss.GetInt(context.Background(), "model.max_concurrency", - "WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency)) - if n > 0 { - limit = int(n) - } +// resolveModelMaxConcurrency reads the per-model background concurrency limit +// from system settings / env, defaulting to defaultModelMaxConcurrency when +// unset. A configured value of 0 (or negative) is honoured and disables the +// governor — that is the supported way to turn throttling off via config/env. +func resolveModelMaxConcurrency(ss interfaces.SystemSettingService) int { + if ss == nil { + return defaultModelMaxConcurrency + } + return int(ss.GetInt(context.Background(), "model.max_concurrency", + "WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency))) +} + +// registerModelConcurrencyLimiter builds the Redis-backed per-model background +// concurrency governor (chat + vlm) and installs it. Only available with Redis +// (the shared semaphore backend); Lite mode uses registerLiteModelConcurrencyLimiter. +func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) { + limit := resolveModelMaxConcurrency(ss) + limiter.SetGovernor(limiter.NewRedisLimiter(rdb), limit) + if limit <= 0 { + logger.Infof(context.Background(), + "[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)") + return } - chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit) logger.Infof(context.Background(), - "[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit) + "[ModelLimiter] background model concurrency governed per-model, limit=%d (distributed via redis)", limit) +} + +// registerLiteModelConcurrencyLimiter installs an in-process per-model governor +// for Lite mode (no Redis). Lite runs a single process, so an in-process +// semaphore is sufficient to keep a background ingestion storm from bursting +// the whole worker pool against one provider. +func registerLiteModelConcurrencyLimiter(ss interfaces.SystemSettingService) { + limit := resolveModelMaxConcurrency(ss) + limiter.SetGovernor(limiter.NewLocalLimiter(), limit) + if limit <= 0 { + logger.Infof(context.Background(), + "[ModelLimiter] background concurrency governor DISABLED (model.max_concurrency<=0)") + return + } + logger.Infof(context.Background(), + "[ModelLimiter] background model concurrency governed per-model, limit=%d (in-process, lite mode)", limit) } func initRedisClient() (*redis.Client, error) { diff --git a/internal/models/chat/concurrency_wrapper.go b/internal/models/chat/concurrency_wrapper.go index d04823a0..09862e3a 100644 --- a/internal/models/chat/concurrency_wrapper.go +++ b/internal/models/chat/concurrency_wrapper.go @@ -2,7 +2,6 @@ package chat import ( "context" - "sync" "github.com/Tencent/WeKnora/internal/models/limiter" "github.com/Tencent/WeKnora/internal/types" @@ -18,29 +17,8 @@ import ( // Only background (asynq worker) calls are throttled; interactive chat is left // untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot // exhaust the provider yet user-facing latency is never gated behind the -// semaphore. -var ( - concurrencyMu sync.RWMutex - concurrencyLimiter limiter.ModelConcurrencyLimiter - concurrencyLimit int -) - -// SetConcurrencyLimiter installs the process-wide background chat concurrency -// governor and the default per-model limit. Called once at startup (see -// container wiring). Passing a nil limiter or a non-positive limit disables -// governance (all calls pass through). -func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) { - concurrencyMu.Lock() - defer concurrencyMu.Unlock() - concurrencyLimiter = l - concurrencyLimit = defaultLimit -} - -func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) { - concurrencyMu.RLock() - defer concurrencyMu.RUnlock() - return concurrencyLimiter, concurrencyLimit -} +// semaphore. The governor singleton itself lives in the limiter package so chat +// and vlm share the same limiter and per-model budget. // concurrencyChat throttles background LLM calls through a per-model // distributed semaphore. It is the outermost wrapper so the slot is held only @@ -53,42 +31,38 @@ type concurrencyChat struct { func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() } func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() } -// gate acquires a concurrency slot when the call is a background task and a -// limiter is installed. Returns a release func (always safe to call) and -// whether the inner call may proceed — it always may; the gate never blocks a -// call permanently and fails open on any limiter error. -func (w *concurrencyChat) gate(ctx context.Context) func() { - lim, limit := getConcurrencyLimiter() - if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) { - return func() {} - } - release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit) - if err != nil || release == nil { - return func() {} - } - return release -} - func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) { - release := w.gate(ctx) + release := limiter.Gate(ctx, w.inner.GetModelID()) defer release() return w.inner.Chat(ctx, messages, opts) } func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) { - release := w.gate(ctx) + release := limiter.Gate(ctx, w.inner.GetModelID()) ch, err := w.inner.ChatStream(ctx, messages, opts) if err != nil || ch == nil { release() return ch, err } - // Hold the slot until the stream fully drains, then release. + // Hold the slot until the stream fully drains, then release. If the + // consumer abandons the stream (stops reading out) we would otherwise + // block forever on the send and never release the slot; select on + // ctx.Done() so a cancelled call frees its slot promptly, and drain the + // inner channel in the background so the upstream producer can exit. out := make(chan types.StreamResponse) go func() { defer close(out) defer release() for resp := range ch { - out <- resp + select { + case out <- resp: + case <-ctx.Done(): + go func() { + for range ch { + } + }() + return + } } }() return out, nil diff --git a/internal/models/chat/concurrency_wrapper_test.go b/internal/models/chat/concurrency_wrapper_test.go new file mode 100644 index 00000000..3ce5cbe3 --- /dev/null +++ b/internal/models/chat/concurrency_wrapper_test.go @@ -0,0 +1,93 @@ +package chat + +import ( + "context" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/models/limiter" + "github.com/Tencent/WeKnora/internal/types" +) + +// fakeChat is a minimal Chat whose stream emits continuously until ctx is done, +// so we can exercise the concurrency wrapper's slot lifecycle. +type fakeChat struct{ id string } + +func (f *fakeChat) GetModelName() string { return f.id } +func (f *fakeChat) GetModelID() string { return f.id } + +func (f *fakeChat) Chat(ctx context.Context, _ []Message, _ *ChatOptions) (*types.ChatResponse, error) { + return &types.ChatResponse{}, nil +} + +func (f *fakeChat) ChatStream(ctx context.Context, _ []Message, _ *ChatOptions) (<-chan types.StreamResponse, error) { + ch := make(chan types.StreamResponse) + go func() { + defer close(ch) + for { + select { + case <-ctx.Done(): + return + case ch <- types.StreamResponse{}: + } + } + }() + return ch, nil +} + +// TestConcurrencyChatInteractiveNotGated verifies interactive calls bypass the +// governor entirely even at limit 1. +func TestConcurrencyChatInteractiveNotGated(t *testing.T) { + t.Cleanup(func() { limiter.SetGovernor(nil, 0) }) + limiter.SetGovernor(limiter.NewLocalLimiter(), 1) + + w := &concurrencyChat{inner: &fakeChat{id: "model-x"}} + // No background marker: neither call should be throttled. + if _, err := w.Chat(context.Background(), nil, nil); err != nil { + t.Fatalf("interactive chat: %v", err) + } + if _, err := w.Chat(context.Background(), nil, nil); err != nil { + t.Fatalf("interactive chat 2: %v", err) + } +} + +// TestConcurrencyChatStreamReleasesOnAbandon verifies that when the consumer +// stops reading and cancels the context, the held slot is released rather than +// leaked (the #9 fix). +func TestConcurrencyChatStreamReleasesOnAbandon(t *testing.T) { + t.Cleanup(func() { limiter.SetGovernor(nil, 0) }) + limiter.SetGovernor(limiter.NewLocalLimiter(), 1) + + const id = "model-y" + w := &concurrencyChat{inner: &fakeChat{id: id}} + + streamCtx, cancel := context.WithCancel(types.WithBackgroundTask(context.Background())) + out, err := w.ChatStream(streamCtx, nil, nil) + if err != nil { + t.Fatalf("stream: %v", err) + } + // Consume one item so the relay goroutine is running and holding the slot. + <-out + + // The single slot is now held: a background acquire must block. + blocked := make(chan struct{}) + go func() { + rel := limiter.Gate(types.WithBackgroundTask(context.Background()), id) + rel() + close(blocked) + }() + select { + case <-blocked: + t.Fatal("slot should be held by the live stream") + case <-time.After(50 * time.Millisecond): + } + + // Abandon the stream: stop reading and cancel. The relay must release. + cancel() + select { + case <-blocked: + // released and reacquired successfully + case <-time.After(2 * time.Second): + t.Fatal("stream slot leaked: not released after consumer abandoned + cancelled") + } +} diff --git a/internal/models/limiter/governor.go b/internal/models/limiter/governor.go new file mode 100644 index 00000000..88598965 --- /dev/null +++ b/internal/models/limiter/governor.go @@ -0,0 +1,89 @@ +package limiter + +import ( + "context" + "sync" + + "github.com/Tencent/WeKnora/internal/types" +) + +// The concurrency governor is process-wide, shared by every model-client layer +// that fronts a provider (chat, vlm). Keeping the singleton here — rather than +// inside one client package — lets all of them gate against the same limiter +// and per-model limit without importing each other. Wired once at startup (see +// container.registerModelConcurrencyLimiter) via SetGovernor. +var ( + governorMu sync.RWMutex + governor ModelConcurrencyLimiter + governorN int +) + +// SetGovernor installs the process-wide background concurrency governor and the +// default per-model limit. Passing a nil limiter or a non-positive limit +// disables governance (all calls pass through). Safe to call at startup. +func SetGovernor(l ModelConcurrencyLimiter, limit int) { + governorMu.Lock() + defer governorMu.Unlock() + governor = l + governorN = limit +} + +// Gate acquires a per-model concurrency slot when the call is a background task +// (see types.IsBackgroundTask) and a governor is installed. It returns a +// release func that is ALWAYS safe to call: on the passthrough / fail-open +// paths it is a cheap no-op. The gate never blocks a call permanently — a +// limiter/Redis outage or a cancelled context fails open. +func Gate(ctx context.Context, modelID string) func() { + governorMu.RLock() + l, limit := governor, governorN + governorMu.RUnlock() + + if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) { + return noop + } + release, err := l.Acquire(ctx, modelID, limit) + if err != nil || release == nil { + return noop + } + return release +} + +// localLimiter is an in-process (single-node) counting semaphore keyed by +// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a +// single process with no Redis, so a shared distributed semaphore is neither +// available nor needed — but background ingestion can still burst the whole +// worker pool against one provider, so we still cap concurrency locally. +type localLimiter struct { + mu sync.Mutex + sems map[string]chan struct{} +} + +// NewLocalLimiter builds an in-process per-key concurrency limiter. +func NewLocalLimiter() ModelConcurrencyLimiter { + return &localLimiter{sems: make(map[string]chan struct{})} +} + +func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) { + if l == nil || limit <= 0 || key == "" { + return noop, nil + } + + l.mu.Lock() + sem, ok := l.sems[key] + if !ok { + // Capacity is fixed at first use for a key; the limit is a + // process-wide constant, so it never changes across acquires. + sem = make(chan struct{}, limit) + l.sems[key] = sem + } + l.mu.Unlock() + + select { + case sem <- struct{}{}: + var once sync.Once + return func() { once.Do(func() { <-sem }) }, nil + case <-ctx.Done(): + // Fail open on cancellation, mirroring the Redis limiter. + return noop, nil + } +} diff --git a/internal/models/limiter/governor_test.go b/internal/models/limiter/governor_test.go new file mode 100644 index 00000000..06e97c1d --- /dev/null +++ b/internal/models/limiter/governor_test.go @@ -0,0 +1,133 @@ +package limiter + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func withBackground() context.Context { + return types.WithBackgroundTask(context.Background()) +} + +// TestGateOnlyGovernsBackground verifies interactive calls always pass through +// (no-op release) and only background calls consult the installed limiter. +func TestGateOnlyGovernsBackground(t *testing.T) { + t.Cleanup(func() { SetGovernor(nil, 0) }) + SetGovernor(NewLocalLimiter(), 1) + + // Interactive (non-background) ctx: never gated, even at limit 1. + rel1 := Gate(context.Background(), "m") + rel2 := Gate(context.Background(), "m") + rel1() + rel2() + + // Background ctx: gated. First slot taken; second would block, so a + // cancelled second acquire must fail open (return a usable release). + held := Gate(withBackground(), "m") + defer held() + + ctx, cancel := context.WithTimeout(withBackground(), 30*time.Millisecond) + defer cancel() + rel := Gate(ctx, "m") + if rel == nil { + t.Fatal("cancelled background gate must fail open with a usable release") + } + rel() +} + +// TestGateDisabledWhenNoGovernor verifies Gate is a passthrough when no +// governor is installed. +func TestGateDisabledWhenNoGovernor(t *testing.T) { + SetGovernor(nil, 0) + if rel := Gate(withBackground(), "m"); rel == nil { + t.Fatal("gate with no governor must return a usable release") + } else { + rel() + } +} + +// TestLocalLimiterCaps verifies the in-process limiter enforces the cap and +// releases slots. +func TestLocalLimiterCaps(t *testing.T) { + l := NewLocalLimiter() + ctx := context.Background() + + r1, _ := l.Acquire(ctx, "k", 2) + r2, _ := l.Acquire(ctx, "k", 2) + + // Third acquire must block while full. + done := make(chan func(), 1) + go func() { + r3, _ := l.Acquire(ctx, "k", 2) + done <- r3 + }() + select { + case <-done: + t.Fatal("third acquire should block while at limit") + case <-time.After(50 * time.Millisecond): + } + + r1() // free a slot; the waiter proceeds + select { + case r3 := <-done: + r3() + case <-time.After(time.Second): + t.Fatal("waiter should proceed after a slot frees") + } + r2() +} + +// TestLocalLimiterIndependentKeys verifies per-key budgets are independent. +func TestLocalLimiterIndependentKeys(t *testing.T) { + l := NewLocalLimiter() + ctx := context.Background() + ra, _ := l.Acquire(ctx, "a", 1) + rb, _ := l.Acquire(ctx, "b", 1) // different key: must not block + ra() + rb() +} + +// TestLocalLimiterFailOpen verifies degraded inputs pass through. +func TestLocalLimiterFailOpen(t *testing.T) { + l := NewLocalLimiter() + ctx := context.Background() + if r, err := l.Acquire(ctx, "", 4); err != nil || r == nil { + t.Fatal("empty key should fail open") + } else { + r() + } + if r, err := l.Acquire(ctx, "k", 0); err != nil || r == nil { + t.Fatal("limit<=0 should fail open") + } else { + r() + } +} + +// TestLocalLimiterReleaseIdempotent verifies double release doesn't +// over-free a slot. +func TestLocalLimiterReleaseIdempotent(t *testing.T) { + l := NewLocalLimiter() + ctx := context.Background() + r1, _ := l.Acquire(ctx, "k", 1) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); r1() }() + go func() { defer wg.Done(); r1() }() + wg.Wait() + + // After idempotent release the single slot is free again. + r2, _ := l.Acquire(ctx, "k", 1) + done := make(chan struct{}) + go func() { r3, _ := l.Acquire(ctx, "k", 1); r3(); close(done) }() + select { + case <-done: + t.Fatal("a double-release must not leave the slot permanently free") + case <-time.After(50 * time.Millisecond): + } + r2() + <-done +} diff --git a/internal/models/limiter/limiter.go b/internal/models/limiter/limiter.go index 1b20c612..fb9b4b92 100644 --- a/internal/models/limiter/limiter.go +++ b/internal/models/limiter/limiter.go @@ -96,6 +96,16 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func token := uuid.NewString() ttlMs := l.ttl.Milliseconds() + // Reuse a single timer across poll iterations rather than allocating a new + // one via time.After each loop: under sustained contention a waiter can + // spin thousands of times, and every time.After timer lives until it fires. + // Start it stopped so the first Reset below arms it cleanly. + timer := time.NewTimer(0) + if !timer.Stop() { + <-timer.C + } + defer timer.Stop() + for { now := time.Now().UnixMilli() res, err := acquireScript.Run(ctx, l.rdb, []string{zkey}, @@ -109,13 +119,14 @@ func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func return l.hold(zkey, token), nil } + timer.Reset(l.pollInterval) select { case <-ctx.Done(): // Fail open on cancellation too: let the inner call observe the // cancelled context and return its own error, rather than us // synthesising one here. return noop, nil - case <-time.After(l.pollInterval): + case <-timer.C: } } } @@ -136,10 +147,19 @@ func (l *redisLimiter) hold(zkey, token string) func() { // Detached ctx: the heartbeat must outlive request ctx up to // release. Best-effort; a failed refresh just risks early // reclamation, which the limit already tolerates. - _ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{ + // + // Refresh BOTH the member lease score AND the ZSET key's own + // TTL. The acquire script only PEXPIREs the key on admission, + // so a semaphore that stays saturated with no slot turnover + // would otherwise let the whole key expire after ttl*2 — + // dropping every live lease and admitting over the limit. The + // heartbeat pushes the key TTL out in lockstep with the lease. + bg := context.Background() + _ = l.rdb.ZAdd(bg, zkey, redis.Z{ Score: float64(now + l.ttl.Milliseconds()), Member: token, }).Err() + _ = l.rdb.PExpire(bg, zkey, l.ttl*2).Err() } } }() diff --git a/internal/models/vlm/concurrency_wrapper.go b/internal/models/vlm/concurrency_wrapper.go new file mode 100644 index 00000000..eaeead1b --- /dev/null +++ b/internal/models/vlm/concurrency_wrapper.go @@ -0,0 +1,35 @@ +package vlm + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/models/limiter" +) + +// Multimodal enrichment (image OCR / caption) is a high-volume, slow background +// stage that hits the same provider budget as chat. Like chat, it must be +// governed at the client layer so an image-heavy ingestion storm can't burst +// the whole worker pool against one VLM provider. Only background (asynq +// worker) calls are throttled — see limiter.Gate / types.IsBackgroundTask. +type concurrencyVLM struct { + inner VLM +} + +func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() } +func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() } + +func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) { + release := limiter.Gate(ctx, w.inner.GetModelID()) + defer release() + return w.inner.Predict(ctx, imgBytes, prompt) +} + +// wrapVLMConcurrency installs the background concurrency governor as the +// outermost VLM decorator. Always applied; a cheap passthrough when no limiter +// is installed or the call is interactive. +func wrapVLMConcurrency(v VLM, err error) (VLM, error) { + if err != nil || v == nil { + return v, err + } + return &concurrencyVLM{inner: v}, nil +} diff --git a/internal/models/vlm/vlm.go b/internal/models/vlm/vlm.go index a33eeebb..3d31a6fb 100644 --- a/internal/models/vlm/vlm.go +++ b/internal/models/vlm/vlm.go @@ -87,7 +87,10 @@ func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) { if logger.LLMDebugEnabled() { v = &debugVLM{inner: v} } - return wrapVLMLangfuse(v, nil) + v, err = wrapVLMLangfuse(v, nil) + // Outermost: hold the per-model concurrency slot only around the real + // provider round-trip, so the wait is excluded from debug/langfuse timing. + return wrapVLMConcurrency(v, err) } func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) { diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go index 848bedbd..72329349 100644 --- a/internal/router/sync_task.go +++ b/internal/router/sync_task.go @@ -80,7 +80,10 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asy time.Sleep(delay) } - ctx := context.Background() + // Tag as a background worker execution so the per-model concurrency + // governor throttles Lite-mode ingestion/enrichment LLM calls, mirroring + // the asynq backgroundTaskMiddleware in the Redis path. + ctx := types.WithBackgroundTask(context.Background()) start := time.Now() logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID)