From 63198c5d9f5ef4f65c1ce5b16fb70ecac929b4a6 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Jul 2026 18:13:09 +0800 Subject: [PATCH] feat(models): cap background LLM concurrency per model Background ingestion/enrichment stages (summary, question, graph, multimodal) all target the same chat model, and asynq queue weights are scheduling priority rather than throttling. A large upload could therefore burst up to the entire worker pool of concurrent chat calls against a single provider, unbounded across replicas, with no model-side rate limit. Govern concurrency at the chat-client layer (the one place that sees every task type), keyed by model ID: - internal/models/limiter: Redis ZSET + Lua distributed semaphore with a lease heartbeat, expired-lease reclaim for crashed holders, and fail-open on any backend error or context cancellation. - chat: concurrencyChat decorator (outermost in NewChat) acquires a per-model slot only for background tasks; interactive chat is never gated. - router/task: asynq middleware tags every worker execution as background. - container: wire the Redis limiter with WEKNORA_MODEL_MAX_CONCURRENCY (model.max_concurrency), default 8; enabled only when Redis is present. Tested with miniredis: concurrency cap, expired-lease reclaim, per-key isolation, and all fail-open paths (nil client, backend down, cancelled ctx). --- go.mod | 2 + go.sum | 6 +- internal/container/container.go | 29 ++++ internal/models/chat/chat.go | 5 +- internal/models/chat/concurrency_wrapper.go | 105 ++++++++++++ internal/models/limiter/limiter.go | 154 +++++++++++++++++ internal/models/limiter/limiter_test.go | 175 ++++++++++++++++++++ internal/router/task.go | 18 ++ internal/types/const.go | 6 + internal/types/context_helpers.go | 19 +++ 10 files changed, 516 insertions(+), 3 deletions(-) create mode 100644 internal/models/chat/concurrency_wrapper.go create mode 100644 internal/models/limiter/limiter.go create mode 100644 internal/models/limiter/limiter_test.go diff --git a/go.mod b/go.mod index efb749fd..e7417eaa 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.1 github.com/PuerkitoBio/goquery v1.12.0 + github.com/alicebob/miniredis/v2 v2.38.0 github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1 github.com/asg017/sqlite-vec-go-bindings v0.1.6 github.com/aws/aws-sdk-go-v2 v1.41.7 @@ -296,6 +297,7 @@ require ( github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect diff --git a/go.sum b/go.sum index 07ef63e7..164b6745 100644 --- a/go.sum +++ b/go.sum @@ -1380,6 +1380,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1 h1:vtiFd0hhPAbyYJjztl0wYUq/PqEGkIlDmVuTIy6zw8Y= github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -2123,8 +2125,6 @@ github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= -github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= -github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/larksuite/oapi-sdk-go/v3 v3.9.7 h1:0AlevGG8zJ5uLhDuxgG4N1ULc6s7GEenV7ZRfER2R+Y= github.com/larksuite/oapi-sdk-go/v3 v3.9.7/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leaanthony/debme v1.2.1 h1:9Tgwf+kjcrbMQ4WnPcEIUcQuIZYqdWftzZkBr+i/oOc= @@ -2617,6 +2617,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= diff --git a/internal/container/container.go b/internal/container/container.go index 66da1bfb..f77d5843 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -78,6 +78,7 @@ import ( "github.com/Tencent/WeKnora/internal/mcp" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/limiter" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/router" "github.com/Tencent/WeKnora/internal/stream" @@ -274,6 +275,10 @@ func BuildContainer(container *dig.Container) *dig.Container { // dequeue of pending/scheduled/retry tasks + active-task cancel). must(container.Provide(router.NewAsynqInspector)) must(container.Provide(router.NewAsynqTaskInspector)) + // Install the distributed per-model chat concurrency governor. Only + // available with Redis (the shared semaphore backend); Lite mode is + // single-process and low-volume, so it runs ungated. + must(container.Invoke(registerModelConcurrencyLimiter)) } else { syncExec := router.NewSyncTaskExecutor() must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec })) @@ -436,6 +441,30 @@ func initLangfuse() (*langfuse.Manager, error) { return langfuse.Init(cfg) } +// defaultModelMaxConcurrency is the per-model cap on concurrent background +// (ingestion/enrichment) chat calls when WEKNORA_MODEL_MAX_CONCURRENCY / +// model.max_concurrency is unset. summary / question / graph enrichment all +// share the same model, so this bounds their combined pressure on one provider +// across every replica. Interactive chat is never gated. +const defaultModelMaxConcurrency = 8 + +// registerModelConcurrencyLimiter builds the Redis-backed per-model chat +// concurrency governor and installs it into the chat package. The limit is +// resolved from system settings / env, defaulting to defaultModelMaxConcurrency. +func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) { + limit := defaultModelMaxConcurrency + if ss != nil { + n := ss.GetInt(context.Background(), "model.max_concurrency", + "WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency)) + if n > 0 { + limit = int(n) + } + } + chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit) + logger.Infof(context.Background(), + "[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit) +} + func initRedisClient() (*redis.Client, error) { redisAddr := os.Getenv("REDIS_ADDR") if redisAddr == "" { diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go index 47ed6941..e2f90d08 100644 --- a/internal/models/chat/chat.go +++ b/internal/models/chat/chat.go @@ -147,7 +147,10 @@ func NewChat(config *ChatConfig, ollamaService *ollama.OllamaService) (Chat, err return nil, fmt.Errorf("unsupported chat model source: %s", config.Source) } c, err = wrapChatDebug(c, err) - return wrapChatLangfuse(c, err) + c, err = wrapChatLangfuse(c, err) + // Outermost: hold the per-model concurrency slot only around the real + // provider round-trip, so the wait is excluded from debug/langfuse timing. + return wrapChatConcurrency(c, err) } // NewRemoteChat 根据 provider 创建远程聊天实例。 diff --git a/internal/models/chat/concurrency_wrapper.go b/internal/models/chat/concurrency_wrapper.go new file mode 100644 index 00000000..d04823a0 --- /dev/null +++ b/internal/models/chat/concurrency_wrapper.go @@ -0,0 +1,105 @@ +package chat + +import ( + "context" + "sync" + + "github.com/Tencent/WeKnora/internal/models/limiter" + "github.com/Tencent/WeKnora/internal/types" +) + +// Model provider budgets are the real bottleneck shared by every LLM-backed +// background stage (summary / question / graph / multimodal enrichment), which +// all target the same model. This governor caps concurrent calls per model at +// the client layer — the one place that sees all task types — instead of at the +// asynq queue layer, whose weights are scheduling priority rather than +// throttling. +// +// Only background (asynq worker) calls are throttled; interactive chat is left +// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot +// exhaust the provider yet user-facing latency is never gated behind the +// semaphore. +var ( + concurrencyMu sync.RWMutex + concurrencyLimiter limiter.ModelConcurrencyLimiter + concurrencyLimit int +) + +// SetConcurrencyLimiter installs the process-wide background chat concurrency +// governor and the default per-model limit. Called once at startup (see +// container wiring). Passing a nil limiter or a non-positive limit disables +// governance (all calls pass through). +func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) { + concurrencyMu.Lock() + defer concurrencyMu.Unlock() + concurrencyLimiter = l + concurrencyLimit = defaultLimit +} + +func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) { + concurrencyMu.RLock() + defer concurrencyMu.RUnlock() + return concurrencyLimiter, concurrencyLimit +} + +// concurrencyChat throttles background LLM calls through a per-model +// distributed semaphore. It is the outermost wrapper so the slot is held only +// around the actual provider round-trip and the wait time is excluded from the +// inner debug/langfuse timing. +type concurrencyChat struct { + inner Chat +} + +func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() } +func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() } + +// gate acquires a concurrency slot when the call is a background task and a +// limiter is installed. Returns a release func (always safe to call) and +// whether the inner call may proceed — it always may; the gate never blocks a +// call permanently and fails open on any limiter error. +func (w *concurrencyChat) gate(ctx context.Context) func() { + lim, limit := getConcurrencyLimiter() + if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) { + return func() {} + } + release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit) + if err != nil || release == nil { + return func() {} + } + return release +} + +func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) { + release := w.gate(ctx) + defer release() + return w.inner.Chat(ctx, messages, opts) +} + +func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) { + release := w.gate(ctx) + ch, err := w.inner.ChatStream(ctx, messages, opts) + if err != nil || ch == nil { + release() + return ch, err + } + // Hold the slot until the stream fully drains, then release. + out := make(chan types.StreamResponse) + go func() { + defer close(out) + defer release() + for resp := range ch { + out <- resp + } + }() + return out, nil +} + +// wrapChatConcurrency installs the background concurrency governor as the +// outermost Chat decorator. It is always applied; when no limiter is installed +// or the call is interactive, the wrapper is a cheap passthrough. +func wrapChatConcurrency(c Chat, err error) (Chat, error) { + if err != nil || c == nil { + return c, err + } + return &concurrencyChat{inner: c}, nil +} diff --git a/internal/models/limiter/limiter.go b/internal/models/limiter/limiter.go new file mode 100644 index 00000000..1b20c612 --- /dev/null +++ b/internal/models/limiter/limiter.go @@ -0,0 +1,154 @@ +// Package limiter provides a distributed, per-key concurrency governor for +// outbound model-provider calls. The shared finite resource is the model +// provider (its request/concurrency budget), so concurrency is capped at the +// model-client layer — keyed by model ID — rather than at the asynq queue layer +// (queue weights are scheduling priority, not throttling). +// +// The Redis implementation is a self-healing distributed semaphore built on a +// sorted set: each held slot is a ZSET member (unique token) scored by its +// lease expiry. Acquire atomically prunes expired leases, counts live holders, +// and admits a new one only while under the limit. A background heartbeat +// refreshes the lease so long calls keep their slot; a crashed holder's lease +// simply expires and is reclaimed. Every backend error fails OPEN (the call is +// allowed) so a limiter/Redis outage can never halt model traffic. +package limiter + +import ( + "context" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/google/uuid" + "github.com/redis/go-redis/v9" +) + +// ModelConcurrencyLimiter caps the number of concurrent in-flight calls per +// key (typically a model ID) across all processes sharing the same backend. +type ModelConcurrencyLimiter interface { + // Acquire blocks until a slot for key is available or ctx is done. It + // returns a release func that MUST be invoked to free the slot. On any + // backend error (or ctx cancellation) it fails open: release is a no-op and + // err is nil, so callers proceed without a slot rather than dropping the + // call. + Acquire(ctx context.Context, key string, limit int) (release func(), err error) +} + +// noop is the release returned on the fail-open / passthrough paths. +func noop() {} + +const ( + // defaultLeaseTTL is how long a held slot survives without a heartbeat + // before another acquirer may reclaim it. Sized well above a typical + // enrichment LLM call so a slow provider never loses its slot mid-call; + // the heartbeat refreshes it for genuinely long calls. + defaultLeaseTTL = 5 * time.Minute + // defaultPollInterval is how often a waiting acquirer re-checks for a free + // slot. Small enough to stay responsive, large enough to avoid hammering + // Redis under contention. + defaultPollInterval = 200 * time.Millisecond + // keyPrefix namespaces the semaphore ZSETs in Redis. + keyPrefix = "weknora:modelsem:" +) + +// acquireScript atomically prunes expired leases, counts live holders, and +// admits the caller (adding its token scored by lease expiry) only while the +// live count is below the limit. Returns 1 on admission, 0 when full. +// +// KEYS[1] = semaphore ZSET key +// ARGV[1] = now (unix ms) +// ARGV[2] = limit +// ARGV[3] = caller token +// ARGV[4] = lease TTL (ms) +var acquireScript = redis.NewScript(` +redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) +local count = redis.call('ZCARD', KEYS[1]) +if count < tonumber(ARGV[2]) then + redis.call('ZADD', KEYS[1], ARGV[1] + ARGV[4], ARGV[3]) + redis.call('PEXPIRE', KEYS[1], ARGV[4] * 2) + return 1 +end +return 0 +`) + +type redisLimiter struct { + rdb *redis.Client + ttl time.Duration + pollInterval time.Duration +} + +// NewRedisLimiter builds a distributed limiter backed by rdb. A nil client +// yields a limiter that always fails open. +func NewRedisLimiter(rdb *redis.Client) ModelConcurrencyLimiter { + return &redisLimiter{ + rdb: rdb, + ttl: defaultLeaseTTL, + pollInterval: defaultPollInterval, + } +} + +func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) { + if l == nil || l.rdb == nil || limit <= 0 || key == "" { + return noop, nil + } + + zkey := keyPrefix + key + token := uuid.NewString() + ttlMs := l.ttl.Milliseconds() + + for { + now := time.Now().UnixMilli() + res, err := acquireScript.Run(ctx, l.rdb, []string{zkey}, + now, limit, token, ttlMs).Int() + if err != nil { + // Fail open: a limiter outage must never block model traffic. + logger.Warnf(ctx, "[ModelLimiter] acquire failed for key=%s, failing open: %v", key, err) + return noop, nil + } + if res == 1 { + return l.hold(zkey, token), nil + } + + select { + case <-ctx.Done(): + // Fail open on cancellation too: let the inner call observe the + // cancelled context and return its own error, rather than us + // synthesising one here. + return noop, nil + case <-time.After(l.pollInterval): + } + } +} + +// hold starts a heartbeat that refreshes the lease and returns an idempotent +// release that stops the heartbeat and drops the slot. +func (l *redisLimiter) hold(zkey, token string) func() { + stop := make(chan struct{}) + go func() { + t := time.NewTicker(l.ttl / 3) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + now := time.Now().UnixMilli() + // Detached ctx: the heartbeat must outlive request ctx up to + // release. Best-effort; a failed refresh just risks early + // reclamation, which the limit already tolerates. + _ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{ + Score: float64(now + l.ttl.Milliseconds()), + Member: token, + }).Err() + } + } + }() + + var once sync.Once + return func() { + once.Do(func() { + close(stop) + _ = l.rdb.ZRem(context.Background(), zkey, token).Err() + }) + } +} diff --git a/internal/models/limiter/limiter_test.go b/internal/models/limiter/limiter_test.go new file mode 100644 index 00000000..ddc0c453 --- /dev/null +++ b/internal/models/limiter/limiter_test.go @@ -0,0 +1,175 @@ +package limiter + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func newTestLimiter(t *testing.T, ttl, poll time.Duration) (*redisLimiter, *miniredis.Miniredis, *redis.Client) { + t.Helper() + s, err := miniredis.Run() + if err != nil { + t.Fatalf("start miniredis: %v", err) + } + t.Cleanup(s.Close) + rdb := redis.NewClient(&redis.Options{Addr: s.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + return &redisLimiter{rdb: rdb, ttl: ttl, pollInterval: poll}, s, rdb +} + +func zcard(t *testing.T, rdb *redis.Client, key string) int64 { + t.Helper() + n, err := rdb.ZCard(context.Background(), keyPrefix+key).Result() + if err != nil { + t.Fatalf("zcard: %v", err) + } + return n +} + +// TestRedisLimiterCapsConcurrency verifies the live-holder count never exceeds +// the limit and that a freed slot lets a waiting acquirer through. +func TestRedisLimiterCapsConcurrency(t *testing.T) { + lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond) + ctx := context.Background() + const key = "m1" + + r1, err := lim.Acquire(ctx, key, 2) + if err != nil { + t.Fatalf("acquire 1: %v", err) + } + r2, err := lim.Acquire(ctx, key, 2) + if err != nil { + t.Fatalf("acquire 2: %v", err) + } + if got := zcard(t, rdb, key); got != 2 { + t.Fatalf("expected 2 live holders, got %d", got) + } + + // The third acquire must block while the semaphore is full. + done := make(chan func(), 1) + go func() { + r3, _ := lim.Acquire(ctx, key, 2) + done <- r3 + }() + + select { + case <-done: + t.Fatal("third acquire should block while at limit") + case <-time.After(100 * time.Millisecond): + } + if got := zcard(t, rdb, key); got != 2 { + t.Fatalf("count must not exceed limit while blocked, got %d", got) + } + + // Freeing a slot lets the waiter in. + r1() + var r3 func() + select { + case r3 = <-done: + case <-time.After(2 * time.Second): + t.Fatal("third acquire should proceed after a slot frees") + } + if got := zcard(t, rdb, key); got != 2 { + t.Fatalf("expected 2 live holders after handoff, got %d", got) + } + + r2() + r3() + if got := zcard(t, rdb, key); got != 0 { + t.Fatalf("expected 0 live holders after release, got %d", got) + } +} + +// TestRedisLimiterReclaimsExpiredLease verifies a crashed holder's stale lease +// (no heartbeat) is pruned so its slot is reclaimed by a new acquirer. +func TestRedisLimiterReclaimsExpiredLease(t *testing.T) { + lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond) + ctx := context.Background() + const key = "m2" + + // Simulate a dead holder: a member whose lease already expired. + expired := float64(time.Now().Add(-time.Second).UnixMilli()) + if err := rdb.ZAdd(ctx, keyPrefix+key, redis.Z{Score: expired, Member: "dead"}).Err(); err != nil { + t.Fatalf("seed dead holder: %v", err) + } + + // limit=1 would be full if the dead lease still counted; pruning reclaims it. + release, err := lim.Acquire(ctx, key, 1) + if err != nil { + t.Fatalf("acquire after expiry: %v", err) + } + if got := zcard(t, rdb, key); got != 1 { + t.Fatalf("expected only the live holder, got %d", got) + } + if exists, _ := rdb.ZScore(ctx, keyPrefix+key, "dead").Result(); exists != 0 { + t.Fatal("expired lease should have been pruned") + } + release() +} + +// TestRedisLimiterIndependentKeys verifies budgets are per key. +func TestRedisLimiterIndependentKeys(t *testing.T) { + lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond) + ctx := context.Background() + + ra, err := lim.Acquire(ctx, "a", 1) + if err != nil { + t.Fatalf("acquire a: %v", err) + } + // A different key has its own budget, so this must not block. + rb, err := lim.Acquire(ctx, "b", 1) + if err != nil { + t.Fatalf("acquire b: %v", err) + } + ra() + rb() +} + +// TestRedisLimiterFailsOpen verifies every degraded path allows the call +// instead of blocking model traffic. +func TestRedisLimiterFailsOpen(t *testing.T) { + ctx := context.Background() + + // Nil client. + lim := &redisLimiter{rdb: nil, ttl: time.Minute, pollInterval: 10 * time.Millisecond} + if release, err := lim.Acquire(ctx, "k", 4); err != nil || release == nil { + t.Fatalf("nil client should fail open, got release==nil=%v err=%v", release == nil, err) + } + + // Non-positive limit is a no-op governor. + lim2, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond) + if release, err := lim2.Acquire(ctx, "k", 0); err != nil || release == nil { + t.Fatalf("limit<=0 should fail open, got release==nil=%v err=%v", release == nil, err) + } + + // Backend error (server down) must fail open too. + lim3, s, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond) + s.Close() + if release, err := lim3.Acquire(ctx, "k", 4); err != nil || release == nil { + t.Fatalf("backend error should fail open, got release==nil=%v err=%v", release == nil, err) + } +} + +// TestRedisLimiterCancelledContextFailsOpen verifies a waiter whose context is +// cancelled while blocked returns a usable (no-op) release rather than erroring. +func TestRedisLimiterCancelledContextFailsOpen(t *testing.T) { + lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond) + + r1, err := lim.Acquire(context.Background(), "m3", 1) + if err != nil { + t.Fatalf("acquire: %v", err) + } + defer r1() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + release, err := lim.Acquire(ctx, "m3", 1) + if err != nil || release == nil { + t.Fatalf("cancelled wait should fail open, got release==nil=%v err=%v", release == nil, err) + } + release() +} diff --git a/internal/router/task.go b/internal/router/task.go index 0060a3ff..c779ebf4 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -154,6 +154,18 @@ func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server { ) } +// backgroundTaskMiddleware tags every task's context as a background worker +// execution (types.WithBackgroundTask) so the per-model chat concurrency +// governor applies to ingestion/enrichment LLM calls but not to interactive +// user-facing chat. +func backgroundTaskMiddleware() asynq.MiddlewareFunc { + return func(next asynq.Handler) asynq.Handler { + return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error { + return next.ProcessTask(types.WithBackgroundTask(ctx), t) + }) + } +} + // NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki. // This is the user-facing document parse / post-process / enrichment pipeline. // Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from @@ -218,6 +230,12 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker) mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer)) + // Mark every asynq worker execution as a background task so the chat + // concurrency governor throttles ingestion/enrichment LLM traffic while + // leaving user-facing interactive chat ungated. Installed early so all + // downstream handlers (and their model calls) inherit the flag. + mux.Use(backgroundTaskMiddleware()) + // Install Langfuse middleware BEFORE handler registration so every task // type is automatically wrapped. When Langfuse is disabled the middleware // is a pass-through; when enabled it resumes the upstream HTTP trace (if diff --git a/internal/types/const.go b/internal/types/const.go index 0ba31df6..553b364b 100644 --- a/internal/types/const.go +++ b/internal/types/const.go @@ -39,6 +39,12 @@ const ( LangfuseTraceContextKey ContextKey = "LangfuseTrace" // SystemAdminContextKey is the context key indicating whether the user is a system administrator SystemAdminContextKey ContextKey = "SystemAdmin" + // BackgroundTaskContextKey marks a context whose model calls originate from + // an asynq background worker (document parse / summary / question / graph / + // multimodal enrichment) rather than a user-facing HTTP request. The chat + // concurrency governor uses this to throttle only background LLM traffic, + // leaving interactive chat latency untouched. See WithBackgroundTask. + BackgroundTaskContextKey ContextKey = "BackgroundTask" // MCPOAuthNonInteractiveContextKey marks a request whose channel cannot // resolve an in-conversation MCP OAuth prompt (e.g. an IM bot: there is no // live client to click "Authorize" and call the resolve endpoint). When set, diff --git a/internal/types/context_helpers.go b/internal/types/context_helpers.go index 13dbd7b5..65d483c9 100644 --- a/internal/types/context_helpers.go +++ b/internal/types/context_helpers.go @@ -131,6 +131,25 @@ func IsMCPOAuthNonInteractive(ctx context.Context) bool { return v } +// WithBackgroundTask marks ctx as originating from an asynq background worker +// (document parse / summary / question / graph / multimodal enrichment). The +// chat concurrency governor throttles only background LLM traffic, so this flag +// determines whether the per-model concurrency limit applies. See +// BackgroundTaskContextKey. +func WithBackgroundTask(ctx context.Context) context.Context { + return context.WithValue(ctx, BackgroundTaskContextKey, true) +} + +// IsBackgroundTask reports whether ctx was marked as a background worker task +// (see WithBackgroundTask). Returns false for interactive / HTTP request paths. +func IsBackgroundTask(ctx context.Context) bool { + if ctx == nil { + return false + } + v, _ := ctx.Value(BackgroundTaskContextKey).(bool) + return v +} + // LanguageFromContext extracts the language locale string from ctx (e.g. "zh-CN", "en-US"). // Returns ("zh-CN", false) when the key is absent. func LanguageFromContext(ctx context.Context) (string, bool) {