feat(models): cap background LLM concurrency per model
Background ingestion/enrichment stages (summary, question, graph, multimodal) all target the same chat model, and asynq queue weights are scheduling priority rather than throttling. A large upload could therefore burst up to the entire worker pool of concurrent chat calls against a single provider, unbounded across replicas, with no model-side rate limit. Govern concurrency at the chat-client layer (the one place that sees every task type), keyed by model ID: - internal/models/limiter: Redis ZSET + Lua distributed semaphore with a lease heartbeat, expired-lease reclaim for crashed holders, and fail-open on any backend error or context cancellation. - chat: concurrencyChat decorator (outermost in NewChat) acquires a per-model slot only for background tasks; interactive chat is never gated. - router/task: asynq middleware tags every worker execution as background. - container: wire the Redis limiter with WEKNORA_MODEL_MAX_CONCURRENCY (model.max_concurrency), default 8; enabled only when Redis is present. Tested with miniredis: concurrency cap, expired-lease reclaim, per-key isolation, and all fail-open paths (nil client, backend down, cancelled ctx).
This commit is contained in:
@@ -7,6 +7,7 @@ require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.1
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/alicebob/miniredis/v2 v2.38.0
|
||||
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1
|
||||
github.com/asg017/sqlite-vec-go-bindings v0.1.6
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
@@ -296,6 +297,7 @@ require (
|
||||
github.com/xuri/efp v0.0.1 // indirect
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
|
||||
@@ -1380,6 +1380,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1 h1:vtiFd0hhPAbyYJjztl0wYUq/PqEGkIlDmVuTIy6zw8Y=
|
||||
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
@@ -2123,8 +2125,6 @@ github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAf
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
|
||||
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.9.7 h1:0AlevGG8zJ5uLhDuxgG4N1ULc6s7GEenV7ZRfER2R+Y=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.9.7/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
|
||||
github.com/leaanthony/debme v1.2.1 h1:9Tgwf+kjcrbMQ4WnPcEIUcQuIZYqdWftzZkBr+i/oOc=
|
||||
@@ -2617,6 +2617,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
|
||||
@@ -78,6 +78,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/mcp"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/router"
|
||||
"github.com/Tencent/WeKnora/internal/stream"
|
||||
@@ -274,6 +275,10 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// dequeue of pending/scheduled/retry tasks + active-task cancel).
|
||||
must(container.Provide(router.NewAsynqInspector))
|
||||
must(container.Provide(router.NewAsynqTaskInspector))
|
||||
// Install the distributed per-model chat concurrency governor. Only
|
||||
// available with Redis (the shared semaphore backend); Lite mode is
|
||||
// single-process and low-volume, so it runs ungated.
|
||||
must(container.Invoke(registerModelConcurrencyLimiter))
|
||||
} else {
|
||||
syncExec := router.NewSyncTaskExecutor()
|
||||
must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec }))
|
||||
@@ -436,6 +441,30 @@ func initLangfuse() (*langfuse.Manager, error) {
|
||||
return langfuse.Init(cfg)
|
||||
}
|
||||
|
||||
// defaultModelMaxConcurrency is the per-model cap on concurrent background
|
||||
// (ingestion/enrichment) chat calls when WEKNORA_MODEL_MAX_CONCURRENCY /
|
||||
// model.max_concurrency is unset. summary / question / graph enrichment all
|
||||
// share the same model, so this bounds their combined pressure on one provider
|
||||
// across every replica. Interactive chat is never gated.
|
||||
const defaultModelMaxConcurrency = 8
|
||||
|
||||
// registerModelConcurrencyLimiter builds the Redis-backed per-model chat
|
||||
// concurrency governor and installs it into the chat package. The limit is
|
||||
// resolved from system settings / env, defaulting to defaultModelMaxConcurrency.
|
||||
func registerModelConcurrencyLimiter(rdb *redis.Client, ss interfaces.SystemSettingService) {
|
||||
limit := defaultModelMaxConcurrency
|
||||
if ss != nil {
|
||||
n := ss.GetInt(context.Background(), "model.max_concurrency",
|
||||
"WEKNORA_MODEL_MAX_CONCURRENCY", int64(defaultModelMaxConcurrency))
|
||||
if n > 0 {
|
||||
limit = int(n)
|
||||
}
|
||||
}
|
||||
chat.SetConcurrencyLimiter(limiter.NewRedisLimiter(rdb), limit)
|
||||
logger.Infof(context.Background(),
|
||||
"[ModelLimiter] background chat concurrency governed per-model, limit=%d (distributed via redis)", limit)
|
||||
}
|
||||
|
||||
func initRedisClient() (*redis.Client, error) {
|
||||
redisAddr := os.Getenv("REDIS_ADDR")
|
||||
if redisAddr == "" {
|
||||
|
||||
@@ -147,7 +147,10 @@ func NewChat(config *ChatConfig, ollamaService *ollama.OllamaService) (Chat, err
|
||||
return nil, fmt.Errorf("unsupported chat model source: %s", config.Source)
|
||||
}
|
||||
c, err = wrapChatDebug(c, err)
|
||||
return wrapChatLangfuse(c, err)
|
||||
c, err = wrapChatLangfuse(c, err)
|
||||
// Outermost: hold the per-model concurrency slot only around the real
|
||||
// provider round-trip, so the wait is excluded from debug/langfuse timing.
|
||||
return wrapChatConcurrency(c, err)
|
||||
}
|
||||
|
||||
// NewRemoteChat 根据 provider 创建远程聊天实例。
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Model provider budgets are the real bottleneck shared by every LLM-backed
|
||||
// background stage (summary / question / graph / multimodal enrichment), which
|
||||
// all target the same model. This governor caps concurrent calls per model at
|
||||
// the client layer — the one place that sees all task types — instead of at the
|
||||
// asynq queue layer, whose weights are scheduling priority rather than
|
||||
// throttling.
|
||||
//
|
||||
// Only background (asynq worker) calls are throttled; interactive chat is left
|
||||
// untouched (see types.IsBackgroundTask), so a document-ingestion storm cannot
|
||||
// exhaust the provider yet user-facing latency is never gated behind the
|
||||
// semaphore.
|
||||
var (
|
||||
concurrencyMu sync.RWMutex
|
||||
concurrencyLimiter limiter.ModelConcurrencyLimiter
|
||||
concurrencyLimit int
|
||||
)
|
||||
|
||||
// SetConcurrencyLimiter installs the process-wide background chat concurrency
|
||||
// governor and the default per-model limit. Called once at startup (see
|
||||
// container wiring). Passing a nil limiter or a non-positive limit disables
|
||||
// governance (all calls pass through).
|
||||
func SetConcurrencyLimiter(l limiter.ModelConcurrencyLimiter, defaultLimit int) {
|
||||
concurrencyMu.Lock()
|
||||
defer concurrencyMu.Unlock()
|
||||
concurrencyLimiter = l
|
||||
concurrencyLimit = defaultLimit
|
||||
}
|
||||
|
||||
func getConcurrencyLimiter() (limiter.ModelConcurrencyLimiter, int) {
|
||||
concurrencyMu.RLock()
|
||||
defer concurrencyMu.RUnlock()
|
||||
return concurrencyLimiter, concurrencyLimit
|
||||
}
|
||||
|
||||
// concurrencyChat throttles background LLM calls through a per-model
|
||||
// distributed semaphore. It is the outermost wrapper so the slot is held only
|
||||
// around the actual provider round-trip and the wait time is excluded from the
|
||||
// inner debug/langfuse timing.
|
||||
type concurrencyChat struct {
|
||||
inner Chat
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) GetModelName() string { return w.inner.GetModelName() }
|
||||
func (w *concurrencyChat) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
// gate acquires a concurrency slot when the call is a background task and a
|
||||
// limiter is installed. Returns a release func (always safe to call) and
|
||||
// whether the inner call may proceed — it always may; the gate never blocks a
|
||||
// call permanently and fails open on any limiter error.
|
||||
func (w *concurrencyChat) gate(ctx context.Context) func() {
|
||||
lim, limit := getConcurrencyLimiter()
|
||||
if lim == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
|
||||
return func() {}
|
||||
}
|
||||
release, err := lim.Acquire(ctx, w.inner.GetModelID(), limit)
|
||||
if err != nil || release == nil {
|
||||
return func() {}
|
||||
}
|
||||
return release
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
defer release()
|
||||
return w.inner.Chat(ctx, messages, opts)
|
||||
}
|
||||
|
||||
func (w *concurrencyChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
release := w.gate(ctx)
|
||||
ch, err := w.inner.ChatStream(ctx, messages, opts)
|
||||
if err != nil || ch == nil {
|
||||
release()
|
||||
return ch, err
|
||||
}
|
||||
// Hold the slot until the stream fully drains, then release.
|
||||
out := make(chan types.StreamResponse)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer release()
|
||||
for resp := range ch {
|
||||
out <- resp
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// wrapChatConcurrency installs the background concurrency governor as the
|
||||
// outermost Chat decorator. It is always applied; when no limiter is installed
|
||||
// or the call is interactive, the wrapper is a cheap passthrough.
|
||||
func wrapChatConcurrency(c Chat, err error) (Chat, error) {
|
||||
if err != nil || c == nil {
|
||||
return c, err
|
||||
}
|
||||
return &concurrencyChat{inner: c}, nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Package limiter provides a distributed, per-key concurrency governor for
|
||||
// outbound model-provider calls. The shared finite resource is the model
|
||||
// provider (its request/concurrency budget), so concurrency is capped at the
|
||||
// model-client layer — keyed by model ID — rather than at the asynq queue layer
|
||||
// (queue weights are scheduling priority, not throttling).
|
||||
//
|
||||
// The Redis implementation is a self-healing distributed semaphore built on a
|
||||
// sorted set: each held slot is a ZSET member (unique token) scored by its
|
||||
// lease expiry. Acquire atomically prunes expired leases, counts live holders,
|
||||
// and admits a new one only while under the limit. A background heartbeat
|
||||
// refreshes the lease so long calls keep their slot; a crashed holder's lease
|
||||
// simply expires and is reclaimed. Every backend error fails OPEN (the call is
|
||||
// allowed) so a limiter/Redis outage can never halt model traffic.
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// ModelConcurrencyLimiter caps the number of concurrent in-flight calls per
|
||||
// key (typically a model ID) across all processes sharing the same backend.
|
||||
type ModelConcurrencyLimiter interface {
|
||||
// Acquire blocks until a slot for key is available or ctx is done. It
|
||||
// returns a release func that MUST be invoked to free the slot. On any
|
||||
// backend error (or ctx cancellation) it fails open: release is a no-op and
|
||||
// err is nil, so callers proceed without a slot rather than dropping the
|
||||
// call.
|
||||
Acquire(ctx context.Context, key string, limit int) (release func(), err error)
|
||||
}
|
||||
|
||||
// noop is the release returned on the fail-open / passthrough paths.
|
||||
func noop() {}
|
||||
|
||||
const (
|
||||
// defaultLeaseTTL is how long a held slot survives without a heartbeat
|
||||
// before another acquirer may reclaim it. Sized well above a typical
|
||||
// enrichment LLM call so a slow provider never loses its slot mid-call;
|
||||
// the heartbeat refreshes it for genuinely long calls.
|
||||
defaultLeaseTTL = 5 * time.Minute
|
||||
// defaultPollInterval is how often a waiting acquirer re-checks for a free
|
||||
// slot. Small enough to stay responsive, large enough to avoid hammering
|
||||
// Redis under contention.
|
||||
defaultPollInterval = 200 * time.Millisecond
|
||||
// keyPrefix namespaces the semaphore ZSETs in Redis.
|
||||
keyPrefix = "weknora:modelsem:"
|
||||
)
|
||||
|
||||
// acquireScript atomically prunes expired leases, counts live holders, and
|
||||
// admits the caller (adding its token scored by lease expiry) only while the
|
||||
// live count is below the limit. Returns 1 on admission, 0 when full.
|
||||
//
|
||||
// KEYS[1] = semaphore ZSET key
|
||||
// ARGV[1] = now (unix ms)
|
||||
// ARGV[2] = limit
|
||||
// ARGV[3] = caller token
|
||||
// ARGV[4] = lease TTL (ms)
|
||||
var acquireScript = redis.NewScript(`
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
|
||||
local count = redis.call('ZCARD', KEYS[1])
|
||||
if count < tonumber(ARGV[2]) then
|
||||
redis.call('ZADD', KEYS[1], ARGV[1] + ARGV[4], ARGV[3])
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[4] * 2)
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
type redisLimiter struct {
|
||||
rdb *redis.Client
|
||||
ttl time.Duration
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
// NewRedisLimiter builds a distributed limiter backed by rdb. A nil client
|
||||
// yields a limiter that always fails open.
|
||||
func NewRedisLimiter(rdb *redis.Client) ModelConcurrencyLimiter {
|
||||
return &redisLimiter{
|
||||
rdb: rdb,
|
||||
ttl: defaultLeaseTTL,
|
||||
pollInterval: defaultPollInterval,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
|
||||
if l == nil || l.rdb == nil || limit <= 0 || key == "" {
|
||||
return noop, nil
|
||||
}
|
||||
|
||||
zkey := keyPrefix + key
|
||||
token := uuid.NewString()
|
||||
ttlMs := l.ttl.Milliseconds()
|
||||
|
||||
for {
|
||||
now := time.Now().UnixMilli()
|
||||
res, err := acquireScript.Run(ctx, l.rdb, []string{zkey},
|
||||
now, limit, token, ttlMs).Int()
|
||||
if err != nil {
|
||||
// Fail open: a limiter outage must never block model traffic.
|
||||
logger.Warnf(ctx, "[ModelLimiter] acquire failed for key=%s, failing open: %v", key, err)
|
||||
return noop, nil
|
||||
}
|
||||
if res == 1 {
|
||||
return l.hold(zkey, token), nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Fail open on cancellation too: let the inner call observe the
|
||||
// cancelled context and return its own error, rather than us
|
||||
// synthesising one here.
|
||||
return noop, nil
|
||||
case <-time.After(l.pollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hold starts a heartbeat that refreshes the lease and returns an idempotent
|
||||
// release that stops the heartbeat and drops the slot.
|
||||
func (l *redisLimiter) hold(zkey, token string) func() {
|
||||
stop := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(l.ttl / 3)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
now := time.Now().UnixMilli()
|
||||
// Detached ctx: the heartbeat must outlive request ctx up to
|
||||
// release. Best-effort; a failed refresh just risks early
|
||||
// reclamation, which the limit already tolerates.
|
||||
_ = l.rdb.ZAdd(context.Background(), zkey, redis.Z{
|
||||
Score: float64(now + l.ttl.Milliseconds()),
|
||||
Member: token,
|
||||
}).Err()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
close(stop)
|
||||
_ = l.rdb.ZRem(context.Background(), zkey, token).Err()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func newTestLimiter(t *testing.T, ttl, poll time.Duration) (*redisLimiter, *miniredis.Miniredis, *redis.Client) {
|
||||
t.Helper()
|
||||
s, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("start miniredis: %v", err)
|
||||
}
|
||||
t.Cleanup(s.Close)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: s.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
return &redisLimiter{rdb: rdb, ttl: ttl, pollInterval: poll}, s, rdb
|
||||
}
|
||||
|
||||
func zcard(t *testing.T, rdb *redis.Client, key string) int64 {
|
||||
t.Helper()
|
||||
n, err := rdb.ZCard(context.Background(), keyPrefix+key).Result()
|
||||
if err != nil {
|
||||
t.Fatalf("zcard: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestRedisLimiterCapsConcurrency verifies the live-holder count never exceeds
|
||||
// the limit and that a freed slot lets a waiting acquirer through.
|
||||
func TestRedisLimiterCapsConcurrency(t *testing.T) {
|
||||
lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
ctx := context.Background()
|
||||
const key = "m1"
|
||||
|
||||
r1, err := lim.Acquire(ctx, key, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire 1: %v", err)
|
||||
}
|
||||
r2, err := lim.Acquire(ctx, key, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire 2: %v", err)
|
||||
}
|
||||
if got := zcard(t, rdb, key); got != 2 {
|
||||
t.Fatalf("expected 2 live holders, got %d", got)
|
||||
}
|
||||
|
||||
// The third acquire must block while the semaphore is full.
|
||||
done := make(chan func(), 1)
|
||||
go func() {
|
||||
r3, _ := lim.Acquire(ctx, key, 2)
|
||||
done <- r3
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("third acquire should block while at limit")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
if got := zcard(t, rdb, key); got != 2 {
|
||||
t.Fatalf("count must not exceed limit while blocked, got %d", got)
|
||||
}
|
||||
|
||||
// Freeing a slot lets the waiter in.
|
||||
r1()
|
||||
var r3 func()
|
||||
select {
|
||||
case r3 = <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("third acquire should proceed after a slot frees")
|
||||
}
|
||||
if got := zcard(t, rdb, key); got != 2 {
|
||||
t.Fatalf("expected 2 live holders after handoff, got %d", got)
|
||||
}
|
||||
|
||||
r2()
|
||||
r3()
|
||||
if got := zcard(t, rdb, key); got != 0 {
|
||||
t.Fatalf("expected 0 live holders after release, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedisLimiterReclaimsExpiredLease verifies a crashed holder's stale lease
|
||||
// (no heartbeat) is pruned so its slot is reclaimed by a new acquirer.
|
||||
func TestRedisLimiterReclaimsExpiredLease(t *testing.T) {
|
||||
lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
ctx := context.Background()
|
||||
const key = "m2"
|
||||
|
||||
// Simulate a dead holder: a member whose lease already expired.
|
||||
expired := float64(time.Now().Add(-time.Second).UnixMilli())
|
||||
if err := rdb.ZAdd(ctx, keyPrefix+key, redis.Z{Score: expired, Member: "dead"}).Err(); err != nil {
|
||||
t.Fatalf("seed dead holder: %v", err)
|
||||
}
|
||||
|
||||
// limit=1 would be full if the dead lease still counted; pruning reclaims it.
|
||||
release, err := lim.Acquire(ctx, key, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire after expiry: %v", err)
|
||||
}
|
||||
if got := zcard(t, rdb, key); got != 1 {
|
||||
t.Fatalf("expected only the live holder, got %d", got)
|
||||
}
|
||||
if exists, _ := rdb.ZScore(ctx, keyPrefix+key, "dead").Result(); exists != 0 {
|
||||
t.Fatal("expired lease should have been pruned")
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// TestRedisLimiterIndependentKeys verifies budgets are per key.
|
||||
func TestRedisLimiterIndependentKeys(t *testing.T) {
|
||||
lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
ctx := context.Background()
|
||||
|
||||
ra, err := lim.Acquire(ctx, "a", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire a: %v", err)
|
||||
}
|
||||
// A different key has its own budget, so this must not block.
|
||||
rb, err := lim.Acquire(ctx, "b", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire b: %v", err)
|
||||
}
|
||||
ra()
|
||||
rb()
|
||||
}
|
||||
|
||||
// TestRedisLimiterFailsOpen verifies every degraded path allows the call
|
||||
// instead of blocking model traffic.
|
||||
func TestRedisLimiterFailsOpen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Nil client.
|
||||
lim := &redisLimiter{rdb: nil, ttl: time.Minute, pollInterval: 10 * time.Millisecond}
|
||||
if release, err := lim.Acquire(ctx, "k", 4); err != nil || release == nil {
|
||||
t.Fatalf("nil client should fail open, got release==nil=%v err=%v", release == nil, err)
|
||||
}
|
||||
|
||||
// Non-positive limit is a no-op governor.
|
||||
lim2, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
if release, err := lim2.Acquire(ctx, "k", 0); err != nil || release == nil {
|
||||
t.Fatalf("limit<=0 should fail open, got release==nil=%v err=%v", release == nil, err)
|
||||
}
|
||||
|
||||
// Backend error (server down) must fail open too.
|
||||
lim3, s, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
s.Close()
|
||||
if release, err := lim3.Acquire(ctx, "k", 4); err != nil || release == nil {
|
||||
t.Fatalf("backend error should fail open, got release==nil=%v err=%v", release == nil, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedisLimiterCancelledContextFailsOpen verifies a waiter whose context is
|
||||
// cancelled while blocked returns a usable (no-op) release rather than erroring.
|
||||
func TestRedisLimiterCancelledContextFailsOpen(t *testing.T) {
|
||||
lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
|
||||
|
||||
r1, err := lim.Acquire(context.Background(), "m3", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire: %v", err)
|
||||
}
|
||||
defer r1()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
release, err := lim.Acquire(ctx, "m3", 1)
|
||||
if err != nil || release == nil {
|
||||
t.Fatalf("cancelled wait should fail open, got release==nil=%v err=%v", release == nil, err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
@@ -154,6 +154,18 @@ func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server {
|
||||
)
|
||||
}
|
||||
|
||||
// backgroundTaskMiddleware tags every task's context as a background worker
|
||||
// execution (types.WithBackgroundTask) so the per-model chat concurrency
|
||||
// governor applies to ingestion/enrichment LLM calls but not to interactive
|
||||
// user-facing chat.
|
||||
func backgroundTaskMiddleware() asynq.MiddlewareFunc {
|
||||
return func(next asynq.Handler) asynq.Handler {
|
||||
return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
|
||||
return next.ProcessTask(types.WithBackgroundTask(ctx), t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki.
|
||||
// This is the user-facing document parse / post-process / enrichment pipeline.
|
||||
// Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from
|
||||
@@ -218,6 +230,12 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker)
|
||||
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
|
||||
|
||||
// Mark every asynq worker execution as a background task so the chat
|
||||
// concurrency governor throttles ingestion/enrichment LLM traffic while
|
||||
// leaving user-facing interactive chat ungated. Installed early so all
|
||||
// downstream handlers (and their model calls) inherit the flag.
|
||||
mux.Use(backgroundTaskMiddleware())
|
||||
|
||||
// Install Langfuse middleware BEFORE handler registration so every task
|
||||
// type is automatically wrapped. When Langfuse is disabled the middleware
|
||||
// is a pass-through; when enabled it resumes the upstream HTTP trace (if
|
||||
|
||||
@@ -39,6 +39,12 @@ const (
|
||||
LangfuseTraceContextKey ContextKey = "LangfuseTrace"
|
||||
// SystemAdminContextKey is the context key indicating whether the user is a system administrator
|
||||
SystemAdminContextKey ContextKey = "SystemAdmin"
|
||||
// BackgroundTaskContextKey marks a context whose model calls originate from
|
||||
// an asynq background worker (document parse / summary / question / graph /
|
||||
// multimodal enrichment) rather than a user-facing HTTP request. The chat
|
||||
// concurrency governor uses this to throttle only background LLM traffic,
|
||||
// leaving interactive chat latency untouched. See WithBackgroundTask.
|
||||
BackgroundTaskContextKey ContextKey = "BackgroundTask"
|
||||
// MCPOAuthNonInteractiveContextKey marks a request whose channel cannot
|
||||
// resolve an in-conversation MCP OAuth prompt (e.g. an IM bot: there is no
|
||||
// live client to click "Authorize" and call the resolve endpoint). When set,
|
||||
|
||||
@@ -131,6 +131,25 @@ func IsMCPOAuthNonInteractive(ctx context.Context) bool {
|
||||
return v
|
||||
}
|
||||
|
||||
// WithBackgroundTask marks ctx as originating from an asynq background worker
|
||||
// (document parse / summary / question / graph / multimodal enrichment). The
|
||||
// chat concurrency governor throttles only background LLM traffic, so this flag
|
||||
// determines whether the per-model concurrency limit applies. See
|
||||
// BackgroundTaskContextKey.
|
||||
func WithBackgroundTask(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, BackgroundTaskContextKey, true)
|
||||
}
|
||||
|
||||
// IsBackgroundTask reports whether ctx was marked as a background worker task
|
||||
// (see WithBackgroundTask). Returns false for interactive / HTTP request paths.
|
||||
func IsBackgroundTask(ctx context.Context) bool {
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
v, _ := ctx.Value(BackgroundTaskContextKey).(bool)
|
||||
return v
|
||||
}
|
||||
|
||||
// LanguageFromContext extracts the language locale string from ctx (e.g. "zh-CN", "en-US").
|
||||
// Returns ("zh-CN", false) when the key is absent.
|
||||
func LanguageFromContext(ctx context.Context) (string, bool) {
|
||||
|
||||
Reference in New Issue
Block a user