diff --git a/internal/config/config.go b/internal/config/config.go index da55f485..8a161d77 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,34 @@ type Config struct { ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"` WebSearch *WebSearchConfig `yaml:"web_search" json:"web_search"` PromptTemplates *PromptTemplatesConfig `yaml:"prompt_templates" json:"prompt_templates"` + IM *IMConfig `yaml:"im" json:"im"` +} + +// IMConfig configures the IM integration service. +// All fields are optional — zero values fall back to built-in defaults so +// existing deployments need no config changes. +type IMConfig struct { + // Workers is the number of concurrent QA worker goroutines per instance. + // Default: 5. + Workers int `yaml:"workers" json:"workers"` + // GlobalMaxWorkers is the maximum number of QA requests that can execute + // concurrently across ALL instances. Enforced via a Redis counter; when the + // global limit is reached, local workers wait until a slot opens. + // Requires Redis — ignored in single-instance mode. + // 0 (default) means no global limit. + GlobalMaxWorkers int `yaml:"global_max_workers" json:"global_max_workers"` + // MaxQueueSize is the maximum number of pending QA requests per instance. + // Default: 50. + MaxQueueSize int `yaml:"max_queue_size" json:"max_queue_size"` + // MaxPerUser limits how many requests a single user can have queued globally. + // Default: 3. + MaxPerUser int `yaml:"max_per_user" json:"max_per_user"` + // RateLimitWindow is the sliding window duration for per-user rate limiting. + // Default: 60s. + RateLimitWindow time.Duration `yaml:"rate_limit_window" json:"rate_limit_window"` + // RateLimitMax is the maximum number of requests allowed per window per user. + // Default: 10. + RateLimitMax int `yaml:"rate_limit_max" json:"rate_limit_max"` } // DocReaderConfig configures the document parser client (gRPC or HTTP). diff --git a/internal/im/qaqueue.go b/internal/im/qaqueue.go index e9655680..f6ac4e96 100644 --- a/internal/im/qaqueue.go +++ b/internal/im/qaqueue.go @@ -9,6 +9,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + "github.com/redis/go-redis/v9" ) const ( @@ -20,6 +21,15 @@ const ( defaultWorkers = 5 // queueTimeout is how long a request can wait in the queue before being discarded. queueTimeout = 60 * time.Second + // redisQueueUserTTL is the TTL for per-user queue counters in Redis. + redisQueueUserTTL = 5 * time.Minute + // globalGateTTL is the TTL for the global active-worker counter in Redis. + // Acts as a safety net: if all instances crash without decrementing, the + // counter self-heals after this duration. + globalGateTTL = 5 * time.Minute + // globalGateRetryInterval is how long a worker waits before retrying when the + // global concurrency limit is reached. + globalGateRetryInterval = 500 * time.Millisecond ) // qaRequest represents a QA request waiting in the queue. @@ -65,6 +75,15 @@ type qaQueue struct { perUser map[string]int // userKey → queued count closed bool + // redis is the optional Redis client for global per-user counting. + // When nil, only local per-user limits are enforced. + redis *redis.Client + + // globalMaxWorkers is the maximum number of QA requests executing + // concurrently across all instances. 0 means no global limit. + // Enforced via Redis INCR/DECR on RedisKeyGlobalGate. + globalMaxWorkers int + // metrics activeWorkers atomic.Int64 totalEnqueued atomic.Int64 @@ -77,14 +96,18 @@ type qaQueue struct { } // newQAQueue creates a new bounded queue with the given worker count. -func newQAQueue(workers, maxSize, maxPerUser int, handler func(req *qaRequest)) *qaQueue { +// globalMaxWorkers controls cross-instance concurrency (0 = no limit). +// redisClient may be nil for single-instance mode. +func newQAQueue(workers, maxSize, maxPerUser, globalMaxWorkers int, handler func(req *qaRequest), redisClient *redis.Client) *qaQueue { q := &qaQueue{ - queue: make([]*qaRequest, 0, maxSize), - maxSize: maxSize, - maxPerUser: maxPerUser, - workers: workers, - perUser: make(map[string]int), - handler: handler, + queue: make([]*qaRequest, 0, maxSize), + maxSize: maxSize, + maxPerUser: maxPerUser, + workers: workers, + globalMaxWorkers: globalMaxWorkers, + perUser: make(map[string]int), + redis: redisClient, + handler: handler, } q.cond = sync.NewCond(&q.mu) return q @@ -109,26 +132,41 @@ func (q *qaQueue) Stop() { // Enqueue adds a request to the queue. Returns the queue position (0-based) // or an error if the queue is full or per-user limit is reached. func (q *qaQueue) Enqueue(req *qaRequest) (position int, err error) { + // Check global per-user limit via Redis before acquiring local lock. + if q.redis != nil { + if err := q.redisCheckAndIncrUser(context.Background(), req.userKey); err != nil { + q.totalRejected.Add(1) + return 0, err + } + } + q.mu.Lock() defer q.mu.Unlock() if q.closed { + q.redisDecrUser(context.Background(), req.userKey) return 0, fmt.Errorf("queue is closed") } if len(q.queue) >= q.maxSize { + q.redisDecrUser(context.Background(), req.userKey) q.totalRejected.Add(1) return 0, fmt.Errorf("queue full (%d/%d)", len(q.queue), q.maxSize) } - if q.perUser[req.userKey] >= q.maxPerUser { + // Local per-user check: only useful when Redis is nil (single-instance mode). + // When Redis is available, redisCheckAndIncrUser already enforces the global + // per-user limit across all instances, making this local check redundant. + if q.redis == nil && q.perUser[req.userKey] >= q.maxPerUser { q.totalRejected.Add(1) return 0, fmt.Errorf("per-user queue limit reached (%d/%d)", q.perUser[req.userKey], q.maxPerUser) } req.enqueuedAt = time.Now() q.queue = append(q.queue, req) - q.perUser[req.userKey]++ + if q.redis == nil { + q.perUser[req.userKey]++ + } q.totalEnqueued.Add(1) pos := len(q.queue) - 1 @@ -146,10 +184,13 @@ func (q *qaQueue) Remove(userKey string) bool { if req.userKey == userKey { req.cancel() q.queue = append(q.queue[:i], q.queue[i+1:]...) - q.perUser[userKey]-- - if q.perUser[userKey] <= 0 { - delete(q.perUser, userKey) + if q.redis == nil { + q.perUser[userKey]-- + if q.perUser[userKey] <= 0 { + delete(q.perUser, userKey) + } } + q.redisDecrUser(context.Background(), userKey) return true } } @@ -182,12 +223,14 @@ func (q *qaQueue) runWorker(id int) { // Skip requests that have been cancelled or timed out while queued. if req.ctx.Err() != nil { q.totalTimeout.Add(1) + q.redisDecrUser(context.Background(), req.userKey) continue } waitDuration := time.Since(req.enqueuedAt) if waitDuration > queueTimeout { q.totalTimeout.Add(1) + q.redisDecrUser(context.Background(), req.userKey) logger.Warnf(req.ctx, "[IM] Queue timeout: user=%s waited=%s, discarding", req.msg.UserID, waitDuration) _ = req.adapter.SendReply(req.ctx, req.msg, &ReplyMessage{ Content: "您的消息等待超时,请重新发送。", @@ -200,10 +243,22 @@ func (q *qaQueue) runWorker(id int) { logger.Infof(req.ctx, "[IM] Dequeued: worker=%d user=%s waited=%s depth=%d", id, req.msg.UserID, waitDuration, q.Metrics().Depth) + // Acquire global concurrency slot (blocks until a slot opens or request is cancelled). + if !q.acquireGlobalGate(req.ctx) { + // Context cancelled while waiting for a global slot — treat as timeout. + q.totalTimeout.Add(1) + q.redisDecrUser(context.Background(), req.userKey) + logger.Warnf(req.ctx, "[IM] Global gate wait cancelled: worker=%d user=%s", id, req.msg.UserID) + req.cancel() + continue + } + q.activeWorkers.Add(1) q.handler(req) q.activeWorkers.Add(-1) q.totalProcessed.Add(1) + q.releaseGlobalGate() + q.redisDecrUser(context.Background(), req.userKey) } } @@ -221,14 +276,112 @@ func (q *qaQueue) dequeue() *qaRequest { req := q.queue[0] q.queue = q.queue[1:] - q.perUser[req.userKey]-- - if q.perUser[req.userKey] <= 0 { - delete(q.perUser, req.userKey) + if q.redis == nil { + q.perUser[req.userKey]-- + if q.perUser[req.userKey] <= 0 { + delete(q.perUser, req.userKey) + } } return req } +// ── Redis global concurrency gate ──────────────────────────────────────────── + +// globalGateScript atomically increments the global active-worker counter and +// checks whether the limit is exceeded. Returns 1 if the slot was acquired, 0 +// if the limit is reached. On success the caller MUST call releaseGlobalGate. +// +// KEYS[1] = RedisKeyGlobalGate +// ARGV[1] = max allowed concurrent workers +// ARGV[2] = TTL in milliseconds (safety net) +var globalGateScript = redis.NewScript(` +local key = KEYS[1] +local maxW = tonumber(ARGV[1]) +local ttlMs = tonumber(ARGV[2]) + +local count = redis.call('INCR', key) +redis.call('PEXPIRE', key, ttlMs) +if count <= maxW then + return 1 +end +redis.call('DECR', key) +return 0 +`) + +// acquireGlobalGate blocks until a global concurrency slot is available. +// Returns true if the slot was acquired, false if ctx was cancelled while waiting. +// When globalMaxWorkers is 0 or Redis is nil, it returns true immediately (no limit). +func (q *qaQueue) acquireGlobalGate(ctx context.Context) bool { + if q.globalMaxWorkers <= 0 || q.redis == nil { + return true + } + + for { + result, err := globalGateScript.Run(ctx, q.redis, + []string{RedisKeyGlobalGate}, + q.globalMaxWorkers, globalGateTTL.Milliseconds(), + ).Int64() + if err != nil { + // Redis error — skip global check to avoid blocking the worker. + logger.Warnf(ctx, "[IM] Global gate Redis error (proceeding without limit): %v", err) + return true + } + if result == 1 { + return true + } + + // Global limit reached — wait and retry. + select { + case <-ctx.Done(): + return false + case <-time.After(globalGateRetryInterval): + } + } +} + +// releaseGlobalGate decrements the global active-worker counter. +func (q *qaQueue) releaseGlobalGate() { + if q.globalMaxWorkers <= 0 || q.redis == nil { + return + } + q.redis.Decr(context.Background(), RedisKeyGlobalGate) +} + +// ── Redis global per-user counting ────────────────────────────────────────── + +// redisCheckAndIncrUser atomically increments the global per-user counter and +// returns an error if the limit is exceeded. On success the caller MUST later +// call redisDecrUser to release the slot. +func (q *qaQueue) redisCheckAndIncrUser(ctx context.Context, userKey string) error { + if q.redis == nil { + return nil + } + key := RedisKeyQueueUser + userKey + count, err := q.redis.Incr(ctx, key).Result() + if err != nil { + // Redis error — skip global check, rely on local limit. + return nil + } + q.redis.Expire(ctx, key, redisQueueUserTTL) + if count > int64(q.maxPerUser) { + q.redis.Decr(ctx, key) + return fmt.Errorf("global per-user queue limit reached (%d/%d)", count, q.maxPerUser) + } + return nil +} + +// redisDecrUser releases one slot in the global per-user counter. +func (q *qaQueue) redisDecrUser(ctx context.Context, userKey string) { + if q.redis == nil { + return + } + key := RedisKeyQueueUser + userKey + q.redis.Decr(ctx, key) +} + +// ── Metrics logging ───────────────────────────────────────────────────────── + const metricsLogInterval = 30 * time.Second // metricsLoop periodically logs queue metrics for operational visibility. diff --git a/internal/im/ratelimit.go b/internal/im/ratelimit.go index b698c5ab..6f1d891d 100644 --- a/internal/im/ratelimit.go +++ b/internal/im/ratelimit.go @@ -1,8 +1,12 @@ package im import ( + "context" + "fmt" "sync" "time" + + "github.com/redis/go-redis/v9" ) const ( @@ -14,6 +18,95 @@ const ( rateLimitCleanupInterval = 1 * time.Minute ) +// ────────────────────────────────────────────────────────────────────────────── +// distributedLimiter: Redis ZSET + local fallback +// ────────────────────────────────────────────────────────────────────────────── + +// rateLimitScript is an atomic Lua script that implements a sliding-window rate +// limiter on a Redis Sorted Set. It prunes expired entries, checks the count, +// and conditionally adds a new member — all in a single round-trip. +// +// KEYS[1] = the rate-limit key +// ARGV[1] = now (Unix milliseconds) +// ARGV[2] = window size (milliseconds) +// ARGV[3] = max allowed requests +// ARGV[4] = unique member value (e.g. now_ms as string) +// +// Returns 1 if the request is allowed, 0 if rate-limited. +var rateLimitScript = redis.NewScript(` +local key = KEYS[1] +local now = tonumber(ARGV[1]) +local window = tonumber(ARGV[2]) +local maxReq = tonumber(ARGV[3]) +local member = ARGV[4] + +redis.call('ZREMRANGEBYSCORE', key, 0, now - window) +local count = redis.call('ZCARD', key) +if count < maxReq then + redis.call('ZADD', key, now, member) + redis.call('PEXPIRE', key, window + 1000) + return 1 +end +return 0 +`) + +// distributedLimiter tries Redis first, falls back to a local sliding-window +// limiter when Redis is unavailable (nil client or transient error). +type distributedLimiter struct { + redisClient *redis.Client + local *slidingWindowLimiter + window time.Duration + maxRequests int + instanceID string // used to disambiguate ZSET members across instances +} + +func newDistributedLimiter(redisClient *redis.Client, window time.Duration, maxRequests int, instanceID string) *distributedLimiter { + return &distributedLimiter{ + redisClient: redisClient, + local: newSlidingWindowLimiter(window, maxRequests), + window: window, + maxRequests: maxRequests, + instanceID: instanceID, + } +} + +// Allow returns true if the request for the given key is within the rate limit. +func (d *distributedLimiter) Allow(key string) bool { + if d.redisClient != nil { + allowed, err := d.redisAllow(context.Background(), key) + if err == nil { + return allowed + } + // Redis failed — fall through to local limiter. + } + return d.local.Allow(key) +} + +func (d *distributedLimiter) redisAllow(ctx context.Context, key string) (bool, error) { + redisKey := RedisKeyRateLimit + key + nowMs := time.Now().UnixMilli() + windowMs := d.window.Milliseconds() + member := fmt.Sprintf("%s:%d", d.instanceID, nowMs) // instanceID prevents ZSET member collision across instances + + result, err := rateLimitScript.Run(ctx, d.redisClient, + []string{redisKey}, + nowMs, windowMs, d.maxRequests, member, + ).Int64() + if err != nil { + return false, err + } + return result == 1, nil +} + +// cleanupLoop delegates to the local limiter's cleanup for the fallback path. +func (d *distributedLimiter) cleanupLoop(stopCh <-chan struct{}) { + d.local.cleanupLoop(stopCh) +} + +// ────────────────────────────────────────────────────────────────────────────── +// slidingWindowLimiter: local in-memory fallback (original implementation) +// ────────────────────────────────────────────────────────────────────────────── + // rateLimitEntry holds the request timestamps for a single key. type rateLimitEntry struct { mu sync.Mutex diff --git a/internal/im/service.go b/internal/im/service.go index 1a50c585..a1601c92 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -11,12 +11,18 @@ import ( "sync" "time" + "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/tracing" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "gorm.io/gorm" ) @@ -34,17 +40,51 @@ const ( streamFlushInterval = 300 * time.Millisecond ) +const ( + // wsLeaderTTL is the TTL for the Redis key used for WebSocket leader election. + wsLeaderTTL = 15 * time.Second + // wsLeaderRenewInterval is how often the leader renews its lock. + wsLeaderRenewInterval = 5 * time.Second + // wsLeaderRetryInterval is how often non-leader instances try to acquire the lock. + wsLeaderRetryInterval = 10 * time.Second + // stopMarkerTTL is the TTL for cross-instance /stop markers in Redis. + stopMarkerTTL = 30 * time.Second + // stopPollInterval is how often in-flight workers check for remote /stop signals. + stopPollInterval = 500 * time.Millisecond +) + +// ── Redis key prefixes ────────────────────────────────────────────────────── +// All IM-related Redis keys are defined here for discoverability and to avoid +// scattered string literals across multiple files. +const ( + RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election + RedisKeyDedup = "im:dedup:" // + messageID — message deduplication + RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution) + RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop + RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter + RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting + RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter +) + // channelState holds runtime state for a running IM channel. type channelState struct { - Channel *IMChannel - Adapter Adapter - Cancel context.CancelFunc // for stopping websocket goroutines + Channel *IMChannel + Adapter Adapter + Cancel context.CancelFunc // for stopping websocket goroutines + leaderCancel context.CancelFunc // stops the leader renewal goroutine (nil if not leader) } // AdapterFactory creates an Adapter from an IMChannel configuration. // The second return value is an optional cleanup function (e.g., for stopping websocket connections). type AdapterFactory func(ctx context.Context, channel *IMChannel, msgHandler func(ctx context.Context, msg *IncomingMessage) error) (Adapter, context.CancelFunc, error) +// inflightEntry tracks a running QA request, keyed by userKey in the inflight map. +type inflightEntry struct { + cancel context.CancelFunc + sessionID string // set after assistant message is created + assistantMessageID string // set after assistant message is created +} + // Service orchestrates IM message handling: // 1. Receives a unified IncomingMessage from an Adapter // 2. Resolves or creates a WeKnora session for the IM channel @@ -67,6 +107,11 @@ type Service struct { // modelService is used to obtain the chat model for generating smart notification replies. modelService interfaces.ModelService + // streamManager writes/reads QA events for distributed stop detection, + // consistent with the web StopSession mechanism. May be nil in Lite mode + // (but NewStreamManager always returns at least a memory implementation). + streamManager interfaces.StreamManager + // cmdRegistry holds all registered slash-commands. cmdRegistry *CommandRegistry @@ -81,16 +126,26 @@ type Service struct { processedMsgs sync.Map // rateLimiter enforces per-user sliding window rate limiting. - rateLimiter *slidingWindowLimiter + // Uses Redis ZSET when available, falls back to local sliding window. + rateLimiter *distributedLimiter - // inflight tracks cancel functions for in-progress QA requests, keyed by - // "channelID:userID:chatID". This allows /stop to abort a running request. - inflight sync.Map // key -> context.CancelFunc + // inflight tracks in-progress QA requests, keyed by userKey + // ("channelID:userID:chatID"). Allows /stop to abort a running request + // on this instance and look up (sessionID, messageID) for StreamManager. + inflight sync.Map // userKey -> *inflightEntry // qaQueue manages bounded queuing and worker-pool execution of QA requests, // providing backpressure to protect downstream LLM resources. qaQueue *qaQueue + // redis is the optional Redis client for distributed state (dedup, rate + // limiting, leader election, cross-instance /stop). When nil the service + // falls back to local in-memory state (single-instance / Lite mode). + redis *redis.Client + + // instanceID uniquely identifies this service instance for leader election. + instanceID string + stopCh chan struct{} } @@ -123,7 +178,44 @@ func buildIMQARequest( } } +// resolveIMConfig extracts IM tuning parameters from the application config, +// falling back to built-in defaults for any zero/nil values. +func resolveIMConfig(appCfg *config.Config) (workers, maxQueue, maxPerUser, globalMaxWorkers int, rlWindow time.Duration, rlMax int) { + workers = defaultWorkers + maxQueue = defaultMaxQueueSize + maxPerUser = defaultMaxPerUser + rlWindow = rateLimitWindow + rlMax = rateLimitMaxRequests + + if appCfg == nil || appCfg.IM == nil { + return + } + im := appCfg.IM + if im.Workers > 0 { + workers = im.Workers + } + if im.MaxQueueSize > 0 { + maxQueue = im.MaxQueueSize + } + if im.MaxPerUser > 0 { + maxPerUser = im.MaxPerUser + } + if im.GlobalMaxWorkers > 0 { + globalMaxWorkers = im.GlobalMaxWorkers + } + if im.RateLimitWindow > 0 { + rlWindow = im.RateLimitWindow + } + if im.RateLimitMax > 0 { + rlMax = im.RateLimitMax + } + return +} + // NewService creates a new IM service. +// redisClient may be nil — in that case the service falls back to local +// in-memory state (Lite / single-instance mode). +// cfg may be nil — in that case built-in defaults are used. func NewService( db *gorm.DB, sessionService interfaces.SessionService, @@ -133,7 +225,13 @@ func NewService( knowledgeService interfaces.KnowledgeService, kbService interfaces.KnowledgeBaseService, modelService interfaces.ModelService, + streamManager interfaces.StreamManager, + redisClient *redis.Client, + appCfg *config.Config, ) *Service { + // Resolve IM configuration with defaults. + workers, maxQueue, maxPerUser, globalMaxWorkers, rlWindow, rlMax := resolveIMConfig(appCfg) + // Build command registry. registry := NewCommandRegistry() registry.Register(newHelpCommand(registry)) @@ -142,6 +240,7 @@ func NewService( registry.Register(newStopCommand()) registry.Register(newClearCommand()) + instanceID := uuid.New().String() s := &Service{ db: db, sessionService: sessionService, @@ -151,21 +250,40 @@ func NewService( knowledgeService: knowledgeService, kbService: kbService, modelService: modelService, + streamManager: streamManager, cmdRegistry: registry, channels: make(map[string]*channelState), adapterFactories: make(map[string]AdapterFactory), - rateLimiter: newSlidingWindowLimiter(rateLimitWindow, rateLimitMaxRequests), + rateLimiter: newDistributedLimiter(redisClient, rlWindow, rlMax, instanceID), + redis: redisClient, + instanceID: instanceID, stopCh: make(chan struct{}), } // Initialize the QA worker pool and bounded queue. - s.qaQueue = newQAQueue(defaultWorkers, defaultMaxQueueSize, defaultMaxPerUser, s.executeQARequest) + s.qaQueue = newQAQueue(workers, maxQueue, maxPerUser, globalMaxWorkers, s.executeQARequest, redisClient) s.qaQueue.Start(s.stopCh) - // Start periodic dedup cleanup instead of per-message goroutines - go s.dedupCleanupLoop() + // Start periodic cleanup loops. + // Dedup cleanup is only needed in single-instance mode (local sync.Map); + // when Redis handles dedup, the TTL on Redis keys handles expiry automatically. + if redisClient == nil { + go s.dedupCleanupLoop() + } go s.rateLimiter.cleanupLoop(s.stopCh) + if redisClient != nil { + globalInfo := "unlimited" + if globalMaxWorkers > 0 { + globalInfo = fmt.Sprintf("%d", globalMaxWorkers) + } + logger.Infof(context.Background(), "[IM] Multi-instance mode enabled (instance=%s, workers=%d, queue=%d, global_max=%s)", + s.instanceID[:8], workers, maxQueue, globalInfo) + } else { + logger.Infof(context.Background(), "[IM] Single-instance mode (no Redis, workers=%d, queue=%d)", + workers, maxQueue) + } + return s } @@ -183,10 +301,7 @@ func (s *Service) Stop() { s.mu.Lock() defer s.mu.Unlock() for id, cs := range s.channels { - if cs.Cancel != nil { - cs.Cancel() - } - delete(s.channels, id) + s.stopChannelLocked(id, cs) } } @@ -233,7 +348,18 @@ func (s *Service) LoadAndStartChannels() error { } // StartChannel creates and registers an adapter for the given channel. +// For WebSocket channels with Redis available, only one instance acquires +// the leader lock and opens the connection; other instances periodically +// retry so they can take over if the leader dies. func (s *Service) StartChannel(channel *IMChannel) error { + _, span := tracing.ContextWithSpan(context.Background(), "im.StartChannel") + defer span.End() + span.SetAttributes( + attribute.String("im.channel_id", channel.ID), + attribute.String("im.platform", channel.Platform), + attribute.String("im.mode", channel.Mode), + ) + s.mu.Lock() factory, ok := s.adapterFactories[channel.Platform] if !ok { @@ -242,13 +368,26 @@ func (s *Service) StartChannel(channel *IMChannel) error { } // Stop existing channel if running if existing, ok := s.channels[channel.ID]; ok { - if existing.Cancel != nil { - existing.Cancel() - } - delete(s.channels, channel.ID) + s.stopChannelLocked(channel.ID, existing) } s.mu.Unlock() + // For WebSocket channels, try leader election to avoid duplicate connections. + if channel.Mode == "websocket" && s.redis != nil { + acquired := s.tryAcquireWSLeader(channel.ID) + if !acquired { + logger.Infof(context.Background(), + "[IM] Channel %s WebSocket owned by another instance, will retry", channel.ID) + go s.wsLeaderRetryLoop(channel) + return nil + } + } + + return s.startChannelInternal(channel, factory) +} + +// startChannelInternal does the actual adapter creation and registration. +func (s *Service) startChannelInternal(channel *IMChannel, factory AdapterFactory) error { // Build the message handler that delegates to HandleMessage with this channel's config msgHandler := func(msgCtx context.Context, msg *IncomingMessage) error { return s.HandleMessage(msgCtx, msg, channel.ID) @@ -257,14 +396,24 @@ func (s *Service) StartChannel(channel *IMChannel) error { ctx := context.Background() adapter, cancelFn, err := factory(ctx, channel, msgHandler) if err != nil { + s.releaseWSLeader(channel.ID) // release lock on failure return fmt.Errorf("create adapter: %w", err) } + // Start leader renewal goroutine for WebSocket channels. + var leaderCancel context.CancelFunc + if channel.Mode == "websocket" && s.redis != nil { + leaderCtx, lCancel := context.WithCancel(context.Background()) + leaderCancel = lCancel + go s.wsLeaderRenewLoop(leaderCtx, channel.ID) + } + s.mu.Lock() s.channels[channel.ID] = &channelState{ - Channel: channel, - Adapter: adapter, - Cancel: cancelFn, + Channel: channel, + Adapter: adapter, + Cancel: cancelFn, + leaderCancel: leaderCancel, } s.mu.Unlock() @@ -276,11 +425,230 @@ func (s *Service) StopChannel(channelID string) { s.mu.Lock() defer s.mu.Unlock() if cs, ok := s.channels[channelID]; ok { - if cs.Cancel != nil { - cs.Cancel() + s.stopChannelLocked(channelID, cs) + } +} + +// stopChannelLocked stops a channel and removes it from the map. +// Caller must hold s.mu. +func (s *Service) stopChannelLocked(channelID string, cs *channelState) { + if cs.leaderCancel != nil { + cs.leaderCancel() + } + if cs.Cancel != nil { + cs.Cancel() + } + delete(s.channels, channelID) + s.releaseWSLeader(channelID) + logger.Infof(context.Background(), "[IM] Stopped channel: id=%s", channelID) +} + +// ── WebSocket leader election ─────────────────────────────────────────────── + +// tryAcquireWSLeader attempts to acquire the Redis lock for a WebSocket channel. +// Returns true if this instance is now the leader. +func (s *Service) tryAcquireWSLeader(channelID string) bool { + if s.redis == nil { + return true // single-instance mode: always leader + } + key := RedisKeyLeader + channelID + ok, err := s.redis.SetNX(context.Background(), key, s.instanceID, wsLeaderTTL).Result() + if err != nil { + logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v, assuming leader", channelID, err) + return true // Redis error: proceed anyway to avoid channel getting stuck + } + return ok +} + +// releaseWSLeader releases the Redis leader lock for a WebSocket channel, +// but only if this instance owns it. +func (s *Service) releaseWSLeader(channelID string) { + if s.redis == nil { + return + } + key := RedisKeyLeader + channelID + // Only delete if we own it (compare-and-delete via Lua). + script := redis.NewScript(` + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + `) + script.Run(context.Background(), s.redis, []string{key}, s.instanceID) +} + +// wsLeaderRenewLoop periodically refreshes the leader lock TTL. +// Stops when ctx is cancelled (channel stopped) or if the lock is lost. +func (s *Service) wsLeaderRenewLoop(ctx context.Context, channelID string) { + key := RedisKeyLeader + channelID + ticker := time.NewTicker(wsLeaderRenewInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // Only renew if we still own the lock. + script := redis.NewScript(` + if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('PEXPIRE', KEYS[1], ARGV[2]) + return 1 + end + return 0 + `) + result, err := script.Run(ctx, s.redis, []string{key}, s.instanceID, wsLeaderTTL.Milliseconds()).Int64() + if err != nil || result == 0 { + logger.Warnf(context.Background(), + "[IM] Lost leadership for channel %s, stopping adapter", channelID) + s.StopChannel(channelID) + return + } + case <-ctx.Done(): + return + } + } +} + +// wsLeaderRetryLoop periodically tries to acquire the WebSocket leader lock. +// When it succeeds, it starts the channel adapter. +func (s *Service) wsLeaderRetryLoop(channel *IMChannel) { + ticker := time.NewTicker(wsLeaderRetryInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // Check if channel is already running (another goroutine may have started it). + if _, _, ok := s.GetChannelAdapter(channel.ID); ok { + return + } + if s.tryAcquireWSLeader(channel.ID) { + logger.Infof(context.Background(), + "[IM] Acquired leadership for channel %s, starting adapter", channel.ID) + s.mu.RLock() + factory, ok := s.adapterFactories[channel.Platform] + s.mu.RUnlock() + if !ok { + return + } + if err := s.startChannelInternal(channel, factory); err != nil { + logger.Warnf(context.Background(), + "[IM] Failed to start channel %s after acquiring leadership: %v", channel.ID, err) + } + return + } + case <-s.stopCh: + return + } + } +} + +// ── Cross-instance /stop via StreamManager ─────────────────────────────────── +// +// The mechanism mirrors the web StopSession flow: +// 1. /stop writes a stop StreamEvent to StreamManager (keyed by sessionID + messageID) +// 2. A per-request watcher polls StreamManager and cancels the context on detection +// +// A Redis marker (im:stop:{userKey}) is kept as a lightweight pre-execution +// check for requests that haven't created an assistant message yet. + +// checkAndClearStopMarker checks if a pre-execution /stop marker exists for +// the given userKey. If found, it deletes the marker and returns true. +func (s *Service) checkAndClearStopMarker(ctx context.Context, userKey string) bool { + if s.redis == nil { + return false + } + stopKey := RedisKeyStop + userKey + deleted, err := s.redis.Del(ctx, stopKey).Result() + if err != nil { + return false + } + return deleted > 0 +} + +// storeInflightMapping writes the (sessionID, assistantMessageID) to Redis so +// that /stop on any instance can look it up and write to StreamManager. +func (s *Service) storeInflightMapping(ctx context.Context, userKey, sessionID, messageID string) { + if s.redis == nil { + return + } + val := sessionID + ":" + messageID + if err := s.redis.Set(ctx, RedisKeyInflight+userKey, val, qaTimeout+30*time.Second).Err(); err != nil { + logger.Warnf(ctx, "[IM] Failed to store inflight mapping: %v", err) + } +} + +// clearInflightMapping removes the inflight mapping from Redis. +func (s *Service) clearInflightMapping(ctx context.Context, userKey string) { + if s.redis == nil { + return + } + s.redis.Del(ctx, RedisKeyInflight+userKey) +} + +// loadInflightMapping retrieves (sessionID, messageID) from Redis. +func (s *Service) loadInflightMapping(ctx context.Context, userKey string) (sessionID, messageID string, ok bool) { + if s.redis == nil { + return "", "", false + } + val, err := s.redis.Get(ctx, RedisKeyInflight+userKey).Result() + if err != nil { + return "", "", false + } + parts := strings.SplitN(val, ":", 2) + if len(parts) != 2 { + return "", "", false + } + return parts[0], parts[1], true +} + +// writeStopEvent writes a stop event to StreamManager, matching the web +// StopSession pattern. The QA watcher goroutine detects it and cancels. +func (s *Service) writeStopEvent(ctx context.Context, sessionID, messageID string) { + stopEvt := interfaces.StreamEvent{ + ID: fmt.Sprintf("stop-%d", time.Now().UnixNano()), + Type: types.ResponseType(event.EventStop), + Content: "", + Done: true, + Timestamp: time.Now(), + Data: map[string]interface{}{ + "session_id": sessionID, + "message_id": messageID, + "reason": "user_requested", + "source": "im", + }, + } + if err := s.streamManager.AppendEvent(ctx, sessionID, messageID, stopEvt); err != nil { + logger.Warnf(ctx, "[IM] Failed to write stop event to StreamManager: %v", err) + } +} + +// watchStreamManagerStop polls StreamManager for stop events and cancels the +// QA context when one is detected. This is the IM equivalent of the web SSE +// handler's stop detection loop. Exits when ctx is done. +func (s *Service) watchStreamManagerStop(ctx context.Context, sessionID, messageID string, cancel context.CancelFunc) { + ticker := time.NewTicker(stopPollInterval) + defer ticker.Stop() + + offset := 0 + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + events, newOffset, err := s.streamManager.GetEvents(ctx, sessionID, messageID, offset) + if err != nil { + continue + } + for _, evt := range events { + if evt.Type == types.ResponseType(event.EventStop) { + logger.Infof(ctx, "[IM] Stop event from StreamManager, cancelling: session=%s message=%s", + sessionID, messageID) + cancel() + return + } + } + offset = newOffset } - delete(s.channels, channelID) - logger.Infof(context.Background(), "[IM] Stopped channel: id=%s", channelID) } } @@ -313,11 +681,47 @@ func (s *Service) GetChannelByIDAndTenant(channelID string, tenantID uint64) (*I return &ch, nil } +// isDuplicate checks if a message has already been processed. +// +// Multi-instance mode (Redis available): uses Redis SetNX for cross-instance +// deduplication. If Redis fails, returns true (fail-closed) to prevent +// duplicate processing across instances — a dropped message can be retried +// by the user, but a duplicate LLM response wastes resources and confuses. +// +// Single-instance mode (no Redis): uses a local sync.Map, which is sufficient +// when only one instance receives messages. +func (s *Service) isDuplicate(ctx context.Context, messageID string) bool { + if s.redis != nil { + key := RedisKeyDedup + messageID + ok, err := s.redis.SetNX(ctx, key, "1", dedupTTL).Result() + if err == nil { + return !ok // SetNX returns true when key was newly set (not a duplicate) + } + // Redis is configured but failed — fail-closed to avoid cross-instance + // duplicate processing. The user can simply resend the message. + logger.Errorf(ctx, "[IM] Redis dedup failed (fail-closed, message dropped): %v", err) + return true + } + // Single-instance mode: local dedup is sufficient. + _, loaded := s.processedMsgs.LoadOrStore(messageID, time.Now()) + return loaded +} + // HandleMessage processes an incoming IM message end-to-end using channel config. func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, channelID string) error { + ctx, span := tracing.ContextWithSpan(ctx, "im.HandleMessage") + defer span.End() + span.SetAttributes( + attribute.String("im.channel_id", channelID), + attribute.String("im.platform", string(msg.Platform)), + attribute.String("im.user_id", msg.UserID), + attribute.String("im.chat_id", msg.ChatID), + attribute.String("im.message_type", string(msg.MessageType)), + ) + // Dedup: skip if this message was already processed (IM platforms may retry) if msg.MessageID != "" { - if _, loaded := s.processedMsgs.LoadOrStore(msg.MessageID, time.Now()); loaded { + if s.isDuplicate(ctx, msg.MessageID) { logger.Infof(ctx, "[IM] Skipping duplicate message: %s", msg.MessageID) return nil } @@ -444,6 +848,7 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann pos, enqueueErr := s.qaQueue.Enqueue(req) if enqueueErr != nil { qaCancel() + span.AddEvent("queue rejected", trace.WithAttributes(attribute.String("reason", enqueueErr.Error()))) logger.Warnf(ctx, "[IM] Queue rejected: user=%s reason=%v", msg.UserID, enqueueErr) _ = adapter.SendReply(ctx, msg, &ReplyMessage{ Content: "当前排队人数较多,请稍后再试。", @@ -454,8 +859,14 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann if pos > 0 { logger.Infof(ctx, "[IM] Enqueued: user=%s pos=%d depth=%d", msg.UserID, pos, s.qaQueue.Metrics().Depth) + // In multi-instance mode the local queue position does not reflect global + // depth, so use a generic "queued" hint instead of an exact number. + queueMsg := fmt.Sprintf("收到,前面还有 %d 条消息在处理,请稍候 ⏳", pos) + if s.redis != nil { + queueMsg = "收到,当前排队中,请稍候 ⏳" + } _ = adapter.SendReply(ctx, msg, &ReplyMessage{ - Content: fmt.Sprintf("收到,前面还有 %d 条消息在处理,请稍候 ⏳", pos), + Content: queueMsg, IsFinal: true, }) } else { @@ -468,13 +879,31 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann // executeQARequest is the worker handler that runs the QA pipeline for a queued request. // It is called by qaQueue workers and must not block indefinitely. func (s *Service) executeQARequest(req *qaRequest) { - ctx := req.ctx + ctx, span := tracing.ContextWithSpan(req.ctx, "im.ExecuteQA") + defer span.End() + span.SetAttributes( + attribute.String("im.channel_id", req.channelID), + attribute.String("im.user_key", req.userKey), + attribute.String("im.user_id", req.msg.UserID), + ) defer req.cancel() // Track in-flight request so /stop can cancel it. - s.inflight.Store(req.userKey, req.cancel) + entry := &inflightEntry{cancel: req.cancel} + s.inflight.Store(req.userKey, entry) defer s.inflight.Delete(req.userKey) + // Check if a pre-execution /stop was issued while this request was queued. + if s.checkAndClearStopMarker(ctx, req.userKey) { + span.AddEvent("cancelled by remote /stop before execution") + logger.Infof(ctx, "[IM] Request cancelled by remote /stop before execution: %s", req.userKey) + return + } + + // NOTE: StreamManager-based stop detection is started inside handleMessageStream / + // runQA after the assistant message is created (that's when we have the + // sessionID + messageID needed to poll StreamManager). + // kbIDs is left empty so the QA pipeline resolves them from the agent config. var kbIDs []string @@ -484,7 +913,8 @@ func (s *Service) executeQARequest(req *qaRequest) { // If the adapter supports streaming and output is not "full", use streaming. if !streamDisabled { if streamer, ok := req.adapter.(StreamSender); ok { - if err := s.handleMessageStream(ctx, req.msg, req.session, req.agent, kbIDs, streamer, req.adapter); err != nil { + if err := s.handleMessageStream(ctx, req.msg, req.session, req.agent, kbIDs, streamer, req.adapter, req.userKey); err != nil { + span.SetStatus(codes.Error, err.Error()) logger.Errorf(ctx, "[IM] Stream QA failed: %v", err) } return @@ -492,8 +922,9 @@ func (s *Service) executeQARequest(req *qaRequest) { } // Non-streaming fallback: collect full answer then send. - answer, err := s.runQA(ctx, req.session, req.msg.Content, req.agent, kbIDs) + answer, err := s.runQA(ctx, req.session, req.msg.Content, req.agent, kbIDs, req.userKey) if err != nil { + span.SetStatus(codes.Error, err.Error()) logger.Errorf(ctx, "[IM] QA failed: %v, sending fallback reply", err) answer = "抱歉,处理您的问题时出现了异常,请稍后再试。" } @@ -523,6 +954,14 @@ func (s *Service) handleCommand( channelSession *ChannelSession, customAgent *types.CustomAgent, ) error { + ctx, span := tracing.ContextWithSpan(ctx, "im.HandleCommand") + defer span.End() + span.SetAttributes( + attribute.String("im.command", cmd.Name()), + attribute.String("im.channel_id", channel.ID), + attribute.String("im.user_id", msg.UserID), + ) + agentName := "" if customAgent != nil { agentName = customAgent.Name @@ -561,14 +1000,44 @@ func (s *Service) handleCommand( logger.Warnf(ctx, "[IM] Failed to clear session context: %v", err) } case ActionStop: - // Cancel the request — first check if it's queued, then check if it's in-flight. inflightKey := makeUserKey(channel.ID, msg.UserID, msg.ChatID) - if s.qaQueue.Remove(inflightKey) { + + // 1. Try local cancel: remove from queue or cancel in-flight. + var localSessionID, localMessageID string + localStopped := s.qaQueue.Remove(inflightKey) + if localStopped { logger.Infof(ctx, "[IM] Cancelled queued QA: key=%s", inflightKey) - } else if cancelFn, loaded := s.inflight.LoadAndDelete(inflightKey); loaded { - cancelFn.(context.CancelFunc)() + } else if raw, loaded := s.inflight.LoadAndDelete(inflightKey); loaded { + e := raw.(*inflightEntry) + e.cancel() + localStopped = true + localSessionID = e.sessionID + localMessageID = e.assistantMessageID logger.Infof(ctx, "[IM] Cancelled in-flight QA: key=%s", inflightKey) } + + // 2. Write stop event to StreamManager (same as web StopSession). + // For local stop with known IDs, write directly. + // For cross-instance, look up Redis inflight mapping to get IDs. + sessionID, messageID := localSessionID, localMessageID + if sessionID == "" || messageID == "" { + // Try cross-instance lookup. + sessionID, messageID, _ = s.loadInflightMapping(ctx, inflightKey) + } + if sessionID != "" && messageID != "" { + s.writeStopEvent(ctx, sessionID, messageID) + logger.Infof(ctx, "[IM] Wrote stop event to StreamManager: session=%s message=%s", sessionID, messageID) + } + + // 3. Set Redis marker as fallback for requests not yet executing + // (no assistant message yet → no StreamManager entry to poll). + if s.redis != nil { + s.redis.Set(ctx, RedisKeyStop+inflightKey, "1", stopMarkerTTL) + } + + if !localStopped && sessionID == "" { + logger.Infof(ctx, "[IM] Set cross-instance stop marker (no inflight found): key=%s", inflightKey) + } } // Send the command reply, respecting the configured output mode. @@ -656,8 +1125,14 @@ func (s *Service) resolveSession(ctx context.Context, msg *IncomingMessage, tena IMChannelID: imChannelID, } if err := s.db.Create(&cs).Error; err != nil { - // If the insert failed due to unique constraint (concurrent request), - // fetch the existing record. + // The insert failed (likely unique constraint from a concurrent request on + // another instance). Clean up the orphaned Session we just created — it has + // no messages yet, so a direct delete is safe. + if delErr := s.db.Where("id = ?", createdSession.ID).Delete(createdSession).Error; delErr != nil { + logger.Warnf(ctx, "[IM] Failed to clean up orphaned session %s: %v", createdSession.ID, delErr) + } + + // Fetch the existing ChannelSession created by the winning instance. var existing ChannelSession if findErr := s.db.Where("platform = ? AND user_id = ? AND chat_id = ? AND tenant_id = ? AND deleted_at IS NULL", string(msg.Platform), msg.UserID, msg.ChatID, tenantID). @@ -773,12 +1248,19 @@ func briefToolSummary(output string) string { // handleMessageStream runs the QA pipeline and streams answer chunks to the IM platform // in real-time via the StreamSender interface. Chunks are batched at streamFlushInterval // to avoid API rate-limiting. -func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, streamer StreamSender, adapter Adapter) error { +func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, streamer StreamSender, adapter Adapter, userKey string) error { + ctx, span := tracing.ContextWithSpan(ctx, "im.StreamQA") + defer span.End() + span.SetAttributes( + attribute.String("im.user_id", msg.UserID), + attribute.String("im.platform", string(msg.Platform)), + ) + // Start the stream on the IM platform (e.g., create Feishu streaming card) streamID, err := streamer.StartStream(ctx, msg) if err != nil { logger.Warnf(ctx, "[IM] StartStream failed, falling back to non-streaming: %v", err) - return s.fallbackNonStream(ctx, msg, session, customAgent, kbIDs, adapter) + return s.fallbackNonStream(ctx, msg, session, customAgent, kbIDs, adapter, userKey) } // Prepare the QA pipeline @@ -956,6 +1438,20 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, return fmt.Errorf("create assistant message: %w", err) } + // Register inflight mapping so cross-instance /stop can find this request + // and write a stop event to StreamManager. + if raw, ok := s.inflight.Load(userKey); ok { + e := raw.(*inflightEntry) + e.sessionID = session.ID + e.assistantMessageID = assistantMsg.ID + } + s.storeInflightMapping(qaCtx, userKey, session.ID, assistantMsg.ID) + defer s.clearInflightMapping(ctx, userKey) + + // Start StreamManager stop watcher — mirrors web's handleAgentEventsForSSE + // stop detection. Cancels qaCtx if a stop event is written by any instance. + go s.watchStreamManagerStop(qaCtx, session.ID, assistantMsg.ID, qaCancel) + // Run QA async go func() { var err error @@ -1048,8 +1544,8 @@ loop: } // fallbackNonStream is used when streaming initialization fails. -func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, adapter Adapter) error { - answer, err := s.runQA(ctx, session, msg.Content, customAgent, kbIDs) +func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, adapter Adapter, userKey string) error { + answer, err := s.runQA(ctx, session, msg.Content, customAgent, kbIDs, userKey) if err != nil { logger.Errorf(ctx, "[IM] QA fallback failed: %v", err) answer = "抱歉,处理您的问题时出现了异常,请稍后再试。" @@ -1059,7 +1555,7 @@ func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, s } // runQA executes the WeKnora QA pipeline and returns the full answer text. -func (s *Service) runQA(ctx context.Context, session *types.Session, query string, customAgent *types.CustomAgent, kbIDs []string) (string, error) { +func (s *Service) runQA(ctx context.Context, session *types.Session, query string, customAgent *types.CustomAgent, kbIDs []string, userKey string) (string, error) { // Add timeout to prevent indefinite blocking ctx, cancel := context.WithTimeout(ctx, qaTimeout) defer cancel() @@ -1132,6 +1628,18 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin return "", fmt.Errorf("create assistant message: %w", err) } + // Register inflight mapping for cross-instance /stop via StreamManager. + if raw, ok := s.inflight.Load(userKey); ok { + e := raw.(*inflightEntry) + e.sessionID = session.ID + e.assistantMessageID = assistantMsg.ID + } + s.storeInflightMapping(ctx, userKey, session.ID, assistantMsg.ID) + defer s.clearInflightMapping(ctx, userKey) + + // Start StreamManager stop watcher. + go s.watchStreamManagerStop(ctx, session.ID, assistantMsg.ID, cancel) + // Run QA async go func() { var err error