diff --git a/.env.example b/.env.example index 824d4eba..b3adb532 100644 --- a/.env.example +++ b/.env.example @@ -624,3 +624,23 @@ DOCREADER_TRANSPORT=grpc # Document processing task timeout. Large files may need more than Asynq's default 30m. # WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h + +# --- Async pipeline tuning (optional) ----------------------------------------- +# Worker pool size for the asynq server. Default 16 — raise it on machines +# that handle many concurrent uploads (default Go runtime.NumCPU() under- +# provisions for the I/O-bound document pipeline). +# WEKNORA_ASYNQ_CONCURRENCY=16 + +# Read/write timeout (in milliseconds) the asynq client uses against Redis. +# Default 500ms (writes scale to 1000ms). Bump if you see "i/o timeout" +# errors during heavy multimodal counter DECRs or large batch uploads. +# WEKNORA_REDIS_OP_TIMEOUT_MS=500 + +# Maximum wall time for a single DocReader call. Caps a hung docreader +# (network partition, OCR runaway) so it cannot pin a worker for hours. +# Default 30m. Must be shorter than WEKNORA_DOCUMENT_PROCESS_TIMEOUT. +# WEKNORA_DOCREADER_CALL_TIMEOUT=30m + +# Background sweep that recovers knowledge stuck in "processing" past +# DocumentProcessTimeout + 10m. Set to "false" to opt out (not recommended). +# WEKNORA_HOUSEKEEPING_ENABLED=true diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 6b97a0d1..f0de2ba1 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -115,20 +115,36 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } + // finalize-once semantics: on success we always decrement the parent's + // pending counter. On failure we only decrement when this is the last + // asynq retry, so a permanently-failing single image cannot leave the + // parent knowledge stuck in "processing" forever — which was the #1 + // cause of "stuck parsing" reports. Intermediate retries skip finalize + // so we don't double-count and prematurely trigger post-process. + var handleErr error + defer func() { + if handleErr == nil || isFinalAsynqAttempt(ctx) { + s.checkAndFinalizeAllImages(ctx, payload) + } else { + logger.Infof(ctx, + "[ImageMultimodal] Skip finalize on retryable error for %s (will count on last attempt)", + payload.ImageURL) + } + }() + vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID) if err != nil { - return fmt.Errorf("resolve VLM: %w", err) + handleErr = fmt.Errorf("resolve VLM: %w", err) + return handleErr } // Read image bytes. A provider:// URL must be resolved via FileService — // it must NEVER be handed to the HTTP downloader (which would fail with // "unsupported URL scheme"). On unrecoverable read failure for a single - // image, skip it and still trigger finalize so the parent knowledge - // doesn't get stuck in "processing" forever (see issue #1282). + // image, skip it (deferred finalize will count it). imgBytes, readErr := s.readImageBytes(ctx, payload) if readErr != nil { logger.Errorf(ctx, "[ImageMultimodal] Skip unreadable image %s: %v", payload.ImageURL, readErr) - s.checkAndFinalizeAllImages(ctx, payload) return nil } @@ -203,13 +219,14 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e } if len(newChunks) == 0 { - s.checkAndFinalizeAllImages(ctx, payload) + // Deferred finalize will count this image on success. return nil } // Persist chunks if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil { - return fmt.Errorf("create multimodal chunks: %w", err) + handleErr = fmt.Errorf("create multimodal chunks: %w", err) + return handleErr } for _, c := range newChunks { logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d", @@ -225,11 +242,32 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e // have real textual content (caption/OCR), we can generate questions. // Note: for documents with multiple images (e.g. PDFs), we also wait until // all images are processed before triggering summary/question generation. - s.checkAndFinalizeAllImages(ctx, payload) - + // Deferred finalize handles the parent knowledge counter. return nil } +// isFinalAsynqAttempt reports whether the current task context belongs to the +// last retry attempt before asynq archives the task as a dead-letter. We use +// this to flip multimodal finalize semantics: during normal retries we skip +// counter decrement (the retry might still succeed), but on the final attempt +// we count the image regardless of outcome so a permanently-failing image +// cannot pin the parent knowledge in "processing" forever. +// +// Returns false when the values are unavailable (e.g. when the handler is +// invoked outside an asynq worker, as in unit tests). Treating that case as +// "not final" keeps test ergonomics — tests should drive finalize explicitly. +func isFinalAsynqAttempt(ctx context.Context) bool { + retried, ok := asynq.GetRetryCount(ctx) + if !ok { + return false + } + maxRetry, ok := asynq.GetMaxRetry(ctx) + if !ok { + return false + } + return retried >= maxRetry +} + // indexChunks indexes the newly created multimodal chunks into the retrieval engine // so they can participate in semantic search. func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) { @@ -433,7 +471,16 @@ func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result() if err != nil && err != redis.Nil { - logger.Warnf(ctx, "[ImageMultimodal] Failed to decrement pending count for %s: %v", payload.KnowledgeID, err) + // Redis hiccup must not strand the parent knowledge. Best-effort: + // enqueue post-process anyway. KnowledgePostProcess is idempotent + // (it transitions parse_status processing → completed under a row + // guard), so a duplicate triggered by a sibling image is harmless. + // The alternative — silently returning — is what produced the + // "permanently stuck" reports we are fixing here. + logger.Warnf(ctx, + "[ImageMultimodal] Decrement failed for %s (%v); fallback-enqueueing post-process", + payload.KnowledgeID, err) + s.enqueueKnowledgePostProcessTask(ctx, payload) return } diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go new file mode 100644 index 00000000..b6434623 --- /dev/null +++ b/internal/application/service/knowledge_housekeeping.go @@ -0,0 +1,157 @@ +// Package service: knowledge housekeeping. +// +// HousekeepingService periodically scans for knowledge rows that have been +// stuck in "processing" longer than any reasonable execution window and +// marks them as failed. This is the safety net that catches anything the +// other defences (asynq retry, dead-letter callback, image_multimodal +// finalize-on-last-attempt) miss — for example: +// +// - Worker process killed mid-handler before any defer could run. +// - DocReader call genuinely exceeding DocReaderCallTimeout AND the +// worker subsequently being lost before retry kicks in. +// - Multimodal Redis counter set to N but ALL N image tasks failing in +// ways that bypass finalize (extremely rare; defence-in-depth here). +// +// Without this sweep, a single unlucky failure mode can leave a knowledge +// row in "processing" forever — invisible to users except as a permanent +// spinner. With this sweep the worst-case latency from stall to user- +// visible failure is bounded to ~1 stale-threshold + 1 sweep interval. +package service + +import ( + "context" + "os" + "strings" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/robfig/cron/v3" + "gorm.io/gorm" +) + +// HousekeepingService runs background sweeps to recover stuck rows. +type HousekeepingService struct { + db *gorm.DB + cfg *config.Config + cron *cron.Cron + + mu sync.Mutex + started bool +} + +// NewHousekeepingService constructs a HousekeepingService. It does NOT start +// the cron — call Start in the application bootstrap so a misconfigured +// cron schedule cannot prevent the rest of the service from coming up. +func NewHousekeepingService(db *gorm.DB, cfg *config.Config) *HousekeepingService { + return &HousekeepingService{ + db: db, + cfg: cfg, + cron: cron.New(cron.WithSeconds(), cron.WithChain( + cron.Recover(cron.DefaultLogger), + )), + } +} + +// Start registers the sweep schedule and begins the background runner. +// Idempotent — repeated calls are a no-op so wiring code can call Start +// without coordinating ordering. +func (h *HousekeepingService) Start(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.started { + return nil + } + if !housekeepingEnabled() { + logger.Infof(ctx, "[Housekeeping] disabled via WEKNORA_HOUSEKEEPING_ENABLED=false") + return nil + } + // Every 5 minutes — frequent enough that user-visible recovery latency + // is acceptable, infrequent enough that the SQL sweep is invisible to + // query load even on large knowledge tables. + if _, err := h.cron.AddFunc("0 */5 * * * *", func() { + // Use Background so a cancelled bootstrap ctx doesn't stop sweeps. + h.runSweep(context.Background()) + }); err != nil { + return err + } + h.cron.Start() + h.started = true + logger.Infof(ctx, "[Housekeeping] started with 5-minute sweep") + return nil +} + +// Stop halts the cron and waits for in-flight sweeps to finish. +func (h *HousekeepingService) Stop() { + h.mu.Lock() + defer h.mu.Unlock() + if !h.started { + return + } + c := h.cron.Stop() + <-c.Done() + h.started = false +} + +// runSweep is exported on the type for testability — tests can drive a +// single sweep without waiting for the cron tick. +func (h *HousekeepingService) runSweep(ctx context.Context) { + threshold := h.staleThreshold() + cutoff := time.Now().Add(-threshold) + + // Sweep A: knowledge stuck in "processing". + resKnowledge := h.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("parse_status = ? AND updated_at < ?", types.ParseStatusProcessing, cutoff). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping", + }) + if resKnowledge.Error != nil { + logger.Warnf(ctx, "[Housekeeping] knowledge sweep failed: %v", resKnowledge.Error) + } else if resKnowledge.RowsAffected > 0 { + logger.Infof(ctx, "[Housekeeping] recovered %d stuck knowledge rows (threshold=%s)", + resKnowledge.RowsAffected, threshold) + } + + // Sweep B: knowledge summary stuck. Summary is post-parse; threshold + // is shorter because summary tasks are bounded by a single LLM call. + summaryCutoff := time.Now().Add(-1 * time.Hour) + resSummary := h.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("summary_status = ? AND updated_at < ?", types.SummaryStatusProcessing, summaryCutoff). + Update("summary_status", types.SummaryStatusFailed) + if resSummary.Error != nil { + logger.Warnf(ctx, "[Housekeeping] summary sweep failed: %v", resSummary.Error) + } else if resSummary.RowsAffected > 0 { + logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected) + } +} + +// staleThreshold returns how long a "processing" row may sit untouched +// before housekeeping treats it as orphaned. The floor is 1 hour so that a +// genuinely slow large-PDF parse cannot be killed mid-flight; the ceiling +// scales with the operator-configured DocumentProcessTimeout plus 10 minute +// buffer to absorb scheduling jitter. +func (h *HousekeepingService) staleThreshold() time.Duration { + base := 1 * time.Hour + if h.cfg != nil && h.cfg.KnowledgeBase != nil && h.cfg.KnowledgeBase.DocumentProcessTimeout > base { + base = h.cfg.KnowledgeBase.DocumentProcessTimeout + } + return base + 10*time.Minute +} + +func housekeepingEnabled() bool { + // Default-on: missing/empty env enables the sweep. Operators must + // explicitly set "false" to opt out, matching the plan's commitment + // that no env change is required for the safety net to engage. + v := strings.TrimSpace(os.Getenv("WEKNORA_HOUSEKEEPING_ENABLED")) + if v == "" { + return true + } + switch strings.ToLower(v) { + case "0", "false", "off", "no": + return false + } + return true +} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index afe46eb6..f78464ce 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -2340,7 +2340,7 @@ func (s *knowledgeService) convert( req.FileType = fileType } - result, err := reader.Read(ctx, req) + result, err := s.callDocReaderWithTimeout(ctx, reader, req) if err != nil { return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err) } @@ -2356,7 +2356,42 @@ func (s *knowledgeService) convert( return result, nil } -// resolveDocReader returns the appropriate DocReader for the given engine. +// callDocReaderWithTimeout wraps the DocReader RPC in a child context whose +// deadline is min(parent_deadline, DocReaderCallTimeout). Without this cap, +// a hung docreader (network partition, GC pause, OCR runaway) silently +// burns the whole DocumentProcessTimeout budget and pins a worker for hours +// — the #1 cause of "knowledge stuck in processing" reports. +// +// On timeout we annotate the error so retries / dead-letter consumers can +// distinguish "docreader was slow" from "docreader returned an error". +func (s *knowledgeService) callDocReaderWithTimeout( + ctx context.Context, reader interfaces.DocReader, req *types.ReadRequest, +) (*types.ReadResult, error) { + timeout := 30 * time.Minute + if s.config != nil && s.config.KnowledgeBase != nil && s.config.KnowledgeBase.DocReaderCallTimeout > 0 { + timeout = s.config.KnowledgeBase.DocReaderCallTimeout + } + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + start := time.Now() + result, err := reader.Read(callCtx, req) + elapsed := time.Since(start) + if err != nil { + // Promote DeadlineExceeded into a clearer message; retain underlying + // error via %w so errors.Is(callCtx.Err(), context.DeadlineExceeded) + // still works for upstream classification. + if errors.Is(callCtx.Err(), context.DeadlineExceeded) && !errors.Is(ctx.Err(), context.DeadlineExceeded) { + logger.Errorf(ctx, "[convert] docreader call timed out after %s (limit %s) for %q", + elapsed, timeout, req.FileName) + return nil, fmt.Errorf("docreader call timeout after %s: %w", timeout, err) + } + return nil, err + } + logger.Infof(ctx, "[convert] docreader call ok in %s for %q", elapsed, req.FileName) + return result, nil +} + // Returns nil when the required service is unavailable. func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { switch engine { diff --git a/internal/config/config.go b/internal/config/config.go index 4640a01a..b526c930 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -168,7 +168,13 @@ type KnowledgeBaseConfig struct { SplitMarkers []string `yaml:"split_markers" json:"split_markers"` KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"` ImageProcessing *ImageProcessingConfig `yaml:"image_processing" json:"image_processing"` - DocumentProcessTimeout time.Duration `yaml:"document_process_timeout" json:"document_process_timeout"` + DocumentProcessTimeout time.Duration `yaml:"document_process_timeout" json:"document_process_timeout"` + // DocReaderCallTimeout caps a single DocReader RPC. Without this the + // gRPC call inherits the asynq task context (whole DocumentProcessTimeout, + // default 2h+), so a hung docreader would block a worker for hours and + // leave knowledge in "processing". Default 30 minutes is generous enough + // for OCR-heavy large PDFs while ensuring forward progress. + DocReaderCallTimeout time.Duration `yaml:"docreader_call_timeout" json:"docreader_call_timeout"` } // ImageProcessingConfig 图像处理配置 @@ -716,6 +722,14 @@ func applyKnowledgeBaseEnvOverrides(cfg *Config) { cfg.KnowledgeBase.DocumentProcessTimeout = d } } + if cfg.KnowledgeBase.DocReaderCallTimeout <= 0 { + cfg.KnowledgeBase.DocReaderCallTimeout = 30 * time.Minute + } + if value := strings.TrimSpace(os.Getenv("WEKNORA_DOCREADER_CALL_TIMEOUT")); value != "" { + if d, err := time.ParseDuration(value); err == nil && d > 0 { + cfg.KnowledgeBase.DocReaderCallTimeout = d + } + } } func applyAgentEnvOverrides(cfg *Config) { diff --git a/internal/container/container.go b/internal/container/container.go index 8a164e0b..8559262c 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -271,6 +271,9 @@ func BuildContainer(container *dig.Container) *dig.Container { logger.Debugf(ctx, "[Container] Data source sync framework registered") must(container.Invoke(startAuditLogRetention)) logger.Debugf(ctx, "[Container] Audit log retention runner registered") + must(container.Provide(service.NewHousekeepingService)) + must(container.Invoke(startHousekeepingService)) + logger.Debugf(ctx, "[Container] Knowledge housekeeping runner registered") must(container.Provide(chatpipeline.NewEventManager)) must(container.Invoke(chatpipeline.NewPluginSearch)) must(container.Invoke(chatpipeline.NewPluginRerank)) @@ -623,49 +626,74 @@ func syncSequences(db *gorm.DB) { } // resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing -// due to an unexpected application restart when using in-memory queues (Lite mode). +// due to an unexpected application restart. +// +// In Lite mode (no REDIS_ADDR) every queued task lived in process memory, so +// any "processing" row at startup is definitively orphaned and must be marked +// failed wholesale. In distributed mode (Asynq on Redis) the active queue +// survives restart, but the *currently executing* task on the dead instance +// is lost — Asynq won't reschedule it until at-least-once retry kicks in, +// which can take minutes or never (e.g. if the deadline has passed). To bound +// the worst case we mark only "long-stale" rows failed: anything that hasn't +// been touched for 30 minutes is well past any reasonable in-flight window. +// Newer rows are left alone so we don't race a peer instance that's mid-process. func resetPendingTasks(db *gorm.DB) { - if os.Getenv("REDIS_ADDR") != "" { - return // Distributed queue (Asynq) will handle its own retries + distributed := os.Getenv("REDIS_ADDR") != "" + + knowledgeQuery := db.Model(&types.Knowledge{}). + Where("parse_status IN ?", []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusDeleting}) + summaryQuery := db.Model(&types.Knowledge{}). + Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing}) + syncQuery := db.Model(&types.SyncLog{}). + Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}) + + if distributed { + // 30-minute stale threshold — comfortably longer than any individual + // stage's expected duration (DocReader 30m, embedding seconds, etc.) + // while short enough that operators don't wait hours after a crash. + staleCutoff := time.Now().Add(-30 * time.Minute) + knowledgeQuery = knowledgeQuery.Where("updated_at < ?", staleCutoff) + summaryQuery = summaryQuery.Where("updated_at < ?", staleCutoff) + syncQuery = syncQuery.Where("start_time < ?", staleCutoff) } // 1. Reset knowledge parsing tasks - result := db.Model(&types.Knowledge{}). - Where("parse_status IN ?", []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusDeleting}). - Updates(map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - "error_message": "Task interrupted due to application restart", - }) + result := knowledgeQuery.Updates(map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": "Task interrupted due to application restart", + }) if result.Error != nil { logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error) } else if result.RowsAffected > 0 { - logger.Infof(context.Background(), "Reset %d stuck knowledge parsing tasks to failed state", result.RowsAffected) + logger.Infof(context.Background(), + "Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)", + result.RowsAffected, distributed) } // 2. Reset knowledge summary tasks - resultSummary := db.Model(&types.Knowledge{}). - Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing}). - Updates(map[string]interface{}{ - "summary_status": types.SummaryStatusFailed, - }) + resultSummary := summaryQuery.Updates(map[string]interface{}{ + "summary_status": types.SummaryStatusFailed, + }) if resultSummary.Error != nil { logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error) } else if resultSummary.RowsAffected > 0 { - logger.Infof(context.Background(), "Reset %d stuck summary generation tasks to failed state", resultSummary.RowsAffected) + logger.Infof(context.Background(), + "Reset %d stuck summary generation tasks to failed state (distributed=%v)", + resultSummary.RowsAffected, distributed) } // 3. Reset data source sync tasks - resultSync := db.Model(&types.SyncLog{}). - Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}). - Updates(map[string]interface{}{ - "status": types.SyncLogStatusFailed, - "error_message": "Sync interrupted due to application restart", - "end_time": time.Now(), - }) + resultSync := syncQuery.Updates(map[string]interface{}{ + "status": types.SyncLogStatusFailed, + "error_message": "Sync interrupted due to application restart", + "end_time": time.Now(), + }) if resultSync.Error != nil { logger.Warnf(context.Background(), "Failed to reset pending data source sync tasks: %v", resultSync.Error) } else if resultSync.RowsAffected > 0 { - logger.Infof(context.Background(), "Reset %d stuck data source sync tasks to failed state", resultSync.RowsAffected) + logger.Infof(context.Background(), + "Reset %d stuck data source sync tasks to failed state (distributed=%v)", + resultSync.RowsAffected, distributed) } } @@ -1359,6 +1387,24 @@ func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interface }) } +// startHousekeepingService starts the knowledge housekeeping cron and registers +// cleanup. This is the safety net that recovers any knowledge stuck in +// "processing" past a configurable threshold (see HousekeepingService for +// rationale). Best-effort: a startup error is logged but does NOT abort the +// container — the rest of the system stays usable. +func startHousekeepingService(svc *service.HousekeepingService, cleaner interfaces.ResourceCleaner) { + if svc == nil { + return + } + if err := svc.Start(context.Background()); err != nil { + logger.Warnf(context.Background(), "[Container] housekeeping start failed: %v", err) + } + cleaner.RegisterWithName("KnowledgeHousekeeping", func() error { + svc.Stop() + return nil + }) +} + // startAuditLogRetention spins up the daily audit_logs purge sweep // and registers shutdown cleanup. Mirrors the data-source-scheduler // pattern: container init kicks the goroutine, ResourceCleaner stops diff --git a/internal/middleware/asynqdl/asynqdl.go b/internal/middleware/asynqdl/asynqdl.go index 20f63fc2..cfa75f2f 100644 --- a/internal/middleware/asynqdl/asynqdl.go +++ b/internal/middleware/asynqdl/asynqdl.go @@ -46,7 +46,22 @@ import ( // // This middleware should be installed once on the asynq mux, BEFORE // other middlewares that may transform errors. See router/task.go. -func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc { +// OnDeadLetter is an optional callback invoked AFTER the dead-letter row has +// been recorded. It exists so callers can flip user-visible state alongside +// the bookkeeping row — e.g. mark a Knowledge as failed when its document +// processing task is permanently archived. Without this hook the only signal +// to the UI is the timestamp on the knowledge row going stale, which is what +// produced "stuck in processing forever" reports. +// +// The callback receives the raw asynq Task and the final error. It runs with +// context.Background() (NOT the task ctx) so a cancelled task ctx during +// shutdown doesn't also drop the state update. Errors are logged and +// swallowed; the callback must NEVER alter the original task error. +type OnDeadLetter func(ctx context.Context, t *asynq.Task, taskErr error) + +// MiddlewareWithCallback is the extended form of Middleware. The callback may +// be nil, in which case behaviour matches Middleware exactly. +func MiddlewareWithCallback(repo interfaces.TaskDeadLetterRepository, cb OnDeadLetter) asynq.MiddlewareFunc { return func(next asynq.Handler) asynq.Handler { return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error { err := next.ProcessTask(ctx, t) @@ -56,39 +71,47 @@ func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc { if !isFinalAttempt(ctx) { return err } - if repo == nil { - return err - } - // Capture the actual attempt count from asynq so the - // dead-letter row reflects how many tries were made - // before the task was archived. On the final attempt - // asynq's counter satisfies retried == maxRetry, so the - // total attempt count is retried + 1 (initial try + - // retries). Both helpers can fail outside a worker ctx; - // in that case we record 0 so it's clear the count - // wasn't observable. attempts := 0 if retried, ok := asynq.GetRetryCount(ctx); ok { attempts = retried + 1 } - dl := buildDeadLetter(t, err, attempts) - if dl == nil { - return err + if repo != nil { + dl := buildDeadLetter(t, err, attempts) + if dl != nil { + if insertErr := repo.Insert(context.Background(), dl); insertErr != nil { + logger.Warnf(ctx, + "asynq dead-letter: failed to record %s task: %v (original task error: %v)", + t.Type(), insertErr, err, + ) + } + } } - // context.Background() so a cancelled task ctx (e.g. server - // shutdown signaling cancellation right as a task fails) - // doesn't also drop the dead-letter record. - if insertErr := repo.Insert(context.Background(), dl); insertErr != nil { - logger.Warnf(ctx, - "asynq dead-letter: failed to record %s task: %v (original task error: %v)", - t.Type(), insertErr, err, - ) + if cb != nil { + // Wrap callback so a panic in user code doesn't escape + // into asynq's worker goroutine. + safeInvokeCallback(ctx, t, err, cb) } return err }) } } +func safeInvokeCallback(ctx context.Context, t *asynq.Task, taskErr error, cb OnDeadLetter) { + defer func() { + if r := recover(); r != nil { + logger.Warnf(ctx, "asynq dead-letter callback panicked for %s: %v", t.Type(), r) + } + }() + cb(context.Background(), t, taskErr) +} + +// Middleware preserves the original signature for callers that don't need +// the dead-letter state-linkback hook. Equivalent to MiddlewareWithCallback +// with a nil callback. +func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc { + return MiddlewareWithCallback(repo, nil) +} + // isFinalAttempt reports whether the just-finished handler invocation // was the LAST one asynq will run before archiving the task. Asynq // increments retry_count *after* a successful retry, so on the final diff --git a/internal/router/task.go b/internal/router/task.go index 0a4fc606..72a3cb1d 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -1,13 +1,17 @@ package router import ( + "context" + "encoding/json" "errors" "log" "os" "strconv" + "strings" "time" "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/middleware/asynqdl" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" @@ -32,6 +36,24 @@ type AsynqTaskParams struct { DeadLetterRepo interfaces.TaskDeadLetterRepository } +// defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms +// floor was tight enough to cause spurious i/o timeout errors during bursty +// workloads (large batch uploads, multimodal counter DECRs under load), so we +// raise the default to 500ms while still allowing operators to tune via env. +const defaultRedisOpTimeoutMs = 500 + +// readRedisOpTimeoutMs reads WEKNORA_REDIS_OP_TIMEOUT_MS, falling back to +// defaultRedisOpTimeoutMs on missing/invalid input. Kept as a separate helper +// so both ReadTimeout and WriteTimeout share the same source of truth. +func readRedisOpTimeoutMs() int { + if v := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_OP_TIMEOUT_MS")); v != "" { + if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 { + return parsed + } + } + return defaultRedisOpTimeoutMs +} + func getAsynqRedisClientOpt() *asynq.RedisClientOpt { db := 0 if dbStr := os.Getenv("REDIS_DB"); dbStr != "" { @@ -39,12 +61,16 @@ func getAsynqRedisClientOpt() *asynq.RedisClientOpt { db = parsed } } + timeoutMs := readRedisOpTimeoutMs() opt := &asynq.RedisClientOpt{ - Addr: os.Getenv("REDIS_ADDR"), - Username: os.Getenv("REDIS_USERNAME"), - Password: os.Getenv("REDIS_PASSWORD"), - ReadTimeout: 100 * time.Millisecond, - WriteTimeout: 200 * time.Millisecond, + Addr: os.Getenv("REDIS_ADDR"), + Username: os.Getenv("REDIS_USERNAME"), + Password: os.Getenv("REDIS_PASSWORD"), + ReadTimeout: time.Duration(timeoutMs) * time.Millisecond, + // Writes are typically more sensitive to congestion than reads + // (RESP pipelining, BRPOPLPUSH on Asynq dequeue), so we keep + // WriteTimeout slightly larger to absorb head-of-line stalls. + WriteTimeout: time.Duration(timeoutMs*2) * time.Millisecond, DB: db, } return opt @@ -83,11 +109,34 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration { return asynq.DefaultRetryDelayFunc(n, e, t) } +// defaultAsynqConcurrency is the worker pool size used when +// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to +// runtime.NumCPU(), which under-provisions during batch document uploads: +// a single 4-core container can only process 4 documents in parallel even +// when 100 are queued, so the queue wait time eats into each task's +// DocumentProcessTimeout budget. 16 is a safer default for the I/O-bound +// nature of doc parsing (most time is spent in DocReader / embedding RPCs, +// not on local CPU). +const defaultAsynqConcurrency = 16 + +func readAsynqConcurrency() int { + if v := strings.TrimSpace(os.Getenv("WEKNORA_ASYNQ_CONCURRENCY")); v != "" { + if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 { + return parsed + } + } + return defaultAsynqConcurrency +} + func NewAsynqServer() *asynq.Server { opt := getAsynqRedisClientOpt() + concurrency := readAsynqConcurrency() + log.Printf("asynq server starting with concurrency=%d redis_op_timeout=%dms", + concurrency, readRedisOpTimeoutMs()) srv := asynq.NewServer( opt, asynq.Config{ + Concurrency: concurrency, Queues: map[string]int{ "critical": 6, // Highest priority queue "default": 3, // Default priority queue @@ -110,7 +159,14 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // failures by task type, scope, or tenant without scraping logs. // Best-effort: a DB failure is logged and swallowed; the original task // error always propagates upstream to asynq for retry/archival. - mux.Use(asynqdl.Middleware(params.DeadLetterRepo)) + // + // The callback flips Knowledge.parse_status to "failed" the moment a + // document-related task exhausts its retry budget. Without this hook, + // a permanently-failing task left its parent knowledge stranded in + // "processing" until housekeeping cron caught it minutes later — the + // UI signal users actually see. + knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService) + mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer)) // Install Langfuse middleware BEFORE handler registration so every task // type is automatically wrapped. When Langfuse is disabled the middleware @@ -174,3 +230,76 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { }() return mux } + +// deadLetterKnowledgePayload extracts only the field we need from any +// document-related asynq payload. Kept narrow so we don't accidentally +// depend on the full payload schema and survive future field churn. +type deadLetterKnowledgePayload struct { + KnowledgeID string `json:"knowledge_id,omitempty"` +} + +// taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose +// dead-letter event should flip the parent Knowledge to "failed". Only +// terminal task types are listed here: +// +// - TypeDocumentProcess: the entry point of the parsing pipeline. +// - TypeImageMultimodal: a single image hitting dead-letter would have +// been counted by isFinalAsynqAttempt (see image_multimodal.go), so +// the parent might still complete via remaining images. We DO NOT mark +// the parent failed for this case — finalize-on-last-attempt already +// ensures progress. +// - TypeKnowledgePostProcess: terminal stage; failure here strands the +// knowledge in "processing". +// - TypeManualProcess: same shape as DocumentProcess for re-indexing. +// +// Question/Summary generation are NOT included: they run after parse_status +// has already become "completed" and have their own status fields. +var taskTypesAffectingKnowledgeStatus = map[string]struct{}{ + types.TypeDocumentProcess: {}, + types.TypeKnowledgePostProcess: {}, + types.TypeManualProcess: {}, +} + +// newDeadLetterKnowledgeFailer returns the callback wired into the asynq +// dead-letter middleware. When a document-related task exhausts its retry +// budget, this callback marks the corresponding Knowledge row as failed so +// the UI surfaces the error instead of a perpetual spinner. +// +// All work is best-effort: missing payload, missing knowledge_id, or DB +// errors are logged and swallowed. The dead-letter record is the source of +// truth — this is purely a UX shortcut so users don't wait for the +// housekeeping cron's next sweep. +func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDeadLetter { + if ks == nil { + return nil + } + repo := ks.GetRepository() + if repo == nil { + return nil + } + return func(ctx context.Context, t *asynq.Task, taskErr error) { + if t == nil { + return + } + if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok { + return + } + var probe deadLetterKnowledgePayload + if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" { + return + } + errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error() + // 8KB is the same cap the dead-letter row uses for last_error. + if len(errMsg) > 8192 { + errMsg = errMsg[:8192] + } + if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err) + return + } + if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err) + } + logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type()) + } +}