Files
Enterprise-Knowledge-Base/internal/router/task.go
T
wizardchen 3ae3ea97c5 fix(knowledge): prevent documents from getting stuck in "processing"
Several failure modes left Knowledge.parse_status pinned at "processing"
forever, with no signal to users beyond a permanent spinner. This commit
addresses the root causes and adds a safety net.

- Asynq worker pool: explicit Concurrency (default 16, env-tunable via
  WEKNORA_ASYNQ_CONCURRENCY) so batch uploads don't queue behind a
  CPU-count-sized worker pool. Redis op timeouts raised to 500ms/1000ms
  (WEKNORA_REDIS_OP_TIMEOUT_MS) to absorb bursty multimodal counter ops.

- DocReader RPC: cap each call with WEKNORA_DOCREADER_CALL_TIMEOUT
  (default 30m). Without this, a hung docreader pinned a worker for the
  full DocumentProcessTimeout window.

- ImageMultimodal: finalize-on-last-attempt semantics. A permanently
  failing single image no longer strands the parent — the asynq retry
  is allowed to run, but on the final attempt we count the image
  regardless of outcome. Redis DECR errors fall back to enqueuing the
  post-process task instead of returning silently.

- Dead-letter callback: when DocumentProcess / KnowledgePostProcess /
  ManualProcess exhausts retries, immediately mark the corresponding
  Knowledge as failed with the last error. This surfaces the failure
  in the UI without waiting for the housekeeping sweep.

- HousekeepingService: 5-minute cron that flips knowledge rows stuck
  in "processing" past DocumentProcessTimeout + 10m to failed, plus
  summary rows stuck > 1h. Catches anything the other safety nets
  miss (worker SIGKILL mid-handler, etc.). Disable with
  WEKNORA_HOUSEKEEPING_ENABLED=false.

- Distributed startup recovery: previously the post-restart sweep was
  skipped whenever REDIS_ADDR was set, even though Asynq does not
  reschedule the task that was actively running on the dead instance.
  Now the sweep runs in distributed mode too, but only against rows
  older than 30 minutes to avoid racing peer instances.
2026-05-28 15:14:45 +08:00

306 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package router
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/middleware/asynqdl"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"go.uber.org/dig"
)
type AsynqTaskParams struct {
dig.In
Server *asynq.Server
KnowledgeService interfaces.KnowledgeService
KnowledgeBaseService interfaces.KnowledgeBaseService
TagService interfaces.KnowledgeTagService
DataSourceService interfaces.DataSourceService
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"`
WikiIngest interfaces.TaskHandler `name:"wikiIngest"`
DeadLetterRepo interfaces.TaskDeadLetterRepository
}
// defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms
// floor was tight enough to cause spurious i/o timeout errors during bursty
// workloads (large batch uploads, multimodal counter DECRs under load), so we
// raise the default to 500ms while still allowing operators to tune via env.
const defaultRedisOpTimeoutMs = 500
// readRedisOpTimeoutMs reads WEKNORA_REDIS_OP_TIMEOUT_MS, falling back to
// defaultRedisOpTimeoutMs on missing/invalid input. Kept as a separate helper
// so both ReadTimeout and WriteTimeout share the same source of truth.
func readRedisOpTimeoutMs() int {
if v := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_OP_TIMEOUT_MS")); v != "" {
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
return parsed
}
}
return defaultRedisOpTimeoutMs
}
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
db := 0
if dbStr := os.Getenv("REDIS_DB"); dbStr != "" {
if parsed, err := strconv.Atoi(dbStr); err == nil {
db = parsed
}
}
timeoutMs := readRedisOpTimeoutMs()
opt := &asynq.RedisClientOpt{
Addr: os.Getenv("REDIS_ADDR"),
Username: os.Getenv("REDIS_USERNAME"),
Password: os.Getenv("REDIS_PASSWORD"),
ReadTimeout: time.Duration(timeoutMs) * time.Millisecond,
// Writes are typically more sensitive to congestion than reads
// (RESP pipelining, BRPOPLPUSH on Asynq dequeue), so we keep
// WriteTimeout slightly larger to absorb head-of-line stalls.
WriteTimeout: time.Duration(timeoutMs*2) * time.Millisecond,
DB: db,
}
return opt
}
func NewAsyncqClient() (*asynq.Client, error) {
opt := getAsynqRedisClientOpt()
client := asynq.NewClient(opt)
err := client.Ping()
if err != nil {
return nil, err
}
return client, nil
}
// wikiIngestRetryDelay is a fixed, short backoff for wiki ingest lock
// conflicts. Must be slightly longer than the active-lock TTL's worst-case
// "just got set" window so the retry is highly likely to succeed without
// burning through retries; but short enough that users don't feel the stall.
const wikiIngestRetryDelay = 15 * time.Second
// asynqRetryDelayFunc customizes per-task retry backoff.
//
// Default asynq backoff is exponential (≈10s, 40s, 90s, 2.5m, ...), which
// is appropriate for transient errors like remote HTTP failures. But for
// wiki ingest lock conflicts (ErrWikiIngestConcurrent), exponential
// backoff is harmful: a freshly orphaned lock expires in ≤60s, so a 15s
// fixed retry virtually guarantees the next attempt succeeds. Without
// this override, a crash-restart cycle can leave a KB unable to make
// progress for 7–10 minutes while the orphan lock expires AND the retry
// schedule catches up.
func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
if errors.Is(e, service.ErrWikiIngestConcurrent) {
return wikiIngestRetryDelay
}
return asynq.DefaultRetryDelayFunc(n, e, t)
}
// defaultAsynqConcurrency is the worker pool size used when
// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to
// runtime.NumCPU(), which under-provisions during batch document uploads:
// a single 4-core container can only process 4 documents in parallel even
// when 100 are queued, so the queue wait time eats into each task's
// DocumentProcessTimeout budget. 16 is a safer default for the I/O-bound
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
// not on local CPU).
const defaultAsynqConcurrency = 16
func readAsynqConcurrency() int {
if v := strings.TrimSpace(os.Getenv("WEKNORA_ASYNQ_CONCURRENCY")); v != "" {
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
return parsed
}
}
return defaultAsynqConcurrency
}
func NewAsynqServer() *asynq.Server {
opt := getAsynqRedisClientOpt()
concurrency := readAsynqConcurrency()
log.Printf("asynq server starting with concurrency=%d redis_op_timeout=%dms",
concurrency, readRedisOpTimeoutMs())
srv := asynq.NewServer(
opt,
asynq.Config{
Concurrency: concurrency,
Queues: map[string]int{
"critical": 6, // Highest priority queue
"default": 3, // Default priority queue
"low": 1, // Lowest priority queue
},
RetryDelayFunc: asynqRetryDelayFunc,
},
)
return srv
}
func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Create a new mux and register all handlers
mux := asynq.NewServeMux()
// Install the dead-letter middleware FIRST so it sees the raw error
// returned by the handler, before any other middleware that might
// transform it. The middleware records one task_dead_letters row per
// task that exhausts its retry budget — operators can then SQL-query
// failures by task type, scope, or tenant without scraping logs.
// Best-effort: a DB failure is logged and swallowed; the original task
// error always propagates upstream to asynq for retry/archival.
//
// The callback flips Knowledge.parse_status to "failed" the moment a
// document-related task exhausts its retry budget. Without this hook,
// a permanently-failing task left its parent knowledge stranded in
// "processing" until housekeeping cron caught it minutes later — the
// UI signal users actually see.
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService)
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
// Install Langfuse middleware BEFORE handler registration so every task
// type is automatically wrapped. When Langfuse is disabled the middleware
// is a pass-through; when enabled it resumes the upstream HTTP trace (if
// the payload carries one) or opens a standalone trace, then wraps the
// handler execution in a SPAN so all child generations (embedding / VLM /
// chat / rerank / ASR) nest correctly in the Langfuse UI.
mux.Use(langfuse.AsynqMiddleware())
// Register extract handlers - router will dispatch to appropriate handler
mux.HandleFunc(types.TypeChunkExtract, params.ChunkExtractor.Handle)
mux.HandleFunc(types.TypeDataTableSummary, params.DataTableSummary.Handle)
// Register document processing handler
mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
// Register manual knowledge processing handler (cleanup + re-indexing)
mux.HandleFunc(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
// Register FAQ import handler (includes dry run mode)
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
// Register question generation handler
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
// Register summary generation handler
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
// Register KB clone handler
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
// Register knowledge move handler
mux.HandleFunc(types.TypeKnowledgeMove, params.KnowledgeService.ProcessKnowledgeMove)
// Register knowledge list delete handler
mux.HandleFunc(types.TypeKnowledgeListDelete, params.KnowledgeService.ProcessKnowledgeListDelete)
// Register index delete handler
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
// Register KB delete handler
mux.HandleFunc(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete)
// Register image multimodal handler
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
// Register knowledge post process handler
mux.HandleFunc(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
// Register data source sync handler
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
// Register wiki ingest handler
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
go func() {
// Start the server
if err := params.Server.Run(mux); err != nil {
log.Fatalf("could not run server: %v", err)
}
}()
return mux
}
// deadLetterKnowledgePayload extracts only the field we need from any
// document-related asynq payload. Kept narrow so we don't accidentally
// depend on the full payload schema and survive future field churn.
type deadLetterKnowledgePayload struct {
KnowledgeID string `json:"knowledge_id,omitempty"`
}
// taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose
// dead-letter event should flip the parent Knowledge to "failed". Only
// terminal task types are listed here:
//
// - TypeDocumentProcess: the entry point of the parsing pipeline.
// - TypeImageMultimodal: a single image hitting dead-letter would have
// been counted by isFinalAsynqAttempt (see image_multimodal.go), so
// the parent might still complete via remaining images. We DO NOT mark
// the parent failed for this case — finalize-on-last-attempt already
// ensures progress.
// - TypeKnowledgePostProcess: terminal stage; failure here strands the
// knowledge in "processing".
// - TypeManualProcess: same shape as DocumentProcess for re-indexing.
//
// Question/Summary generation are NOT included: they run after parse_status
// has already become "completed" and have their own status fields.
var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
types.TypeDocumentProcess: {},
types.TypeKnowledgePostProcess: {},
types.TypeManualProcess: {},
}
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
// dead-letter middleware. When a document-related task exhausts its retry
// budget, this callback marks the corresponding Knowledge row as failed so
// the UI surfaces the error instead of a perpetual spinner.
//
// All work is best-effort: missing payload, missing knowledge_id, or DB
// errors are logged and swallowed. The dead-letter record is the source of
// truth — this is purely a UX shortcut so users don't wait for the
// housekeeping cron's next sweep.
func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDeadLetter {
if ks == nil {
return nil
}
repo := ks.GetRepository()
if repo == nil {
return nil
}
return func(ctx context.Context, t *asynq.Task, taskErr error) {
if t == nil {
return
}
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
return
}
var probe deadLetterKnowledgePayload
if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" {
return
}
errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error()
// 8KB is the same cap the dead-letter row uses for last_error.
if len(errMsg) > 8192 {
errMsg = errMsg[:8192]
}
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err)
return
}
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err)
}
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
}
}