Files
Enterprise-Knowledge-Base/internal/router/task.go
T
wizardchen 70e2e85fa6 feat(wiki): dedicated worker pool with concurrent claim-based batching
Isolate the wiki generation pipeline from the document parse pipeline so
the two can no longer starve each other during concurrent uploads.

- Dedicated asynq worker pool for QueueWiki (NewWikiAsynqServer) with its
  own concurrency budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY / asynq.wiki_
  concurrency, default 16), split from the parse pool (NewParseAsynqServer).
  Both run the same mux over disjoint queue sets. Replaces the old
  weighted-lottery approach where wiki sat on the low queue (weight 1).
- Task queue: add ClaimBatch (SELECT ... FOR UPDATE SKIP LOCKED + claimed_at)
  and ReleaseByIDs so multiple concurrent consumers pull disjoint rows
  without an external per-KB lock; PeekBatch retained for Lite mode.
- New TypeWikiFinalize debounced KB-global finalize task (index rebuild /
  dead-link cleanup / cross-linking), registered on both Redis and Lite paths.
- Two-level wiki ingest concurrency: IngestMaxInflight caps concurrent
  batches per KB; Map/Reduce fan-out sized within a batch. New config knob
  + system setting + swagger docs.
2026-07-09 17:41:07 +08:00

428 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package router
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/middleware/asynqdl"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"go.uber.org/dig"
)
type AsynqTaskParams struct {
dig.In
// ParseServer drains every queue EXCEPT QueueWiki: the user-facing
// document parsing / post-processing / enrichment pipeline. WikiServer
// drains QueueWiki only. Two independent worker pools give hard capacity
// isolation between the upstream parse pipeline and the downstream wiki
// pipeline (see NewParseAsynqServer / NewWikiAsynqServer). The same mux is
// run on both; asynq routes by the queue each server subscribes to, and
// queue↔task-type is 1:1 by convention, so no task is processed twice.
ParseServer *asynq.Server `name:"parseAsynqServer"`
WikiServer *asynq.Server `name:"wikiAsynqServer"`
KnowledgeService interfaces.KnowledgeService
KnowledgeBaseService interfaces.KnowledgeBaseService
TagService interfaces.KnowledgeTagService
DataSourceService interfaces.DataSourceService
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"`
WikiIngest interfaces.TaskHandler `name:"wikiIngest"`
DeadLetterRepo interfaces.TaskDeadLetterRepository
SpanTracker service.SpanTracker
}
// defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms
// floor was tight enough to cause spurious i/o timeout errors during bursty
// workloads (large batch uploads, multimodal counter DECRs under load), so we
// raise the default to 500ms while still allowing operators to tune via env.
const defaultRedisOpTimeoutMs = 500
// readRedisOpTimeoutMs reads WEKNORA_REDIS_OP_TIMEOUT_MS, falling back to
// defaultRedisOpTimeoutMs on missing/invalid input. Kept as a separate helper
// so both ReadTimeout and WriteTimeout share the same source of truth.
func readRedisOpTimeoutMs() int {
if v := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_OP_TIMEOUT_MS")); v != "" {
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
return parsed
}
}
return defaultRedisOpTimeoutMs
}
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
db := 0
if dbStr := os.Getenv("REDIS_DB"); dbStr != "" {
if parsed, err := strconv.Atoi(dbStr); err == nil {
db = parsed
}
}
timeoutMs := readRedisOpTimeoutMs()
opt := &asynq.RedisClientOpt{
Addr: os.Getenv("REDIS_ADDR"),
Username: os.Getenv("REDIS_USERNAME"),
Password: os.Getenv("REDIS_PASSWORD"),
ReadTimeout: time.Duration(timeoutMs) * time.Millisecond,
// Writes are typically more sensitive to congestion than reads
// (RESP pipelining, BRPOPLPUSH on Asynq dequeue), so we keep
// WriteTimeout slightly larger to absorb head-of-line stalls.
WriteTimeout: time.Duration(timeoutMs*2) * time.Millisecond,
DB: db,
TLSConfig: common.RedisTLSConfig(),
}
return opt
}
func NewAsyncqClient() (*asynq.Client, error) {
opt := getAsynqRedisClientOpt()
client := asynq.NewClient(opt)
err := client.Ping()
if err != nil {
return nil, err
}
return client, nil
}
// wikiIngestRetryDelay is a fixed, short backoff for wiki ingest lock
// conflicts. Must be slightly longer than the active-lock TTL's worst-case
// "just got set" window so the retry is highly likely to succeed without
// burning through retries; but short enough that users don't feel the stall.
const wikiIngestRetryDelay = 15 * time.Second
// asynqRetryDelayFunc customizes per-task retry backoff.
//
// Default asynq backoff is exponential (≈10s, 40s, 90s, 2.5m, ...), which
// is appropriate for transient errors like remote HTTP failures. But for
// wiki ingest lock conflicts (ErrWikiIngestConcurrent), exponential
// backoff is harmful: a freshly orphaned lock expires in ≤60s, so a 15s
// fixed retry virtually guarantees the next attempt succeeds. Without
// this override, a crash-restart cycle can leave a KB unable to make
// progress for 7–10 minutes while the orphan lock expires AND the retry
// schedule catches up.
func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
if errors.Is(e, service.ErrWikiIngestConcurrent) {
return wikiIngestRetryDelay
}
return asynq.DefaultRetryDelayFunc(n, e, t)
}
// defaultAsynqConcurrency is the worker pool size used when
// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to
// runtime.NumCPU(), which under-provisions during batch document uploads:
// a single 4-core container can only process 4 documents in parallel even
// when 100 are queued, so the queue wait time eats into each task's
// DocumentProcessTimeout budget. 32 is a safer default for the I/O-bound
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
// not on local CPU).
const defaultAsynqConcurrency = 32
// defaultWikiAsynqConcurrency is the worker pool size for the dedicated wiki
// pool (QueueWiki) when WEKNORA_WIKI_ASYNQ_CONCURRENCY is unset. Kept
// independent from the parse pool so the two can be sized separately for the
// host / LLM-provider budget. The wiki pipeline is LLM-bound (synthesis model
// calls dominate), so a moderate default avoids overrunning upstream provider
// concurrency while still letting many documents' wiki generation run in
// parallel across knowledge bases.
const defaultWikiAsynqConcurrency = 16
// newAsynqServer builds an asynq server bound to a specific queue set and
// concurrency. Shared by the parse and wiki pools so both inherit the same
// Redis options and retry-delay policy.
func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server {
opt := getAsynqRedisClientOpt()
return asynq.NewServer(
opt,
asynq.Config{
Concurrency: concurrency,
Queues: queues,
RetryDelayFunc: asynqRetryDelayFunc,
},
)
}
// NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki.
// This is the user-facing document parse / post-process / enrichment pipeline.
// Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from
// the historical single-pool behaviour so existing deployments see the same
// parse throughput.
func NewParseAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
concurrency := defaultAsynqConcurrency
if svc != nil {
n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultAsynqConcurrency)
if n > 0 {
concurrency = int(n)
}
}
log.Printf("asynq parse-pool server starting with concurrency=%d redis_op_timeout=%dms",
concurrency, readRedisOpTimeoutMs())
return newAsynqServer(concurrency, map[string]int{
types.QueueCritical: 6, // Highest priority queue
types.QueueDefault: 3, // Default priority queue
types.QueueLow: 1, // Lowest priority queue
types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks
types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks
types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks
})
}
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. Runs the
// same mux as the parse pool but only ever pulls wiki tasks, so its concurrency
// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on
// wiki generation. This is the hard capacity isolation that prevents the parse
// pipeline from starving wiki (and vice-versa) during concurrent uploads.
func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
concurrency := defaultWikiAsynqConcurrency
if svc != nil {
n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiAsynqConcurrency)
if n > 0 {
concurrency = int(n)
}
}
log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency)
return newAsynqServer(concurrency, map[string]int{
types.QueueWiki: 1,
})
}
func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Create a new mux and register all handlers
mux := asynq.NewServeMux()
// Install the dead-letter middleware FIRST so it sees the raw error
// returned by the handler, before any other middleware that might
// transform it. The middleware records one task_dead_letters row per
// task that exhausts its retry budget — operators can then SQL-query
// failures by task type, scope, or tenant without scraping logs.
// Best-effort: a DB failure is logged and swallowed; the original task
// error always propagates upstream to asynq for retry/archival.
//
// The callback flips Knowledge.parse_status to "failed" the moment a
// document-related task exhausts its retry budget. Without this hook,
// a permanently-failing task left its parent knowledge stranded in
// "processing" until housekeeping cron caught it minutes later — the
// UI signal users actually see.
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker)
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
// Install Langfuse middleware BEFORE handler registration so every task
// type is automatically wrapped. When Langfuse is disabled the middleware
// is a pass-through; when enabled it resumes the upstream HTTP trace (if
// the payload carries one) or opens a standalone trace, then wraps the
// handler execution in a SPAN so all child generations (embedding / VLM /
// chat / rerank / ASR) nest correctly in the Langfuse UI.
mux.Use(langfuse.AsynqMiddleware())
// Register extract handlers - router will dispatch to appropriate handler
mux.HandleFunc(types.TypeChunkExtract, params.ChunkExtractor.Handle)
mux.HandleFunc(types.TypeDataTableSummary, params.DataTableSummary.Handle)
// Register document processing handler
mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
// Register manual knowledge processing handler (cleanup + re-indexing)
mux.HandleFunc(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
// Register FAQ import handler (includes dry run mode)
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
// Register question generation handler
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
// Register summary generation handler
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
// Register KB clone handler
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
// Register knowledge move handler
mux.HandleFunc(types.TypeKnowledgeMove, params.KnowledgeService.ProcessKnowledgeMove)
// Register knowledge list delete handler
mux.HandleFunc(types.TypeKnowledgeListDelete, params.KnowledgeService.ProcessKnowledgeListDelete)
// Register knowledge list reparse handler
mux.HandleFunc(types.TypeKnowledgeListReparse, params.KnowledgeService.ProcessKnowledgeListReparse)
// Register index delete handler
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
// Register KB delete handler
mux.HandleFunc(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete)
// Register image multimodal handler
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
// Register knowledge post process handler
mux.HandleFunc(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
// Register data source sync handler
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
// Register wiki ingest handler + the debounced KB-global finalize handler.
// Both route to the same dispatch (WikiIngest.Handle switches on task type)
// and both land on QueueWiki, so the dedicated wiki pool serves them.
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle)
// Run the same mux on both pools. Each server subscribes to a disjoint set
// of queues (parse pool = everything except QueueWiki; wiki pool =
// QueueWiki only), so a given task is dequeued by exactly one pool. This is
// how the two pools get independent concurrency budgets while sharing one
// handler registration.
runPool := func(name string, srv *asynq.Server) {
go func() {
if err := srv.Run(mux); err != nil {
log.Fatalf("could not run %s asynq server: %v", name, err)
}
}()
}
runPool("parse-pool", params.ParseServer)
runPool("wiki-pool", params.WikiServer)
return mux
}
// deadLetterKnowledgePayload extracts only the field we need from any
// document-related asynq payload. Kept narrow so we don't accidentally
// depend on the full payload schema and survive future field churn.
type deadLetterKnowledgePayload struct {
KnowledgeID string `json:"knowledge_id,omitempty"`
// Attempt threads through DocumentProcess / ManualProcess /
// KnowledgePostProcess payloads (added when span tracking shipped)
// — extracted here so the dead-letter callback can also close the
// matching root span as failed. Older in-flight payloads without
// this field decode as 0 and the tracker call no-ops.
Attempt int `json:"attempt,omitempty"`
}
// taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose
// dead-letter event should flip the parent Knowledge to "failed". Only
// terminal task types are listed here:
//
// - TypeDocumentProcess: the entry point of the parsing pipeline.
// - TypeImageMultimodal: a single image hitting dead-letter would have
// been counted by isFinalAsynqAttempt (see image_multimodal.go), so
// the parent might still complete via remaining images. We DO NOT mark
// the parent failed for this case — finalize-on-last-attempt already
// ensures progress.
// - TypeKnowledgePostProcess: terminal stage; failure here strands the
// knowledge in "processing".
// - TypeManualProcess: same shape as DocumentProcess for re-indexing.
//
// Question/Summary generation are NOT included: they run after parse_status
// has already become "completed" and have their own status fields.
var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
types.TypeDocumentProcess: {},
types.TypeKnowledgePostProcess: {},
types.TypeManualProcess: {},
}
type deadLetterKnowledgeListDeletePayload struct {
KnowledgeIDs []string `json:"knowledge_ids,omitempty"`
}
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
// dead-letter middleware. When a document-related task exhausts its retry
// budget, this callback marks the corresponding Knowledge row as failed so
// the UI surfaces the error instead of a perpetual spinner.
//
// All work is best-effort: missing payload, missing knowledge_id, or DB
// errors are logged and swallowed. The dead-letter record is the source of
// truth — this is purely a UX shortcut so users don't wait for the
// housekeeping cron's next sweep.
func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker service.SpanTracker) asynqdl.OnDeadLetter {
if ks == nil {
return nil
}
repo := ks.GetRepository()
if repo == nil {
return nil
}
return func(ctx context.Context, t *asynq.Task, taskErr error) {
if t == nil {
return
}
if t.Type() == types.TypeKnowledgeListDelete {
markKnowledgeListDeleteFailed(ctx, repo, t, taskErr)
return
}
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
return
}
var probe deadLetterKnowledgePayload
if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" {
return
}
errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error()
// 8KB is the same cap the dead-letter row uses for last_error.
if len(errMsg) > 8192 {
errMsg = errMsg[:8192]
}
// Single UPDATE so we never end up with parse_status=failed but
// stale error_message (or vice versa) when the second write
// fails.
if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": errMsg,
}); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err)
return
}
// Close the matching root span so the timeline stops showing
// "进行中" after dead-letter exhaustion. Best-effort: nil
// tracker / missing attempt / missing root all no-op cleanly.
if tracker != nil && probe.Attempt > 0 {
tracker.FinalizeAttempt(ctx, probe.KnowledgeID, probe.Attempt,
types.SpanStatusFailed, nil, "TASK_TIMEOUT", errMsg)
}
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
}
}
func markKnowledgeListDeleteFailed(
ctx context.Context,
repo interfaces.KnowledgeRepository,
t *asynq.Task,
taskErr error,
) {
var payload deadLetterKnowledgeListDeletePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil || len(payload.KnowledgeIDs) == 0 {
return
}
errMsg := "delete task exhausted retries: " + taskErr.Error()
if len(errMsg) > 8192 {
errMsg = errMsg[:8192]
}
for _, knowledgeID := range payload.KnowledgeIDs {
if knowledgeID == "" {
continue
}
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, knowledgeID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": errMsg,
})
if err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to mark delete failure for knowledge %s: %v", knowledgeID, err)
continue
}
if !updated {
logger.Infof(ctx, "dead-letter callback: skipped marking knowledge %s after delete task exhaustion because it is no longer active deleting", knowledgeID)
continue
}
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed after delete task exhausted retries", knowledgeID)
}
}