70e2e85fa6
Isolate the wiki generation pipeline from the document parse pipeline so the two can no longer starve each other during concurrent uploads. - Dedicated asynq worker pool for QueueWiki (NewWikiAsynqServer) with its own concurrency budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY / asynq.wiki_ concurrency, default 16), split from the parse pool (NewParseAsynqServer). Both run the same mux over disjoint queue sets. Replaces the old weighted-lottery approach where wiki sat on the low queue (weight 1). - Task queue: add ClaimBatch (SELECT ... FOR UPDATE SKIP LOCKED + claimed_at) and ReleaseByIDs so multiple concurrent consumers pull disjoint rows without an external per-KB lock; PeekBatch retained for Lite mode. - New TypeWikiFinalize debounced KB-global finalize task (index rebuild / dead-link cleanup / cross-linking), registered on both Redis and Lite paths. - Two-level wiki ingest concurrency: IngestMaxInflight caps concurrent batches per KB; Map/Reduce fan-out sized within a batch. New config knob + system setting + swagger docs.
428 lines
18 KiB
Go
428 lines
18 KiB
Go
package router
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"log"
|
||
"os"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/Tencent/WeKnora/internal/application/service"
|
||
"github.com/Tencent/WeKnora/internal/common"
|
||
"github.com/Tencent/WeKnora/internal/logger"
|
||
"github.com/Tencent/WeKnora/internal/middleware/asynqdl"
|
||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||
"github.com/Tencent/WeKnora/internal/types"
|
||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||
"github.com/hibiken/asynq"
|
||
"go.uber.org/dig"
|
||
)
|
||
|
||
type AsynqTaskParams struct {
|
||
dig.In
|
||
|
||
// ParseServer drains every queue EXCEPT QueueWiki: the user-facing
|
||
// document parsing / post-processing / enrichment pipeline. WikiServer
|
||
// drains QueueWiki only. Two independent worker pools give hard capacity
|
||
// isolation between the upstream parse pipeline and the downstream wiki
|
||
// pipeline (see NewParseAsynqServer / NewWikiAsynqServer). The same mux is
|
||
// run on both; asynq routes by the queue each server subscribes to, and
|
||
// queue↔task-type is 1:1 by convention, so no task is processed twice.
|
||
ParseServer *asynq.Server `name:"parseAsynqServer"`
|
||
WikiServer *asynq.Server `name:"wikiAsynqServer"`
|
||
KnowledgeService interfaces.KnowledgeService
|
||
KnowledgeBaseService interfaces.KnowledgeBaseService
|
||
TagService interfaces.KnowledgeTagService
|
||
DataSourceService interfaces.DataSourceService
|
||
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
|
||
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
|
||
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
|
||
KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"`
|
||
WikiIngest interfaces.TaskHandler `name:"wikiIngest"`
|
||
DeadLetterRepo interfaces.TaskDeadLetterRepository
|
||
SpanTracker service.SpanTracker
|
||
}
|
||
|
||
// defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms
|
||
// floor was tight enough to cause spurious i/o timeout errors during bursty
|
||
// workloads (large batch uploads, multimodal counter DECRs under load), so we
|
||
// raise the default to 500ms while still allowing operators to tune via env.
|
||
const defaultRedisOpTimeoutMs = 500
|
||
|
||
// readRedisOpTimeoutMs reads WEKNORA_REDIS_OP_TIMEOUT_MS, falling back to
|
||
// defaultRedisOpTimeoutMs on missing/invalid input. Kept as a separate helper
|
||
// so both ReadTimeout and WriteTimeout share the same source of truth.
|
||
func readRedisOpTimeoutMs() int {
|
||
if v := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_OP_TIMEOUT_MS")); v != "" {
|
||
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
|
||
return parsed
|
||
}
|
||
}
|
||
return defaultRedisOpTimeoutMs
|
||
}
|
||
|
||
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||
db := 0
|
||
if dbStr := os.Getenv("REDIS_DB"); dbStr != "" {
|
||
if parsed, err := strconv.Atoi(dbStr); err == nil {
|
||
db = parsed
|
||
}
|
||
}
|
||
timeoutMs := readRedisOpTimeoutMs()
|
||
opt := &asynq.RedisClientOpt{
|
||
Addr: os.Getenv("REDIS_ADDR"),
|
||
Username: os.Getenv("REDIS_USERNAME"),
|
||
Password: os.Getenv("REDIS_PASSWORD"),
|
||
ReadTimeout: time.Duration(timeoutMs) * time.Millisecond,
|
||
// Writes are typically more sensitive to congestion than reads
|
||
// (RESP pipelining, BRPOPLPUSH on Asynq dequeue), so we keep
|
||
// WriteTimeout slightly larger to absorb head-of-line stalls.
|
||
WriteTimeout: time.Duration(timeoutMs*2) * time.Millisecond,
|
||
DB: db,
|
||
TLSConfig: common.RedisTLSConfig(),
|
||
}
|
||
return opt
|
||
}
|
||
|
||
func NewAsyncqClient() (*asynq.Client, error) {
|
||
opt := getAsynqRedisClientOpt()
|
||
client := asynq.NewClient(opt)
|
||
err := client.Ping()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return client, nil
|
||
}
|
||
|
||
// wikiIngestRetryDelay is a fixed, short backoff for wiki ingest lock
|
||
// conflicts. Must be slightly longer than the active-lock TTL's worst-case
|
||
// "just got set" window so the retry is highly likely to succeed without
|
||
// burning through retries; but short enough that users don't feel the stall.
|
||
const wikiIngestRetryDelay = 15 * time.Second
|
||
|
||
// asynqRetryDelayFunc customizes per-task retry backoff.
|
||
//
|
||
// Default asynq backoff is exponential (≈10s, 40s, 90s, 2.5m, ...), which
|
||
// is appropriate for transient errors like remote HTTP failures. But for
|
||
// wiki ingest lock conflicts (ErrWikiIngestConcurrent), exponential
|
||
// backoff is harmful: a freshly orphaned lock expires in ≤60s, so a 15s
|
||
// fixed retry virtually guarantees the next attempt succeeds. Without
|
||
// this override, a crash-restart cycle can leave a KB unable to make
|
||
// progress for 7–10 minutes while the orphan lock expires AND the retry
|
||
// schedule catches up.
|
||
func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
|
||
if errors.Is(e, service.ErrWikiIngestConcurrent) {
|
||
return wikiIngestRetryDelay
|
||
}
|
||
return asynq.DefaultRetryDelayFunc(n, e, t)
|
||
}
|
||
|
||
// defaultAsynqConcurrency is the worker pool size used when
|
||
// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to
|
||
// runtime.NumCPU(), which under-provisions during batch document uploads:
|
||
// a single 4-core container can only process 4 documents in parallel even
|
||
// when 100 are queued, so the queue wait time eats into each task's
|
||
// DocumentProcessTimeout budget. 32 is a safer default for the I/O-bound
|
||
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
|
||
// not on local CPU).
|
||
const defaultAsynqConcurrency = 32
|
||
|
||
// defaultWikiAsynqConcurrency is the worker pool size for the dedicated wiki
|
||
// pool (QueueWiki) when WEKNORA_WIKI_ASYNQ_CONCURRENCY is unset. Kept
|
||
// independent from the parse pool so the two can be sized separately for the
|
||
// host / LLM-provider budget. The wiki pipeline is LLM-bound (synthesis model
|
||
// calls dominate), so a moderate default avoids overrunning upstream provider
|
||
// concurrency while still letting many documents' wiki generation run in
|
||
// parallel across knowledge bases.
|
||
const defaultWikiAsynqConcurrency = 16
|
||
|
||
// newAsynqServer builds an asynq server bound to a specific queue set and
|
||
// concurrency. Shared by the parse and wiki pools so both inherit the same
|
||
// Redis options and retry-delay policy.
|
||
func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server {
|
||
opt := getAsynqRedisClientOpt()
|
||
return asynq.NewServer(
|
||
opt,
|
||
asynq.Config{
|
||
Concurrency: concurrency,
|
||
Queues: queues,
|
||
RetryDelayFunc: asynqRetryDelayFunc,
|
||
},
|
||
)
|
||
}
|
||
|
||
// NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki.
|
||
// This is the user-facing document parse / post-process / enrichment pipeline.
|
||
// Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from
|
||
// the historical single-pool behaviour so existing deployments see the same
|
||
// parse throughput.
|
||
func NewParseAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||
concurrency := defaultAsynqConcurrency
|
||
if svc != nil {
|
||
n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultAsynqConcurrency)
|
||
if n > 0 {
|
||
concurrency = int(n)
|
||
}
|
||
}
|
||
log.Printf("asynq parse-pool server starting with concurrency=%d redis_op_timeout=%dms",
|
||
concurrency, readRedisOpTimeoutMs())
|
||
return newAsynqServer(concurrency, map[string]int{
|
||
types.QueueCritical: 6, // Highest priority queue
|
||
types.QueueDefault: 3, // Default priority queue
|
||
types.QueueLow: 1, // Lowest priority queue
|
||
types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks
|
||
types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks
|
||
types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks
|
||
})
|
||
}
|
||
|
||
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. Runs the
|
||
// same mux as the parse pool but only ever pulls wiki tasks, so its concurrency
|
||
// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on
|
||
// wiki generation. This is the hard capacity isolation that prevents the parse
|
||
// pipeline from starving wiki (and vice-versa) during concurrent uploads.
|
||
func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||
concurrency := defaultWikiAsynqConcurrency
|
||
if svc != nil {
|
||
n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiAsynqConcurrency)
|
||
if n > 0 {
|
||
concurrency = int(n)
|
||
}
|
||
}
|
||
log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency)
|
||
return newAsynqServer(concurrency, map[string]int{
|
||
types.QueueWiki: 1,
|
||
})
|
||
}
|
||
|
||
func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||
// Create a new mux and register all handlers
|
||
mux := asynq.NewServeMux()
|
||
|
||
// Install the dead-letter middleware FIRST so it sees the raw error
|
||
// returned by the handler, before any other middleware that might
|
||
// transform it. The middleware records one task_dead_letters row per
|
||
// task that exhausts its retry budget — operators can then SQL-query
|
||
// failures by task type, scope, or tenant without scraping logs.
|
||
// Best-effort: a DB failure is logged and swallowed; the original task
|
||
// error always propagates upstream to asynq for retry/archival.
|
||
//
|
||
// The callback flips Knowledge.parse_status to "failed" the moment a
|
||
// document-related task exhausts its retry budget. Without this hook,
|
||
// a permanently-failing task left its parent knowledge stranded in
|
||
// "processing" until housekeeping cron caught it minutes later — the
|
||
// UI signal users actually see.
|
||
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker)
|
||
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
|
||
|
||
// Install Langfuse middleware BEFORE handler registration so every task
|
||
// type is automatically wrapped. When Langfuse is disabled the middleware
|
||
// is a pass-through; when enabled it resumes the upstream HTTP trace (if
|
||
// the payload carries one) or opens a standalone trace, then wraps the
|
||
// handler execution in a SPAN so all child generations (embedding / VLM /
|
||
// chat / rerank / ASR) nest correctly in the Langfuse UI.
|
||
mux.Use(langfuse.AsynqMiddleware())
|
||
|
||
// Register extract handlers - router will dispatch to appropriate handler
|
||
mux.HandleFunc(types.TypeChunkExtract, params.ChunkExtractor.Handle)
|
||
mux.HandleFunc(types.TypeDataTableSummary, params.DataTableSummary.Handle)
|
||
|
||
// Register document processing handler
|
||
mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
|
||
|
||
// Register manual knowledge processing handler (cleanup + re-indexing)
|
||
mux.HandleFunc(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
|
||
|
||
// Register FAQ import handler (includes dry run mode)
|
||
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
|
||
|
||
// Register question generation handler
|
||
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
|
||
|
||
// Register summary generation handler
|
||
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
|
||
|
||
// Register KB clone handler
|
||
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
|
||
|
||
// Register knowledge move handler
|
||
mux.HandleFunc(types.TypeKnowledgeMove, params.KnowledgeService.ProcessKnowledgeMove)
|
||
|
||
// Register knowledge list delete handler
|
||
mux.HandleFunc(types.TypeKnowledgeListDelete, params.KnowledgeService.ProcessKnowledgeListDelete)
|
||
|
||
// Register knowledge list reparse handler
|
||
mux.HandleFunc(types.TypeKnowledgeListReparse, params.KnowledgeService.ProcessKnowledgeListReparse)
|
||
|
||
// Register index delete handler
|
||
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
|
||
|
||
// Register KB delete handler
|
||
mux.HandleFunc(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete)
|
||
|
||
// Register image multimodal handler
|
||
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
|
||
|
||
// Register knowledge post process handler
|
||
mux.HandleFunc(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
|
||
|
||
// Register data source sync handler
|
||
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
|
||
|
||
// Register wiki ingest handler + the debounced KB-global finalize handler.
|
||
// Both route to the same dispatch (WikiIngest.Handle switches on task type)
|
||
// and both land on QueueWiki, so the dedicated wiki pool serves them.
|
||
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
|
||
mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle)
|
||
|
||
// Run the same mux on both pools. Each server subscribes to a disjoint set
|
||
// of queues (parse pool = everything except QueueWiki; wiki pool =
|
||
// QueueWiki only), so a given task is dequeued by exactly one pool. This is
|
||
// how the two pools get independent concurrency budgets while sharing one
|
||
// handler registration.
|
||
runPool := func(name string, srv *asynq.Server) {
|
||
go func() {
|
||
if err := srv.Run(mux); err != nil {
|
||
log.Fatalf("could not run %s asynq server: %v", name, err)
|
||
}
|
||
}()
|
||
}
|
||
runPool("parse-pool", params.ParseServer)
|
||
runPool("wiki-pool", params.WikiServer)
|
||
return mux
|
||
}
|
||
|
||
// deadLetterKnowledgePayload extracts only the field we need from any
|
||
// document-related asynq payload. Kept narrow so we don't accidentally
|
||
// depend on the full payload schema and survive future field churn.
|
||
type deadLetterKnowledgePayload struct {
|
||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||
// Attempt threads through DocumentProcess / ManualProcess /
|
||
// KnowledgePostProcess payloads (added when span tracking shipped)
|
||
// — extracted here so the dead-letter callback can also close the
|
||
// matching root span as failed. Older in-flight payloads without
|
||
// this field decode as 0 and the tracker call no-ops.
|
||
Attempt int `json:"attempt,omitempty"`
|
||
}
|
||
|
||
// taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose
|
||
// dead-letter event should flip the parent Knowledge to "failed". Only
|
||
// terminal task types are listed here:
|
||
//
|
||
// - TypeDocumentProcess: the entry point of the parsing pipeline.
|
||
// - TypeImageMultimodal: a single image hitting dead-letter would have
|
||
// been counted by isFinalAsynqAttempt (see image_multimodal.go), so
|
||
// the parent might still complete via remaining images. We DO NOT mark
|
||
// the parent failed for this case — finalize-on-last-attempt already
|
||
// ensures progress.
|
||
// - TypeKnowledgePostProcess: terminal stage; failure here strands the
|
||
// knowledge in "processing".
|
||
// - TypeManualProcess: same shape as DocumentProcess for re-indexing.
|
||
//
|
||
// Question/Summary generation are NOT included: they run after parse_status
|
||
// has already become "completed" and have their own status fields.
|
||
var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
|
||
types.TypeDocumentProcess: {},
|
||
types.TypeKnowledgePostProcess: {},
|
||
types.TypeManualProcess: {},
|
||
}
|
||
|
||
type deadLetterKnowledgeListDeletePayload struct {
|
||
KnowledgeIDs []string `json:"knowledge_ids,omitempty"`
|
||
}
|
||
|
||
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
|
||
// dead-letter middleware. When a document-related task exhausts its retry
|
||
// budget, this callback marks the corresponding Knowledge row as failed so
|
||
// the UI surfaces the error instead of a perpetual spinner.
|
||
//
|
||
// All work is best-effort: missing payload, missing knowledge_id, or DB
|
||
// errors are logged and swallowed. The dead-letter record is the source of
|
||
// truth — this is purely a UX shortcut so users don't wait for the
|
||
// housekeeping cron's next sweep.
|
||
func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker service.SpanTracker) asynqdl.OnDeadLetter {
|
||
if ks == nil {
|
||
return nil
|
||
}
|
||
repo := ks.GetRepository()
|
||
if repo == nil {
|
||
return nil
|
||
}
|
||
return func(ctx context.Context, t *asynq.Task, taskErr error) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
if t.Type() == types.TypeKnowledgeListDelete {
|
||
markKnowledgeListDeleteFailed(ctx, repo, t, taskErr)
|
||
return
|
||
}
|
||
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
|
||
return
|
||
}
|
||
var probe deadLetterKnowledgePayload
|
||
if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" {
|
||
return
|
||
}
|
||
errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error()
|
||
// 8KB is the same cap the dead-letter row uses for last_error.
|
||
if len(errMsg) > 8192 {
|
||
errMsg = errMsg[:8192]
|
||
}
|
||
// Single UPDATE so we never end up with parse_status=failed but
|
||
// stale error_message (or vice versa) when the second write
|
||
// fails.
|
||
if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{
|
||
"parse_status": types.ParseStatusFailed,
|
||
"error_message": errMsg,
|
||
}); err != nil {
|
||
logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err)
|
||
return
|
||
}
|
||
// Close the matching root span so the timeline stops showing
|
||
// "进行中" after dead-letter exhaustion. Best-effort: nil
|
||
// tracker / missing attempt / missing root all no-op cleanly.
|
||
if tracker != nil && probe.Attempt > 0 {
|
||
tracker.FinalizeAttempt(ctx, probe.KnowledgeID, probe.Attempt,
|
||
types.SpanStatusFailed, nil, "TASK_TIMEOUT", errMsg)
|
||
}
|
||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
|
||
}
|
||
}
|
||
|
||
func markKnowledgeListDeleteFailed(
|
||
ctx context.Context,
|
||
repo interfaces.KnowledgeRepository,
|
||
t *asynq.Task,
|
||
taskErr error,
|
||
) {
|
||
var payload deadLetterKnowledgeListDeletePayload
|
||
if err := json.Unmarshal(t.Payload(), &payload); err != nil || len(payload.KnowledgeIDs) == 0 {
|
||
return
|
||
}
|
||
errMsg := "delete task exhausted retries: " + taskErr.Error()
|
||
if len(errMsg) > 8192 {
|
||
errMsg = errMsg[:8192]
|
||
}
|
||
for _, knowledgeID := range payload.KnowledgeIDs {
|
||
if knowledgeID == "" {
|
||
continue
|
||
}
|
||
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, knowledgeID, map[string]interface{}{
|
||
"parse_status": types.ParseStatusFailed,
|
||
"error_message": errMsg,
|
||
})
|
||
if err != nil {
|
||
logger.Warnf(ctx, "dead-letter callback: failed to mark delete failure for knowledge %s: %v", knowledgeID, err)
|
||
continue
|
||
}
|
||
if !updated {
|
||
logger.Infof(ctx, "dead-letter callback: skipped marking knowledge %s after delete task exhaustion because it is no longer active deleting", knowledgeID)
|
||
continue
|
||
}
|
||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed after delete task exhausted retries", knowledgeID)
|
||
}
|
||
}
|