From 035093dc35a0d07e43328dd82293d4c27326954b Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 4 Jun 2026 14:21:50 +0800 Subject: [PATCH] feat(configuration): update asynq concurrency settings and enhance FAQ handling - Increased the default worker pool size for the asynq server from 16 to 32 to improve performance during concurrent uploads. - Updated the .env.example and docker-compose.yml files to reflect the new default concurrency setting. - Enhanced the handling of FAQ entries in the tool results, ensuring distinct identification through standard questions, improving user experience and clarity in the output. - Refactored related components to utilize the new FAQ attributes, ensuring consistent display and interaction across the application. --- .env.example | 4 +- docker-compose.yml | 1 + frontend/src/types/tool-results.ts | 9 +++ .../components/tool-results/GrepResults.vue | 2 +- .../components/tool-results/SearchResults.vue | 18 +++-- internal/agent/tools/grep_chunks.go | 30 ++++++-- internal/agent/tools/list_knowledge_chunks.go | 32 +++++++- internal/application/service/knowledge_faq.go | 14 +++- .../service/knowledge_housekeeping.go | 75 ++++++++++++++++-- .../service/knowledge_housekeeping_test.go | 77 ++++++++++++++++++- .../application/service/system_setting.go | 4 +- internal/router/task.go | 4 +- internal/router/task_inspector.go | 72 +++++++++++++++++ internal/types/interfaces/task_inspector.go | 16 ++++ 14 files changed, 330 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index a1cd105b..866ca8f3 100644 --- a/.env.example +++ b/.env.example @@ -691,12 +691,12 @@ DOCREADER_TRANSPORT=grpc # WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h # --- Async pipeline tuning (optional) ----------------------------------------- -# Worker pool size for the asynq server. Default 16 — raise it on machines +# Worker pool size for the asynq server. Default 32 — raise it on machines # that handle many concurrent uploads (default Go runtime.NumCPU() under- # provisions for the I/O-bound document pipeline). Can also be set in the # management UI under Settings → System settings (asynq.concurrency); # UI changes require a process restart. -# WEKNORA_ASYNQ_CONCURRENCY=16 +# WEKNORA_ASYNQ_CONCURRENCY=32 # Read/write timeout (in milliseconds) the asynq client uses against Redis. # Default 500ms (writes scale to 1000ms). Bump if you see "i/o timeout" diff --git a/docker-compose.yml b/docker-compose.yml index 7b3c0e15..685cc649 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,6 +199,7 @@ services: # >0 强制限额;=0 走 handler 默认;<0 关闭限额(不建议共享部署使用) - WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-} - WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-} + - WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32} - APK_MIRROR_ARG=${APK_MIRROR_ARG:-} - WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-} depends_on: diff --git a/frontend/src/types/tool-results.ts b/frontend/src/types/tool-results.ts index 8f177ba3..bdfbe6f0 100644 --- a/frontend/src/types/tool-results.ts +++ b/frontend/src/types/tool-results.ts @@ -36,6 +36,12 @@ export interface SearchResultItem { knowledge_id: string; knowledge_title: string; match_type: string; + knowledge_base_type?: string; + // FAQ entries share the owning document's title; the standard question + // gives each entry a distinct, human-readable label. + faq_standard_question?: string; + faq_similar_questions?: string[]; + faq_answers?: string[]; } // Chunk item @@ -203,6 +209,9 @@ export interface GrepKnowledgeResult { knowledge_id: string; knowledge_base_id: string; knowledge_title: string; + // Standard question of the first matched FAQ entry, used as the row label + // since FAQ entries share the owning document's title. + faq_question?: string; title_match?: boolean; chunk_hit_count: number; match_snippet?: string; diff --git a/frontend/src/views/chat/components/tool-results/GrepResults.vue b/frontend/src/views/chat/components/tool-results/GrepResults.vue index 6abb7037..189d218c 100644 --- a/frontend/src/views/chat/components/tool-results/GrepResults.vue +++ b/frontend/src/views/chat/components/tool-results/GrepResults.vue @@ -5,7 +5,7 @@ v-for="(result, index) in results" :key="result.knowledge_id" :index="index + 1" - :title="result.knowledge_title || $t('knowledge.untitledDocument')" + :title="result.faq_question || result.knowledge_title || $t('knowledge.untitledDocument')" :meta="formatMeta(result)" :popup-key="result.knowledge_id || index" :show-popup="!!cleanedSnippet(result)" diff --git a/frontend/src/views/chat/components/tool-results/SearchResults.vue b/frontend/src/views/chat/components/tool-results/SearchResults.vue index 18c929d5..8d05ca4e 100644 --- a/frontend/src/views/chat/components/tool-results/SearchResults.vue +++ b/frontend/src/views/chat/components/tool-results/SearchResults.vue @@ -4,9 +4,9 @@
props.data.results || []); const kbCounts = computed(() => props.data.kb_counts); interface GroupedResult { + key: string; knowledge_id: string; - knowledge_title: string; + title: string; chunks: { content: string; chunk_id: string; knowledge_id: string }[]; } // Hybrid retrieval can return several chunks from the same document; collapse -// them into one row so the same file is not listed repeatedly. +// them into one row so the same file is not listed repeatedly. FAQ entries are +// the exception: they all share the owning document's title, so each entry is +// kept as its own row labelled by its standard question to stay distinguishable. const groupedResults = computed(() => { const map = new Map(); const order: string[] = []; for (const r of results.value) { - const key = r.knowledge_id || r.chunk_id; + const faqQuestion = r.faq_standard_question?.trim(); + const isFaq = !!faqQuestion; + const key = isFaq ? r.chunk_id : r.knowledge_id || r.chunk_id; if (!map.has(key)) { map.set(key, { + key, knowledge_id: r.knowledge_id, - knowledge_title: r.knowledge_title, + title: (isFaq ? faqQuestion : r.knowledge_title) || r.knowledge_title, chunks: [], }); order.push(key); diff --git a/internal/agent/tools/grep_chunks.go b/internal/agent/tools/grep_chunks.go index 7d328223..f11f8e6b 100644 --- a/internal/agent/tools/grep_chunks.go +++ b/internal/agent/tools/grep_chunks.go @@ -258,7 +258,7 @@ func (t *GrepChunksTool) searchChunks( query := t.db.WithContext(ctx).Table("chunks"). Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, "+ - "chunks.knowledge_base_id, chunks.chunk_type, chunks.created_at, "+ + "chunks.knowledge_base_id, chunks.chunk_type, chunks.metadata, chunks.created_at, "+ "knowledges.title as knowledge_title"). Joins("JOIN knowledges ON chunks.knowledge_id = knowledges.id"). Where("chunks.is_enabled = ?", true). @@ -375,6 +375,13 @@ func (t *GrepChunksTool) formatOutput( counts := countRegexHits(r.Content, compiled, queries) snippet := extractSnippetRegex(r.Content, compiled) + // FAQ entries share the owning knowledge's title, so expose the + // standard question as a per-chunk identifier when available. + faqAttr := "" + if q := faqStandardQuestion(&r.Chunk); q != "" { + faqAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q)) + } + t.mu.Lock() seen := t.seenChunks[r.ID] t.seenChunks[r.ID] = true @@ -382,10 +389,11 @@ func (t *GrepChunksTool) formatOutput( if seen { b.WriteString(fmt.Sprintf( - "\n", + "\n", xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle), + faqAttr, r.ChunkIndex, r.MatchScore, )) @@ -401,10 +409,11 @@ func (t *GrepChunksTool) formatOutput( } b.WriteString(fmt.Sprintf( - "\n", + "\n", xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle), + faqAttr, r.ChunkIndex, r.MatchScore, )) @@ -426,9 +435,13 @@ func (t *GrepChunksTool) formatOutput( } type knowledgeAggregation struct { - KnowledgeID string `json:"knowledge_id"` - KnowledgeBaseID string `json:"knowledge_base_id"` - KnowledgeTitle string `json:"knowledge_title"` + KnowledgeID string `json:"knowledge_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeTitle string `json:"knowledge_title"` + // FAQQuestion is the standard question of the first matched FAQ entry in + // this knowledge. FAQ entries share the owning knowledge's title, so the + // frontend uses this to give the row a distinct, human-readable label. + FAQQuestion string `json:"faq_question,omitempty"` TitleMatch bool `json:"title_match"` ChunkHitCount int `json:"chunk_hit_count"` TotalChunkCount int `json:"total_chunk_count"` @@ -484,6 +497,11 @@ func (t *GrepChunksTool) aggregateByKnowledge( if chunk.TitleMatch { entry.TitleMatch = true } + if entry.FAQQuestion == "" { + if q := faqStandardQuestion(&chunk.Chunk); q != "" { + entry.FAQQuestion = q + } + } if entry.MatchSnippet == "" { if snippet := extractSnippetRegex(chunk.Content, compiled); snippet != "" { entry.MatchSnippet = snippet diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index 50c6078c..2d27cf64 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -230,6 +230,11 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess "parent_chunk_id": c.ParentChunkID, } + // FAQ 条目共用所属 Knowledge 的标题,补上标准问作为单条身份标识 + if q := faqStandardQuestion(c); q != "" { + chunkData["faq_question"] = q + } + // 添加图片信息 if c.ImageInfo != "" { var imageInfos []types.ImageInfo @@ -310,8 +315,15 @@ func (t *ListKnowledgeChunksTool) buildOutput( } for _, c := range chunks { - fmt.Fprintf(&b, "\n", - c.ID, c.ChunkIndex, c.ChunkType) + // FAQ entries share the owning knowledge's title, so expose the + // standard question as a per-chunk identifier when available. + if q := faqStandardQuestion(c); q != "" { + fmt.Fprintf(&b, "\n", + c.ID, c.ChunkIndex, c.ChunkType, xmlEscape(q)) + } else { + fmt.Fprintf(&b, "\n", + c.ID, c.ChunkIndex, c.ChunkType) + } fmt.Fprintf(&b, "%s\n", summarizeContent(c.Content)) if c.ImageInfo != "" { @@ -345,6 +357,22 @@ func (t *ListKnowledgeChunksTool) buildOutput( return b.String() } +// faqStandardQuestion returns the FAQ standard question for an FAQ-type chunk, +// or "" for non-FAQ chunks (or when metadata is missing/unparseable). All FAQ +// entries inside one knowledge share the same knowledge title, so surfacing the +// standard question gives each entry a distinct, human-readable identity in +// tool output that would otherwise look like duplicate same-titled chunks. +func faqStandardQuestion(c *types.Chunk) string { + if c == nil || c.ChunkType != types.ChunkTypeFAQ { + return "" + } + meta, err := c.FAQMetadata() + if err != nil || meta == nil { + return "" + } + return strings.TrimSpace(meta.StandardQuestion) +} + // summarizeContent summarizes the content of a chunk func summarizeContent(content string) string { cleaned := strings.TrimSpace(content) diff --git a/internal/application/service/knowledge_faq.go b/internal/application/service/knowledge_faq.go index 463e2fdf..364770a5 100644 --- a/internal/application/service/knowledge_faq.go +++ b/internal/application/service/knowledge_faq.go @@ -1566,7 +1566,7 @@ func (s *knowledgeService) ensureFAQKnowledge( KnowledgeBaseID: kb.ID, Type: types.KnowledgeTypeFAQ, Channel: types.ChannelWeb, - Title: fmt.Sprintf("%s - FAQ", kb.Name), + Title: buildFAQKnowledgeTitle(kb.Name), Description: "FAQ 条目容器", Source: types.KnowledgeTypeFAQ, ParseStatus: "completed", @@ -1581,6 +1581,18 @@ func (s *knowledgeService) ensureFAQKnowledge( return knowledge, nil } +// buildFAQKnowledgeTitle derives the display title for the FAQ container +// knowledge. The title is simply the knowledge base name (falling back to "FAQ" +// when empty) — no suffix is appended, so a KB named "FAQ" stays "FAQ" instead +// of becoming the redundant "FAQ - FAQ". +func buildFAQKnowledgeTitle(kbName string) string { + name := strings.TrimSpace(kbName) + if name == "" { + return "FAQ" + } + return name +} + func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase, tagSeqIDMap map[string]int64) (*types.FAQEntry, error) { meta, err := chunk.FAQMetadata() if err != nil { diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index 73298e66..386efb64 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -28,6 +28,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/robfig/cron/v3" "gorm.io/gorm" ) @@ -38,6 +39,13 @@ type HousekeepingService struct { cfg *config.Config cron *cron.Cron + // inspector lets the sweep distinguish a genuinely orphaned row from + // one whose enrichment subtasks are merely backlogged behind a busy + // queue (no span heartbeat yet because no worker has picked them up). + // nil-safe — a nil inspector disables the queue check and the sweep + // falls back to the span/updated_at heuristics alone. + inspector interfaces.TaskInspector + mu sync.Mutex started bool } @@ -45,10 +53,13 @@ type HousekeepingService struct { // NewHousekeepingService constructs a HousekeepingService. It does NOT start // the cron — call Start in the application bootstrap so a misconfigured // cron schedule cannot prevent the rest of the service from coming up. -func NewHousekeepingService(db *gorm.DB, cfg *config.Config) *HousekeepingService { +func NewHousekeepingService( + db *gorm.DB, cfg *config.Config, inspector interfaces.TaskInspector, +) *HousekeepingService { return &HousekeepingService{ - db: db, - cfg: cfg, + db: db, + cfg: cfg, + inspector: inspector, cron: cron.New(cron.WithSeconds(), cron.WithChain( cron.Recover(cron.DefaultLogger), )), @@ -132,6 +143,18 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { } stuck := h.filterByLastSpanActivity(ctx, candidates, cutoff) + spanSkipped := len(candidates) - len(stuck) + + // Second-stage gate: a row can have a stale span heartbeat yet still + // be perfectly healthy when its enrichment subtasks (summary / + // question / graph / wiki) are merely backlogged behind a busy queue + // — no worker has picked them up, so no span has been written since + // post-process fanned them out. Killing such a row is the false- + // positive users hit under heavy upload bursts. Drop any candidate + // that still has a queued/active task referencing it; only rows with + // nothing left in the queue are treated as genuinely orphaned. + stuck, queueSkipped := h.filterOutQueued(ctx, stuck) + if len(stuck) > 0 { stuckIDs := make([]string, 0, len(stuck)) for _, k := range stuck { @@ -152,13 +175,22 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { res.RowsAffected, threshold) } } - if active := len(candidates) - len(stuck); active > 0 { + if spanSkipped > 0 { // Visibility into "we considered killing N rows but their // span tree showed they're still progressing". Ops can grep // for this if they suspect housekeeping over- or under-fires. logger.Infof(ctx, "[Housekeeping] %d candidate(s) skipped — span heartbeat within threshold", - active) + spanSkipped) + } + if queueSkipped > 0 { + // Visibility into "stale span heartbeat but tasks still queued" + // — i.e. backpressure, not a stuck row. Persistent counts here + // mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY + // or document_process_timeout), not that housekeeping misfires. + logger.Infof(ctx, + "[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)", + queueSkipped) } // Sweep B: knowledge summary stuck. Summary is post-parse; threshold @@ -233,6 +265,39 @@ func (h *HousekeepingService) filterByLastSpanActivity(ctx context.Context, cand return out } +// filterOutQueued returns the subset of candidates that have NO task left +// in the queue backend, plus a count of how many were dropped because a +// task still references them. A dropped candidate is "backlogged, not +// orphaned" — its enrichment subtasks are waiting for a worker, so the +// missing span heartbeat is expected and recovering it would be a false +// positive. When no inspector is wired (nil) the gate is a pass-through +// so behaviour matches the pre-existing span-only sweep. On inspector +// error we fail safe by KEEPING the candidate as stuck (recover it), +// matching the span heartbeat query's fail-safe direction. +func (h *HousekeepingService) filterOutQueued( + ctx context.Context, candidates []types.Knowledge, +) (kept []types.Knowledge, skipped int) { + if h.inspector == nil || len(candidates) == 0 { + return candidates, 0 + } + out := candidates[:0] + for _, k := range candidates { + queued, err := h.inspector.HasQueuedTasksForKnowledge(ctx, k.ID) + if err != nil { + logger.Warnf(ctx, + "[Housekeeping] queue probe failed for %s: %v (will fail safe and treat as stuck)", k.ID, err) + out = append(out, k) + continue + } + if queued { + skipped++ + continue + } + out = append(out, k) + } + return out, skipped +} + // parseHeartbeatTime accepts the timestamp formats Postgres and SQLite // emit for a TIMESTAMP column read back through MAX(). Returns false if // none parse — the caller treats unparseable rows as "no heartbeat", diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index 329fc018..2edf26f3 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -2,11 +2,13 @@ package service import ( "context" + "errors" "testing" "time" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" @@ -95,14 +97,41 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat ).Error) } +// fakeTaskInspector is a controllable TaskInspector for the housekeeping +// suite. queued maps knowledge_id → "still has a queued task"; err forces +// the probe to fail so the fail-safe branch can be exercised. +type fakeTaskInspector struct { + queued map[string]bool + err error +} + +func (f fakeTaskInspector) CancelTasksForKnowledge( + _ context.Context, _ string, +) (int, int, error) { + return 0, 0, nil +} + +func (f fakeTaskInspector) HasQueuedTasksForKnowledge( + _ context.Context, knowledgeID string, +) (bool, error) { + if f.err != nil { + return false, f.err + } + return f.queued[knowledgeID], nil +} + func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService { + return newHousekeepingSvcWithInspector(db, fakeTaskInspector{}) +} + +func newHousekeepingSvcWithInspector(db *gorm.DB, inspector interfaces.TaskInspector) *HousekeepingService { cfg := &config.Config{KnowledgeBase: &config.KnowledgeBaseConfig{ // 1h floor + 10min buffer = 70min cutoff. Tight enough to keep // the test's relative timestamps in seconds; the production // default of 2h+10min is just a constant scale factor. DocumentProcessTimeout: 1 * time.Hour, }} - return NewHousekeepingService(db, cfg) + return NewHousekeepingService(db, cfg, inspector) } // TestHousekeeping_RecoversAbandoned exercises the happy path: a @@ -170,6 +199,52 @@ func TestHousekeeping_NoFalseKill_StaleSpanRecovers(t *testing.T) { "genuinely stuck knowledge (knowledge AND spans both stale) must still be recovered") } +// TestHousekeeping_NoFalseKill_TasksStillQueued is the regression test +// for the backpressure case: a finalizing row whose span heartbeat has +// gone stale (enrichment subtasks fanned out but no worker has picked +// them up yet) must NOT be killed while its tasks are still queued. +func TestHousekeeping_NoFalseKill_TasksStillQueued(t *testing.T) { + db := setupHousekeepingDB(t) + svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{ + queued: map[string]bool{"kid-backlogged": true}, + }) + stale := time.Now().Add(-3 * time.Hour) + // finalizing + stale knowledge + stale span: span-only heuristics + // would flag this as stuck, but the queue still holds its subtasks. + insertKnowledge(t, db, "kid-backlogged", types.ParseStatusFinalizing, stale) + insertSpan(t, db, "kid-backlogged", 1, "post-1", types.SpanStatusRunning, stale) + + svc.runSweep(context.Background()) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-backlogged", + ).Row().Scan(&status)) + assert.Equal(t, types.ParseStatusFinalizing, status, + "finalizing row with tasks still queued must NOT be flipped to failed") +} + +// TestHousekeeping_QueueProbeError_FailsSafe confirms the fail-safe +// direction: when the queue probe errors we still recover the row rather +// than leaving it stranded forever. +func TestHousekeeping_QueueProbeError_FailsSafe(t *testing.T) { + db := setupHousekeepingDB(t) + svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{ + err: errors.New("redis unavailable"), + }) + stale := time.Now().Add(-3 * time.Hour) + insertKnowledge(t, db, "kid-probeerr", types.ParseStatusProcessing, stale) + + svc.runSweep(context.Background()) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-probeerr", + ).Row().Scan(&status)) + assert.Equal(t, types.ParseStatusFailed, status, + "queue probe error must fail safe and still recover the stuck row") +} + // TestHousekeeping_PreservesRecentlyTouched: any knowledge whose // updated_at is within the cutoff is left alone — that's the cheap // fast path that doesn't even consult the spans table. diff --git a/internal/application/service/system_setting.go b/internal/application/service/system_setting.go index b6b0a7d4..ea58b068 100644 --- a/internal/application/service/system_setting.go +++ b/internal/application/service/system_setting.go @@ -168,11 +168,11 @@ var registry = map[string]settingSpec{ // asynq.concurrency is the asynq worker pool size (parallel in-flight // tasks). Read once when the asynq server starts — changing it in the // UI requires a process restart to take effect. Mirrors - // WEKNORA_ASYNQ_CONCURRENCY (default 16). + // WEKNORA_ASYNQ_CONCURRENCY (default 32). "asynq.concurrency": { Type: "int", EnvName: "WEKNORA_ASYNQ_CONCURRENCY", - Default: int64(16), + Default: int64(32), Category: "worker", RequiresRestart: true, Description: "异步任务 worker 并发数(asynq 线程池大小)。" + diff --git a/internal/router/task.go b/internal/router/task.go index 9efb7f00..6c8afa67 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -115,10 +115,10 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration { // runtime.NumCPU(), which under-provisions during batch document uploads: // a single 4-core container can only process 4 documents in parallel even // when 100 are queued, so the queue wait time eats into each task's -// DocumentProcessTimeout budget. 16 is a safer default for the I/O-bound +// DocumentProcessTimeout budget. 32 is a safer default for the I/O-bound // nature of doc parsing (most time is spent in DocReader / embedding RPCs, // not on local CPU). -const defaultAsynqConcurrency = 16 +const defaultAsynqConcurrency = 32 func NewAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { opt := getAsynqRedisClientOpt() diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index 600c42b8..3ff4188b 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -106,6 +106,69 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge( return deleted, cancelled, nil } +// HasQueuedTasksForKnowledge reports whether any pending / scheduled / +// retry / active task referencing knowledgeID still lives in the queue. +// Read-only counterpart of CancelTasksForKnowledge — the housekeeping +// sweep uses it to avoid flagging a backlogged-but-not-orphaned row as +// failed. Short-circuits on the first match and never deletes anything. +func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( + ctx context.Context, knowledgeID string, +) (bool, error) { + if a == nil || a.inspector == nil || knowledgeID == "" { + return false, nil + } + listers := []struct { + state string + list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error) + }{ + {"pending", a.inspector.ListPendingTasks}, + {"scheduled", a.inspector.ListScheduledTasks}, + {"retry", a.inspector.ListRetryTasks}, + {"active", a.inspector.ListActiveTasks}, + } + for _, queue := range queuesScanned { + for _, l := range listers { + if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) { + return true, nil + } + } + } + return false, nil +} + +// queueStateHasMatch pages through one (queue, state) list looking for a +// task that references knowledgeID. Mirrors the delete* scanners but is +// strictly read-only and returns early on the first hit. A backend error +// is logged and treated as "no match" (false); the caller's fail-safe +// then errs toward recovering the row rather than preserving it forever. +func (a *asynqTaskInspector) queueStateHasMatch( + ctx context.Context, queue, knowledgeID, state string, + list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error), +) bool { + page := 1 + for { + tasks, err := list(queue, asynq.PageSize(listPageSize), asynq.Page(page)) + if err != nil { + if !errors.Is(err, asynq.ErrQueueNotFound) { + logger.Warnf(ctx, "[TaskInspector] probe %s queue=%s page=%d: %v", state, queue, page, err) + } + return false + } + if len(tasks) == 0 { + return false + } + for _, t := range tasks { + if matchesKnowledge(t.Type, t.Payload, knowledgeID) { + return true + } + } + if len(tasks) < listPageSize { + return false + } + page++ + } +} + // matchesKnowledge returns true when the task type is one we cancel // AND its payload references the target knowledge ID. func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool { @@ -262,3 +325,12 @@ func (noopTaskInspector) CancelTasksForKnowledge( ) (int, int, error) { return 0, 0, nil } + +// HasQueuedTasksForKnowledge always reports false in Lite mode: inline +// executors never enqueue, so there is no backlog to protect against and +// the housekeeping sweep's span/updated_at checks stay authoritative. +func (noopTaskInspector) HasQueuedTasksForKnowledge( + ctx context.Context, knowledgeID string, +) (bool, error) { + return false, nil +} diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go index 794cbd02..ca597dfb 100644 --- a/internal/types/interfaces/task_inspector.go +++ b/internal/types/interfaces/task_inspector.go @@ -23,4 +23,20 @@ type TaskInspector interface { // best-effort: the row-level abort flag remains the source of // truth, this just prevents wasted work. CancelTasksForKnowledge(ctx context.Context, knowledgeID string) (deleted int, cancelled int, err error) + + // HasQueuedTasksForKnowledge reports whether any pending / scheduled + // / retry / active task referencing the given knowledge ID still + // lives in the queue backend. It is the read-only counterpart of + // CancelTasksForKnowledge: the housekeeping sweep calls it before + // flipping a long-idle "processing"/"finalizing" row to "failed" so + // it can tell a genuinely orphaned row (no task anywhere) from one + // whose enrichment subtasks are merely backlogged behind a busy + // queue (no span heartbeat yet because no worker has picked them up). + // + // Best-effort and short-circuiting: it returns true as soon as the + // first match is seen. On backend error it returns (false, err); + // callers decide the fail-safe direction. Lite mode (no Redis) + // always returns false — inline executors never queue, so the + // span/updated_at checks remain authoritative there. + HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error) }