feat(configuration): update asynq concurrency settings and enhance FAQ handling
- Increased the default worker pool size for the asynq server from 16 to 32 to improve performance during concurrent uploads. - Updated the .env.example and docker-compose.yml files to reflect the new default concurrency setting. - Enhanced the handling of FAQ entries in the tool results, ensuring distinct identification through standard questions, improving user experience and clarity in the output. - Refactored related components to utilize the new FAQ attributes, ensuring consistent display and interaction across the application.
This commit is contained in:
+2
-2
@@ -691,12 +691,12 @@ DOCREADER_TRANSPORT=grpc
|
||||
# WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h
|
||||
|
||||
# --- Async pipeline tuning (optional) -----------------------------------------
|
||||
# Worker pool size for the asynq server. Default 16 — raise it on machines
|
||||
# Worker pool size for the asynq server. Default 32 — raise it on machines
|
||||
# that handle many concurrent uploads (default Go runtime.NumCPU() under-
|
||||
# provisions for the I/O-bound document pipeline). Can also be set in the
|
||||
# management UI under Settings → System settings (asynq.concurrency);
|
||||
# UI changes require a process restart.
|
||||
# WEKNORA_ASYNQ_CONCURRENCY=16
|
||||
# WEKNORA_ASYNQ_CONCURRENCY=32
|
||||
|
||||
# Read/write timeout (in milliseconds) the asynq client uses against Redis.
|
||||
# Default 500ms (writes scale to 1000ms). Bump if you see "i/o timeout"
|
||||
|
||||
@@ -199,6 +199,7 @@ services:
|
||||
# >0 强制限额;=0 走 handler 默认;<0 关闭限额(不建议共享部署使用)
|
||||
- WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-}
|
||||
- WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-}
|
||||
- WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32}
|
||||
- APK_MIRROR_ARG=${APK_MIRROR_ARG:-}
|
||||
- WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-}
|
||||
depends_on:
|
||||
|
||||
@@ -36,6 +36,12 @@ export interface SearchResultItem {
|
||||
knowledge_id: string;
|
||||
knowledge_title: string;
|
||||
match_type: string;
|
||||
knowledge_base_type?: string;
|
||||
// FAQ entries share the owning document's title; the standard question
|
||||
// gives each entry a distinct, human-readable label.
|
||||
faq_standard_question?: string;
|
||||
faq_similar_questions?: string[];
|
||||
faq_answers?: string[];
|
||||
}
|
||||
|
||||
// Chunk item
|
||||
@@ -203,6 +209,9 @@ export interface GrepKnowledgeResult {
|
||||
knowledge_id: string;
|
||||
knowledge_base_id: string;
|
||||
knowledge_title: string;
|
||||
// Standard question of the first matched FAQ entry, used as the row label
|
||||
// since FAQ entries share the owning document's title.
|
||||
faq_question?: string;
|
||||
title_match?: boolean;
|
||||
chunk_hit_count: number;
|
||||
match_snippet?: string;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
v-for="(result, index) in results"
|
||||
:key="result.knowledge_id"
|
||||
:index="index + 1"
|
||||
:title="result.knowledge_title || $t('knowledge.untitledDocument')"
|
||||
:title="result.faq_question || result.knowledge_title || $t('knowledge.untitledDocument')"
|
||||
:meta="formatMeta(result)"
|
||||
:popup-key="result.knowledge_id || index"
|
||||
:show-popup="!!cleanedSnippet(result)"
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
<div v-if="groupedResults.length > 0" class="results-list">
|
||||
<ResultRow
|
||||
v-for="(group, idx) in groupedResults"
|
||||
:key="group.knowledge_id"
|
||||
:key="group.key"
|
||||
:index="idx + 1"
|
||||
:title="group.knowledge_title"
|
||||
:title="group.title"
|
||||
:meta="$t('agentStream.grepResults.chunkHits', { count: group.chunks.length })"
|
||||
:popup-key="group.knowledge_id"
|
||||
:chunks="group.chunks"
|
||||
@@ -41,22 +41,28 @@ const results = computed(() => props.data.results || []);
|
||||
const kbCounts = computed(() => props.data.kb_counts);
|
||||
|
||||
interface GroupedResult {
|
||||
key: string;
|
||||
knowledge_id: string;
|
||||
knowledge_title: string;
|
||||
title: string;
|
||||
chunks: { content: string; chunk_id: string; knowledge_id: string }[];
|
||||
}
|
||||
|
||||
// Hybrid retrieval can return several chunks from the same document; collapse
|
||||
// them into one row so the same file is not listed repeatedly.
|
||||
// them into one row so the same file is not listed repeatedly. FAQ entries are
|
||||
// the exception: they all share the owning document's title, so each entry is
|
||||
// kept as its own row labelled by its standard question to stay distinguishable.
|
||||
const groupedResults = computed<GroupedResult[]>(() => {
|
||||
const map = new Map<string, GroupedResult>();
|
||||
const order: string[] = [];
|
||||
for (const r of results.value) {
|
||||
const key = r.knowledge_id || r.chunk_id;
|
||||
const faqQuestion = r.faq_standard_question?.trim();
|
||||
const isFaq = !!faqQuestion;
|
||||
const key = isFaq ? r.chunk_id : r.knowledge_id || r.chunk_id;
|
||||
if (!map.has(key)) {
|
||||
map.set(key, {
|
||||
key,
|
||||
knowledge_id: r.knowledge_id,
|
||||
knowledge_title: r.knowledge_title,
|
||||
title: (isFaq ? faqQuestion : r.knowledge_title) || r.knowledge_title,
|
||||
chunks: [],
|
||||
});
|
||||
order.push(key);
|
||||
|
||||
@@ -258,7 +258,7 @@ func (t *GrepChunksTool) searchChunks(
|
||||
|
||||
query := t.db.WithContext(ctx).Table("chunks").
|
||||
Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, "+
|
||||
"chunks.knowledge_base_id, chunks.chunk_type, chunks.created_at, "+
|
||||
"chunks.knowledge_base_id, chunks.chunk_type, chunks.metadata, chunks.created_at, "+
|
||||
"knowledges.title as knowledge_title").
|
||||
Joins("JOIN knowledges ON chunks.knowledge_id = knowledges.id").
|
||||
Where("chunks.is_enabled = ?", true).
|
||||
@@ -375,6 +375,13 @@ func (t *GrepChunksTool) formatOutput(
|
||||
counts := countRegexHits(r.Content, compiled, queries)
|
||||
snippet := extractSnippetRegex(r.Content, compiled)
|
||||
|
||||
// FAQ entries share the owning knowledge's title, so expose the
|
||||
// standard question as a per-chunk identifier when available.
|
||||
faqAttr := ""
|
||||
if q := faqStandardQuestion(&r.Chunk); q != "" {
|
||||
faqAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q))
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
seen := t.seenChunks[r.ID]
|
||||
t.seenChunks[r.ID] = true
|
||||
@@ -382,10 +389,11 @@ func (t *GrepChunksTool) formatOutput(
|
||||
|
||||
if seen {
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\" chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
|
||||
xmlEscape(r.ID),
|
||||
xmlEscape(r.KnowledgeID),
|
||||
xmlEscape(r.KnowledgeTitle),
|
||||
faqAttr,
|
||||
r.ChunkIndex,
|
||||
r.MatchScore,
|
||||
))
|
||||
@@ -401,10 +409,11 @@ func (t *GrepChunksTool) formatOutput(
|
||||
}
|
||||
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\" chunk_index=\"%d\" score=\"%.3f\">\n",
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\">\n",
|
||||
xmlEscape(r.ID),
|
||||
xmlEscape(r.KnowledgeID),
|
||||
xmlEscape(r.KnowledgeTitle),
|
||||
faqAttr,
|
||||
r.ChunkIndex,
|
||||
r.MatchScore,
|
||||
))
|
||||
@@ -426,9 +435,13 @@ func (t *GrepChunksTool) formatOutput(
|
||||
}
|
||||
|
||||
type knowledgeAggregation struct {
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeTitle string `json:"knowledge_title"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeTitle string `json:"knowledge_title"`
|
||||
// FAQQuestion is the standard question of the first matched FAQ entry in
|
||||
// this knowledge. FAQ entries share the owning knowledge's title, so the
|
||||
// frontend uses this to give the row a distinct, human-readable label.
|
||||
FAQQuestion string `json:"faq_question,omitempty"`
|
||||
TitleMatch bool `json:"title_match"`
|
||||
ChunkHitCount int `json:"chunk_hit_count"`
|
||||
TotalChunkCount int `json:"total_chunk_count"`
|
||||
@@ -484,6 +497,11 @@ func (t *GrepChunksTool) aggregateByKnowledge(
|
||||
if chunk.TitleMatch {
|
||||
entry.TitleMatch = true
|
||||
}
|
||||
if entry.FAQQuestion == "" {
|
||||
if q := faqStandardQuestion(&chunk.Chunk); q != "" {
|
||||
entry.FAQQuestion = q
|
||||
}
|
||||
}
|
||||
if entry.MatchSnippet == "" {
|
||||
if snippet := extractSnippetRegex(chunk.Content, compiled); snippet != "" {
|
||||
entry.MatchSnippet = snippet
|
||||
|
||||
@@ -230,6 +230,11 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
|
||||
"parent_chunk_id": c.ParentChunkID,
|
||||
}
|
||||
|
||||
// FAQ 条目共用所属 Knowledge 的标题,补上标准问作为单条身份标识
|
||||
if q := faqStandardQuestion(c); q != "" {
|
||||
chunkData["faq_question"] = q
|
||||
}
|
||||
|
||||
// 添加图片信息
|
||||
if c.ImageInfo != "" {
|
||||
var imageInfos []types.ImageInfo
|
||||
@@ -310,8 +315,15 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
}
|
||||
|
||||
for _, c := range chunks {
|
||||
fmt.Fprintf(&b, "<chunk chunk_id=\"%s\" chunk_index=\"%d\" type=\"%s\">\n",
|
||||
c.ID, c.ChunkIndex, c.ChunkType)
|
||||
// FAQ entries share the owning knowledge's title, so expose the
|
||||
// standard question as a per-chunk identifier when available.
|
||||
if q := faqStandardQuestion(c); q != "" {
|
||||
fmt.Fprintf(&b, "<chunk chunk_id=\"%s\" chunk_index=\"%d\" type=\"%s\" question=\"%s\">\n",
|
||||
c.ID, c.ChunkIndex, c.ChunkType, xmlEscape(q))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "<chunk chunk_id=\"%s\" chunk_index=\"%d\" type=\"%s\">\n",
|
||||
c.ID, c.ChunkIndex, c.ChunkType)
|
||||
}
|
||||
fmt.Fprintf(&b, "<content>%s</content>\n", summarizeContent(c.Content))
|
||||
|
||||
if c.ImageInfo != "" {
|
||||
@@ -345,6 +357,22 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// faqStandardQuestion returns the FAQ standard question for an FAQ-type chunk,
|
||||
// or "" for non-FAQ chunks (or when metadata is missing/unparseable). All FAQ
|
||||
// entries inside one knowledge share the same knowledge title, so surfacing the
|
||||
// standard question gives each entry a distinct, human-readable identity in
|
||||
// tool output that would otherwise look like duplicate same-titled chunks.
|
||||
func faqStandardQuestion(c *types.Chunk) string {
|
||||
if c == nil || c.ChunkType != types.ChunkTypeFAQ {
|
||||
return ""
|
||||
}
|
||||
meta, err := c.FAQMetadata()
|
||||
if err != nil || meta == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(meta.StandardQuestion)
|
||||
}
|
||||
|
||||
// summarizeContent summarizes the content of a chunk
|
||||
func summarizeContent(content string) string {
|
||||
cleaned := strings.TrimSpace(content)
|
||||
|
||||
@@ -1566,7 +1566,7 @@ func (s *knowledgeService) ensureFAQKnowledge(
|
||||
KnowledgeBaseID: kb.ID,
|
||||
Type: types.KnowledgeTypeFAQ,
|
||||
Channel: types.ChannelWeb,
|
||||
Title: fmt.Sprintf("%s - FAQ", kb.Name),
|
||||
Title: buildFAQKnowledgeTitle(kb.Name),
|
||||
Description: "FAQ 条目容器",
|
||||
Source: types.KnowledgeTypeFAQ,
|
||||
ParseStatus: "completed",
|
||||
@@ -1581,6 +1581,18 @@ func (s *knowledgeService) ensureFAQKnowledge(
|
||||
return knowledge, nil
|
||||
}
|
||||
|
||||
// buildFAQKnowledgeTitle derives the display title for the FAQ container
|
||||
// knowledge. The title is simply the knowledge base name (falling back to "FAQ"
|
||||
// when empty) — no suffix is appended, so a KB named "FAQ" stays "FAQ" instead
|
||||
// of becoming the redundant "FAQ - FAQ".
|
||||
func buildFAQKnowledgeTitle(kbName string) string {
|
||||
name := strings.TrimSpace(kbName)
|
||||
if name == "" {
|
||||
return "FAQ"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase, tagSeqIDMap map[string]int64) (*types.FAQEntry, error) {
|
||||
meta, err := chunk.FAQMetadata()
|
||||
if err != nil {
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/robfig/cron/v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -38,6 +39,13 @@ type HousekeepingService struct {
|
||||
cfg *config.Config
|
||||
cron *cron.Cron
|
||||
|
||||
// inspector lets the sweep distinguish a genuinely orphaned row from
|
||||
// one whose enrichment subtasks are merely backlogged behind a busy
|
||||
// queue (no span heartbeat yet because no worker has picked them up).
|
||||
// nil-safe — a nil inspector disables the queue check and the sweep
|
||||
// falls back to the span/updated_at heuristics alone.
|
||||
inspector interfaces.TaskInspector
|
||||
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
}
|
||||
@@ -45,10 +53,13 @@ type HousekeepingService struct {
|
||||
// NewHousekeepingService constructs a HousekeepingService. It does NOT start
|
||||
// the cron — call Start in the application bootstrap so a misconfigured
|
||||
// cron schedule cannot prevent the rest of the service from coming up.
|
||||
func NewHousekeepingService(db *gorm.DB, cfg *config.Config) *HousekeepingService {
|
||||
func NewHousekeepingService(
|
||||
db *gorm.DB, cfg *config.Config, inspector interfaces.TaskInspector,
|
||||
) *HousekeepingService {
|
||||
return &HousekeepingService{
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
inspector: inspector,
|
||||
cron: cron.New(cron.WithSeconds(), cron.WithChain(
|
||||
cron.Recover(cron.DefaultLogger),
|
||||
)),
|
||||
@@ -132,6 +143,18 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
}
|
||||
|
||||
stuck := h.filterByLastSpanActivity(ctx, candidates, cutoff)
|
||||
spanSkipped := len(candidates) - len(stuck)
|
||||
|
||||
// Second-stage gate: a row can have a stale span heartbeat yet still
|
||||
// be perfectly healthy when its enrichment subtasks (summary /
|
||||
// question / graph / wiki) are merely backlogged behind a busy queue
|
||||
// — no worker has picked them up, so no span has been written since
|
||||
// post-process fanned them out. Killing such a row is the false-
|
||||
// positive users hit under heavy upload bursts. Drop any candidate
|
||||
// that still has a queued/active task referencing it; only rows with
|
||||
// nothing left in the queue are treated as genuinely orphaned.
|
||||
stuck, queueSkipped := h.filterOutQueued(ctx, stuck)
|
||||
|
||||
if len(stuck) > 0 {
|
||||
stuckIDs := make([]string, 0, len(stuck))
|
||||
for _, k := range stuck {
|
||||
@@ -152,13 +175,22 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
res.RowsAffected, threshold)
|
||||
}
|
||||
}
|
||||
if active := len(candidates) - len(stuck); active > 0 {
|
||||
if spanSkipped > 0 {
|
||||
// Visibility into "we considered killing N rows but their
|
||||
// span tree showed they're still progressing". Ops can grep
|
||||
// for this if they suspect housekeeping over- or under-fires.
|
||||
logger.Infof(ctx,
|
||||
"[Housekeeping] %d candidate(s) skipped — span heartbeat within threshold",
|
||||
active)
|
||||
spanSkipped)
|
||||
}
|
||||
if queueSkipped > 0 {
|
||||
// Visibility into "stale span heartbeat but tasks still queued"
|
||||
// — i.e. backpressure, not a stuck row. Persistent counts here
|
||||
// mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY
|
||||
// or document_process_timeout), not that housekeeping misfires.
|
||||
logger.Infof(ctx,
|
||||
"[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)",
|
||||
queueSkipped)
|
||||
}
|
||||
|
||||
// Sweep B: knowledge summary stuck. Summary is post-parse; threshold
|
||||
@@ -233,6 +265,39 @@ func (h *HousekeepingService) filterByLastSpanActivity(ctx context.Context, cand
|
||||
return out
|
||||
}
|
||||
|
||||
// filterOutQueued returns the subset of candidates that have NO task left
|
||||
// in the queue backend, plus a count of how many were dropped because a
|
||||
// task still references them. A dropped candidate is "backlogged, not
|
||||
// orphaned" — its enrichment subtasks are waiting for a worker, so the
|
||||
// missing span heartbeat is expected and recovering it would be a false
|
||||
// positive. When no inspector is wired (nil) the gate is a pass-through
|
||||
// so behaviour matches the pre-existing span-only sweep. On inspector
|
||||
// error we fail safe by KEEPING the candidate as stuck (recover it),
|
||||
// matching the span heartbeat query's fail-safe direction.
|
||||
func (h *HousekeepingService) filterOutQueued(
|
||||
ctx context.Context, candidates []types.Knowledge,
|
||||
) (kept []types.Knowledge, skipped int) {
|
||||
if h.inspector == nil || len(candidates) == 0 {
|
||||
return candidates, 0
|
||||
}
|
||||
out := candidates[:0]
|
||||
for _, k := range candidates {
|
||||
queued, err := h.inspector.HasQueuedTasksForKnowledge(ctx, k.ID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx,
|
||||
"[Housekeeping] queue probe failed for %s: %v (will fail safe and treat as stuck)", k.ID, err)
|
||||
out = append(out, k)
|
||||
continue
|
||||
}
|
||||
if queued {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, skipped
|
||||
}
|
||||
|
||||
// parseHeartbeatTime accepts the timestamp formats Postgres and SQLite
|
||||
// emit for a TIMESTAMP column read back through MAX(). Returns false if
|
||||
// none parse — the caller treats unparseable rows as "no heartbeat",
|
||||
|
||||
@@ -2,11 +2,13 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
@@ -95,14 +97,41 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat
|
||||
).Error)
|
||||
}
|
||||
|
||||
// fakeTaskInspector is a controllable TaskInspector for the housekeeping
|
||||
// suite. queued maps knowledge_id → "still has a queued task"; err forces
|
||||
// the probe to fail so the fail-safe branch can be exercised.
|
||||
type fakeTaskInspector struct {
|
||||
queued map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) CancelTasksForKnowledge(
|
||||
_ context.Context, _ string,
|
||||
) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) HasQueuedTasksForKnowledge(
|
||||
_ context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
if f.err != nil {
|
||||
return false, f.err
|
||||
}
|
||||
return f.queued[knowledgeID], nil
|
||||
}
|
||||
|
||||
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
|
||||
return newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
|
||||
}
|
||||
|
||||
func newHousekeepingSvcWithInspector(db *gorm.DB, inspector interfaces.TaskInspector) *HousekeepingService {
|
||||
cfg := &config.Config{KnowledgeBase: &config.KnowledgeBaseConfig{
|
||||
// 1h floor + 10min buffer = 70min cutoff. Tight enough to keep
|
||||
// the test's relative timestamps in seconds; the production
|
||||
// default of 2h+10min is just a constant scale factor.
|
||||
DocumentProcessTimeout: 1 * time.Hour,
|
||||
}}
|
||||
return NewHousekeepingService(db, cfg)
|
||||
return NewHousekeepingService(db, cfg, inspector)
|
||||
}
|
||||
|
||||
// TestHousekeeping_RecoversAbandoned exercises the happy path: a
|
||||
@@ -170,6 +199,52 @@ func TestHousekeeping_NoFalseKill_StaleSpanRecovers(t *testing.T) {
|
||||
"genuinely stuck knowledge (knowledge AND spans both stale) must still be recovered")
|
||||
}
|
||||
|
||||
// TestHousekeeping_NoFalseKill_TasksStillQueued is the regression test
|
||||
// for the backpressure case: a finalizing row whose span heartbeat has
|
||||
// gone stale (enrichment subtasks fanned out but no worker has picked
|
||||
// them up yet) must NOT be killed while its tasks are still queued.
|
||||
func TestHousekeeping_NoFalseKill_TasksStillQueued(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
|
||||
queued: map[string]bool{"kid-backlogged": true},
|
||||
})
|
||||
stale := time.Now().Add(-3 * time.Hour)
|
||||
// finalizing + stale knowledge + stale span: span-only heuristics
|
||||
// would flag this as stuck, but the queue still holds its subtasks.
|
||||
insertKnowledge(t, db, "kid-backlogged", types.ParseStatusFinalizing, stale)
|
||||
insertSpan(t, db, "kid-backlogged", 1, "post-1", types.SpanStatusRunning, stale)
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-backlogged",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.ParseStatusFinalizing, status,
|
||||
"finalizing row with tasks still queued must NOT be flipped to failed")
|
||||
}
|
||||
|
||||
// TestHousekeeping_QueueProbeError_FailsSafe confirms the fail-safe
|
||||
// direction: when the queue probe errors we still recover the row rather
|
||||
// than leaving it stranded forever.
|
||||
func TestHousekeeping_QueueProbeError_FailsSafe(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
|
||||
err: errors.New("redis unavailable"),
|
||||
})
|
||||
stale := time.Now().Add(-3 * time.Hour)
|
||||
insertKnowledge(t, db, "kid-probeerr", types.ParseStatusProcessing, stale)
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-probeerr",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.ParseStatusFailed, status,
|
||||
"queue probe error must fail safe and still recover the stuck row")
|
||||
}
|
||||
|
||||
// TestHousekeeping_PreservesRecentlyTouched: any knowledge whose
|
||||
// updated_at is within the cutoff is left alone — that's the cheap
|
||||
// fast path that doesn't even consult the spans table.
|
||||
|
||||
@@ -168,11 +168,11 @@ var registry = map[string]settingSpec{
|
||||
// asynq.concurrency is the asynq worker pool size (parallel in-flight
|
||||
// tasks). Read once when the asynq server starts — changing it in the
|
||||
// UI requires a process restart to take effect. Mirrors
|
||||
// WEKNORA_ASYNQ_CONCURRENCY (default 16).
|
||||
// WEKNORA_ASYNQ_CONCURRENCY (default 32).
|
||||
"asynq.concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
|
||||
Default: int64(16),
|
||||
Default: int64(32),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "异步任务 worker 并发数(asynq 线程池大小)。" +
|
||||
|
||||
@@ -115,10 +115,10 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
|
||||
// runtime.NumCPU(), which under-provisions during batch document uploads:
|
||||
// a single 4-core container can only process 4 documents in parallel even
|
||||
// when 100 are queued, so the queue wait time eats into each task's
|
||||
// DocumentProcessTimeout budget. 16 is a safer default for the I/O-bound
|
||||
// DocumentProcessTimeout budget. 32 is a safer default for the I/O-bound
|
||||
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
|
||||
// not on local CPU).
|
||||
const defaultAsynqConcurrency = 16
|
||||
const defaultAsynqConcurrency = 32
|
||||
|
||||
func NewAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
|
||||
@@ -106,6 +106,69 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge(
|
||||
return deleted, cancelled, nil
|
||||
}
|
||||
|
||||
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
|
||||
// retry / active task referencing knowledgeID still lives in the queue.
|
||||
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
|
||||
// sweep uses it to avoid flagging a backlogged-but-not-orphaned row as
|
||||
// failed. Short-circuits on the first match and never deletes anything.
|
||||
func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
|
||||
ctx context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
if a == nil || a.inspector == nil || knowledgeID == "" {
|
||||
return false, nil
|
||||
}
|
||||
listers := []struct {
|
||||
state string
|
||||
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error)
|
||||
}{
|
||||
{"pending", a.inspector.ListPendingTasks},
|
||||
{"scheduled", a.inspector.ListScheduledTasks},
|
||||
{"retry", a.inspector.ListRetryTasks},
|
||||
{"active", a.inspector.ListActiveTasks},
|
||||
}
|
||||
for _, queue := range queuesScanned {
|
||||
for _, l := range listers {
|
||||
if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// queueStateHasMatch pages through one (queue, state) list looking for a
|
||||
// task that references knowledgeID. Mirrors the delete* scanners but is
|
||||
// strictly read-only and returns early on the first hit. A backend error
|
||||
// is logged and treated as "no match" (false); the caller's fail-safe
|
||||
// then errs toward recovering the row rather than preserving it forever.
|
||||
func (a *asynqTaskInspector) queueStateHasMatch(
|
||||
ctx context.Context, queue, knowledgeID, state string,
|
||||
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error),
|
||||
) bool {
|
||||
page := 1
|
||||
for {
|
||||
tasks, err := list(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] probe %s queue=%s page=%d: %v", state, queue, page, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return false
|
||||
}
|
||||
page++
|
||||
}
|
||||
}
|
||||
|
||||
// matchesKnowledge returns true when the task type is one we cancel
|
||||
// AND its payload references the target knowledge ID.
|
||||
func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool {
|
||||
@@ -262,3 +325,12 @@ func (noopTaskInspector) CancelTasksForKnowledge(
|
||||
) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
// HasQueuedTasksForKnowledge always reports false in Lite mode: inline
|
||||
// executors never enqueue, so there is no backlog to protect against and
|
||||
// the housekeeping sweep's span/updated_at checks stay authoritative.
|
||||
func (noopTaskInspector) HasQueuedTasksForKnowledge(
|
||||
ctx context.Context, knowledgeID string,
|
||||
) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -23,4 +23,20 @@ type TaskInspector interface {
|
||||
// best-effort: the row-level abort flag remains the source of
|
||||
// truth, this just prevents wasted work.
|
||||
CancelTasksForKnowledge(ctx context.Context, knowledgeID string) (deleted int, cancelled int, err error)
|
||||
|
||||
// HasQueuedTasksForKnowledge reports whether any pending / scheduled
|
||||
// / retry / active task referencing the given knowledge ID still
|
||||
// lives in the queue backend. It is the read-only counterpart of
|
||||
// CancelTasksForKnowledge: the housekeeping sweep calls it before
|
||||
// flipping a long-idle "processing"/"finalizing" row to "failed" so
|
||||
// it can tell a genuinely orphaned row (no task anywhere) from one
|
||||
// whose enrichment subtasks are merely backlogged behind a busy
|
||||
// queue (no span heartbeat yet because no worker has picked them up).
|
||||
//
|
||||
// Best-effort and short-circuiting: it returns true as soon as the
|
||||
// first match is seen. On backend error it returns (false, err);
|
||||
// callers decide the fail-safe direction. Lite mode (no Redis)
|
||||
// always returns false — inline executors never queue, so the
|
||||
// span/updated_at checks remain authoritative there.
|
||||
HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user