feat(task-queue): split upstream workers into isolated core/enrichment/maintenance pools
Route summary/sync/maintenance tasks to dedicated queues and servers so parsing, enrichment, and batch work no longer share one concurrency budget, while extending the runtime dashboard and settings validation to reflect the new topology.
This commit is contained in:
@@ -479,8 +479,8 @@ export async function listSystemAuditLog(
|
||||
|
||||
/**
|
||||
* QueueStat mirrors types.QueueStat on the backend: a read-only depth
|
||||
* snapshot of one asynq queue. `pool` groups the queue by worker pool
|
||||
* ("parse" | "wiki"); `weight` is its scheduling weight within that pool.
|
||||
* snapshot of one asynq queue. `pool` identifies the independent worker pool;
|
||||
* `weight` is the queue's scheduling weight within that pool.
|
||||
* Counts follow asynq.QueueInfo semantics — `active` is the number of
|
||||
* tasks currently being processed (the closest thing to "workers busy"),
|
||||
* `pending` is the backlog waiting to be picked up.
|
||||
@@ -503,17 +503,25 @@ export interface QueueStat {
|
||||
memory_usage_bytes: number
|
||||
}
|
||||
|
||||
export interface RuntimeWorkerPool {
|
||||
name: string
|
||||
concurrency: number
|
||||
queue_count: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Runtime queue dashboard payload. `available` is false in Lite mode
|
||||
* (no Redis/asynq) — render an "unavailable in this deployment" state
|
||||
* rather than an empty table. `parse_concurrency` / `wiki_concurrency`
|
||||
* are the CONFIGURED worker-pool sizes (effective on next restart), not
|
||||
* the live busy-worker count (see QueueStat.active for that).
|
||||
* rather than an empty table. Pool concurrency values are CONFIGURED
|
||||
* per-process capacity (effective on next restart), not live busy-worker
|
||||
* count or cluster-wide capacity.
|
||||
*/
|
||||
export interface RuntimeQueuesResponse {
|
||||
available: boolean
|
||||
upstream_concurrency: number
|
||||
parse_concurrency: number
|
||||
wiki_concurrency: number
|
||||
pools: RuntimeWorkerPool[]
|
||||
queues: QueueStat[]
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
@@ -3461,7 +3461,7 @@ export default {
|
||||
},
|
||||
runtime: {
|
||||
title: "Task Queue Runtime",
|
||||
description: "Live depth of the background task queues (parsing / enrichment) and the worker concurrency config. Read-only; auto-refreshes every 5s.",
|
||||
description: "Live background-queue load and per-process capacity for each isolated worker pool. Read-only; auto-refreshes every 5s.",
|
||||
refresh: "Refresh",
|
||||
autoRefresh: "Auto-refresh (every 5s)",
|
||||
loading: "Loading...",
|
||||
@@ -3472,8 +3472,12 @@ export default {
|
||||
empty: "No queue data available",
|
||||
detailsTitle: "Queue details",
|
||||
detailsDescription: "Live load and wait time for every processing lane.",
|
||||
weight: "Weight {value}",
|
||||
footnote: "Concurrency is the configured worker-pool cap (takes effect after restart); \"Active\" is the number of tasks currently being processed.",
|
||||
poolsTitle: "Worker pools",
|
||||
poolsDescription: "Core parsing, enrichment, maintenance, and Wiki use isolated concurrency budgets.",
|
||||
perInstance: "Concurrency is per process",
|
||||
queueCount: "{value} queues",
|
||||
weight: "Pool weight {value}",
|
||||
footnote: "Concurrency is a per-process cap (restart required); weight only controls scheduling within one pool; Active is cluster-wide.",
|
||||
updatedAt: "Updated at {value}",
|
||||
errors: {
|
||||
generic: "Failed to load queue stats",
|
||||
@@ -3483,7 +3487,7 @@ export default {
|
||||
active: "Active",
|
||||
pending: "Pending",
|
||||
retry: "Retrying",
|
||||
concurrency: "Parse / Wiki concurrency",
|
||||
archived: "Dead letters",
|
||||
},
|
||||
columns: {
|
||||
queue: "Queue",
|
||||
@@ -3503,11 +3507,21 @@ export default {
|
||||
paused: "Paused",
|
||||
},
|
||||
pools: {
|
||||
parse: "Parse pool",
|
||||
core: "Core parsing",
|
||||
enrichment: "Enrichment",
|
||||
maintenance: "Maintenance & sync",
|
||||
wiki: "Wiki pool",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "Document parsing and post-process orchestration",
|
||||
enrichment: "Summaries, images, graph, and question generation",
|
||||
maintenance: "Source sync, batch work, and deletion cleanup",
|
||||
wiki: "Wiki content generation and global finalization",
|
||||
},
|
||||
queueNames: {
|
||||
default: "Default (document parse)",
|
||||
summary: "Summary generation",
|
||||
sync: "Source sync",
|
||||
low: "Maintenance & batch jobs",
|
||||
multimodal: "Multimodal (images)",
|
||||
graph: "Graph extraction",
|
||||
@@ -3515,8 +3529,10 @@ export default {
|
||||
wiki: "Wiki ingest",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "Document parsing and general post-processing",
|
||||
low: "Summaries, source sync, batch reparse, and deletion cleanup",
|
||||
default: "Document parsing, manual updates, and post-process orchestration",
|
||||
summary: "Document and data-table summaries",
|
||||
sync: "Manual and scheduled data-source sync",
|
||||
low: "FAQ import, clone/move, batch reparse, and deletion cleanup",
|
||||
multimodal: "Image OCR and multimodal descriptions",
|
||||
graph: "Knowledge graph extraction from document chunks",
|
||||
question: "Question generation from document chunks",
|
||||
@@ -3535,7 +3551,8 @@ export default {
|
||||
default_storage_quota_gb: 'Default storage quota for new tenants (GB)',
|
||||
},
|
||||
asynq: {
|
||||
concurrency: 'Async task worker concurrency',
|
||||
concurrency: 'Total upstream worker budget',
|
||||
wiki_concurrency: 'Wiki worker concurrency',
|
||||
},
|
||||
model: {
|
||||
max_concurrency: 'Default per-model concurrency limit',
|
||||
@@ -3558,7 +3575,9 @@ export default {
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
'Async task worker concurrency (asynq thread-pool size). Document parsing, embedding, and similar tasks are mostly I/O-bound, so raising this value can shorten queue time for bulk uploads. Requires a service process restart to take effect.',
|
||||
'Total per-process concurrency budget for upstream background work (excluding Wiki). It is split into isolated pools: 1/2 core parsing, 3/8 enrichment, and the remainder maintenance and sync. Minimum 3; requires a service restart.',
|
||||
wiki_concurrency:
|
||||
'Per-process concurrency for the dedicated Wiki worker pool, isolated from upstream tasks. Minimum 1; requires a service restart.',
|
||||
},
|
||||
model: {
|
||||
max_concurrency:
|
||||
|
||||
@@ -2466,7 +2466,7 @@ export default {
|
||||
},
|
||||
runtime: {
|
||||
title: "작업 큐 런타임",
|
||||
description: "백그라운드 작업 큐(파싱/보강)의 실시간 깊이와 워커 동시성 설정입니다. 읽기 전용이며 5초마다 자동 새로고침됩니다.",
|
||||
description: "백그라운드 작업 큐의 실시간 부하와 각 독립 워커 풀의 프로세스별 동시성 설정입니다. 5초마다 자동 새로고침됩니다.",
|
||||
refresh: "새로고침",
|
||||
autoRefresh: "자동 새로고침 (5초마다)",
|
||||
loading: "불러오는 중...",
|
||||
@@ -2477,8 +2477,12 @@ export default {
|
||||
empty: "표시할 큐 데이터가 없습니다",
|
||||
detailsTitle: "큐 세부 정보",
|
||||
detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.",
|
||||
weight: "가중치 {value}",
|
||||
footnote: "동시성은 설정된 워커 풀 상한이며(재시작 후 적용), \"실행 중\"은 현재 처리 중인 작업 수입니다.",
|
||||
poolsTitle: "워커 풀",
|
||||
poolsDescription: "핵심 파싱, 보강, 유지 관리 및 Wiki는 독립된 동시성 예산을 사용합니다.",
|
||||
perInstance: "동시성은 프로세스별 설정",
|
||||
queueCount: "큐 {value}개",
|
||||
weight: "풀 내 가중치 {value}",
|
||||
footnote: "동시성은 프로세스별 상한이며 재시작 후 적용됩니다. 가중치는 같은 풀 안의 스케줄링 비율이며 실행 중 수치는 클러스터 전체입니다.",
|
||||
updatedAt: "{value} 업데이트됨",
|
||||
errors: {
|
||||
generic: "큐 상태를 불러오지 못했습니다",
|
||||
@@ -2488,7 +2492,7 @@ export default {
|
||||
active: "실행 중",
|
||||
pending: "대기 중",
|
||||
retry: "재시도 중",
|
||||
concurrency: "파싱 / Wiki 동시성",
|
||||
archived: "데드 레터",
|
||||
},
|
||||
columns: {
|
||||
queue: "큐",
|
||||
@@ -2508,11 +2512,21 @@ export default {
|
||||
paused: "일시중지됨",
|
||||
},
|
||||
pools: {
|
||||
parse: "파싱 풀",
|
||||
core: "핵심 파싱",
|
||||
enrichment: "콘텐츠 보강",
|
||||
maintenance: "유지 관리 및 동기화",
|
||||
wiki: "Wiki 풀",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "문서 파싱 및 후처리 오케스트레이션",
|
||||
enrichment: "요약, 이미지, 그래프 및 질문 생성",
|
||||
maintenance: "데이터 소스 동기화, 일괄 작업 및 삭제 정리",
|
||||
wiki: "Wiki 콘텐츠 생성 및 전체 마무리",
|
||||
},
|
||||
queueNames: {
|
||||
default: "기본(문서 파싱)",
|
||||
summary: "요약 생성",
|
||||
sync: "데이터 소스 동기화",
|
||||
low: "백그라운드 유지 관리 및 일괄 작업",
|
||||
multimodal: "멀티모달(이미지)",
|
||||
graph: "그래프 추출",
|
||||
@@ -2520,8 +2534,10 @@ export default {
|
||||
wiki: "Wiki 동기화",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "문서 파싱 및 일반 후처리",
|
||||
low: "요약, 데이터 소스 동기화, 일괄 재파싱 및 삭제 정리",
|
||||
default: "문서 파싱, 수동 업데이트 및 후처리 오케스트레이션",
|
||||
summary: "문서 및 데이터 테이블 요약",
|
||||
sync: "수동 및 예약 데이터 소스 동기화",
|
||||
low: "FAQ 가져오기, 복제/이동, 일괄 재파싱 및 삭제 정리",
|
||||
multimodal: "이미지 OCR 및 멀티모달 설명",
|
||||
graph: "문서 청크에서 지식 그래프 추출",
|
||||
question: "문서 청크를 기반으로 질문 생성",
|
||||
@@ -2540,7 +2556,8 @@ export default {
|
||||
default_storage_quota_gb: "신규 테넌트 기본 저장 용량 (GB)",
|
||||
},
|
||||
asynq: {
|
||||
concurrency: "비동기 작업 워커 동시 처리 수",
|
||||
concurrency: "상위 워커 총 동시성 예산",
|
||||
wiki_concurrency: "Wiki 워커 동시성",
|
||||
},
|
||||
model: {
|
||||
max_concurrency: "모델 기본 동시 처리 상한",
|
||||
@@ -2563,7 +2580,9 @@ export default {
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
"비동기 작업 worker 동시 처리 수(asynq 스레드 풀 크기)입니다. 문서 파싱·임베딩 등은 대부분 I/O 대기이므로 값을 올리면 대량 업로드 대기 시간을 줄일 수 있습니다. 적용하려면 서비스 프로세스를 재시작해야 합니다.",
|
||||
"Wiki를 제외한 상위 백그라운드 작업의 프로세스별 총 동시성 예산입니다. 핵심 파싱 1/2, 콘텐츠 보강 3/8, 나머지는 유지 관리 및 동기화로 분리됩니다. 최소값은 3이며 적용하려면 서비스를 재시작해야 합니다.",
|
||||
wiki_concurrency:
|
||||
"상위 작업과 분리된 전용 Wiki 워커 풀의 프로세스별 동시성입니다. 최소값은 1이며 적용하려면 서비스를 재시작해야 합니다.",
|
||||
},
|
||||
model: {
|
||||
max_concurrency:
|
||||
|
||||
@@ -2164,7 +2164,7 @@ export default {
|
||||
},
|
||||
runtime: {
|
||||
title: "Среда выполнения очередей",
|
||||
description: "Актуальная глубина фоновых очередей задач (парсинг/обогащение) и настройки параллелизма воркеров. Только для просмотра; автообновление каждые 5 сек.",
|
||||
description: "Текущая нагрузка фоновых очередей и ёмкость каждого изолированного пула на процесс. Автообновление каждые 5 сек.",
|
||||
refresh: "Обновить",
|
||||
autoRefresh: "Автообновление (каждые 5 сек)",
|
||||
loading: "Загрузка...",
|
||||
@@ -2175,8 +2175,12 @@ export default {
|
||||
empty: "Нет данных об очередях",
|
||||
detailsTitle: "Сведения об очередях",
|
||||
detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.",
|
||||
weight: "Вес {value}",
|
||||
footnote: "Параллелизм — это настроенный предел пула воркеров (вступает в силу после перезапуска); «Активные» — число задач, обрабатываемых сейчас.",
|
||||
poolsTitle: "Пулы воркеров",
|
||||
poolsDescription: "Основной разбор, обогащение, обслуживание и Wiki используют независимые бюджеты параллелизма.",
|
||||
perInstance: "Параллелизм указан на процесс",
|
||||
queueCount: "Очередей: {value}",
|
||||
weight: "Вес в пуле {value}",
|
||||
footnote: "Параллелизм указан на процесс и требует перезапуска; вес действует только внутри пула; активные задачи показаны по всему кластеру.",
|
||||
updatedAt: "Обновлено в {value}",
|
||||
errors: {
|
||||
generic: "Не удалось загрузить состояние очередей",
|
||||
@@ -2186,7 +2190,7 @@ export default {
|
||||
active: "Активные",
|
||||
pending: "В очереди",
|
||||
retry: "Повтор",
|
||||
concurrency: "Параллелизм Parse / Wiki",
|
||||
archived: "Мёртвые задачи",
|
||||
},
|
||||
columns: {
|
||||
queue: "Очередь",
|
||||
@@ -2206,11 +2210,21 @@ export default {
|
||||
paused: "Приостановлено",
|
||||
},
|
||||
pools: {
|
||||
parse: "Пул парсинга",
|
||||
core: "Основной разбор",
|
||||
enrichment: "Обогащение",
|
||||
maintenance: "Обслуживание и синхронизация",
|
||||
wiki: "Пул Wiki",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "Разбор документов и оркестрация постобработки",
|
||||
enrichment: "Сводки, изображения, граф и генерация вопросов",
|
||||
maintenance: "Синхронизация источников, пакетные задачи и удаление",
|
||||
wiki: "Создание содержимого Wiki и финальная обработка",
|
||||
},
|
||||
queueNames: {
|
||||
default: "По умолчанию (парсинг документов)",
|
||||
summary: "Создание сводок",
|
||||
sync: "Синхронизация источников",
|
||||
low: "Фоновое обслуживание и пакетные задачи",
|
||||
multimodal: "Мультимодальные (изображения)",
|
||||
graph: "Извлечение графа",
|
||||
@@ -2218,8 +2232,10 @@ export default {
|
||||
wiki: "Синхронизация Wiki",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "Парсинг документов и общая постобработка",
|
||||
low: "Сводки, синхронизация источников, пакетный повторный разбор и удаление",
|
||||
default: "Разбор документов, ручные обновления и оркестрация постобработки",
|
||||
summary: "Сводки документов и таблиц данных",
|
||||
sync: "Ручная и плановая синхронизация источников",
|
||||
low: "Импорт FAQ, клонирование/перемещение, пакетный повторный разбор и удаление",
|
||||
multimodal: "OCR изображений и мультимодальные описания",
|
||||
graph: "Извлечение графа знаний из фрагментов документов",
|
||||
question: "Генерация вопросов по фрагментам документов",
|
||||
@@ -2238,7 +2254,8 @@ export default {
|
||||
default_storage_quota_gb: 'Квота хранилища для новых тенантов по умолчанию (ГБ)',
|
||||
},
|
||||
asynq: {
|
||||
concurrency: 'Параллелизм воркеров асинхронных задач',
|
||||
concurrency: 'Общий бюджет фоновых воркеров',
|
||||
wiki_concurrency: 'Параллелизм воркеров Wiki',
|
||||
},
|
||||
model: {
|
||||
max_concurrency: 'Лимит параллелизма модели по умолчанию',
|
||||
@@ -2261,7 +2278,9 @@ export default {
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
'Параллелизм воркеров асинхронных задач (размер пула потоков asynq). Парсинг документов, эмбеддинги и подобные задачи в основном ждут I/O, поэтому увеличение значения сокращает очередь при массовой загрузке. Для применения требуется перезапуск процесса сервиса.',
|
||||
'Общий бюджет параллелизма фоновых задач на процесс (без Wiki). Он делится на изолированные пулы: 1/2 основной разбор, 3/8 обогащение, остаток — обслуживание и синхронизация. Минимум 3; требуется перезапуск сервиса.',
|
||||
wiki_concurrency:
|
||||
'Параллелизм выделенного пула Wiki на процесс, изолированного от остальных задач. Минимум 1; требуется перезапуск сервиса.',
|
||||
},
|
||||
model: {
|
||||
max_concurrency:
|
||||
|
||||
@@ -2469,7 +2469,7 @@ export default {
|
||||
},
|
||||
runtime: {
|
||||
title: "任务队列运行时",
|
||||
description: "解析/富化等后台任务队列的实时深度,以及 worker 并发配置。仅供观察,每 5 秒自动刷新。",
|
||||
description: "后台任务队列的实时负载,以及各独立 worker 池的每实例并发配置。仅供观察,每 5 秒自动刷新。",
|
||||
refresh: "刷新",
|
||||
autoRefresh: "自动刷新(每 5 秒)",
|
||||
loading: "加载中...",
|
||||
@@ -2480,8 +2480,12 @@ export default {
|
||||
empty: "暂无队列数据",
|
||||
detailsTitle: "队列明细",
|
||||
detailsDescription: "各处理通道的实时负载与等待情况。",
|
||||
weight: "权重 {value}",
|
||||
footnote: "并发数为配置的 worker 池上限(修改后需重启生效);「运行中」为当前实际处理任务数。",
|
||||
poolsTitle: "Worker 池",
|
||||
poolsDescription: "核心解析、内容富化、维护同步和 Wiki 使用相互隔离的并发预算。",
|
||||
perInstance: "并发数为单实例配置",
|
||||
queueCount: "{value} 个队列",
|
||||
weight: "池内权重 {value}",
|
||||
footnote: "Worker 并发数为单实例上限(修改后需重启);权重只决定同一池内的调度份额;「运行中」为集群当前任务数。",
|
||||
updatedAt: "更新于 {value}",
|
||||
errors: {
|
||||
generic: "获取队列状态失败",
|
||||
@@ -2491,7 +2495,7 @@ export default {
|
||||
active: "运行中",
|
||||
pending: "排队中",
|
||||
retry: "重试中",
|
||||
concurrency: "解析 / Wiki 并发",
|
||||
archived: "死信",
|
||||
},
|
||||
columns: {
|
||||
queue: "队列",
|
||||
@@ -2511,11 +2515,21 @@ export default {
|
||||
paused: "已暂停",
|
||||
},
|
||||
pools: {
|
||||
parse: "解析池",
|
||||
core: "核心解析",
|
||||
enrichment: "内容富化",
|
||||
maintenance: "维护与同步",
|
||||
wiki: "Wiki 池",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "文档解析与后处理调度",
|
||||
enrichment: "摘要、图片、图谱与问题生成",
|
||||
maintenance: "数据源同步、批处理与删除清理",
|
||||
wiki: "Wiki 内容生成与全局收尾",
|
||||
},
|
||||
queueNames: {
|
||||
default: "默认(文档解析)",
|
||||
summary: "摘要生成",
|
||||
sync: "数据源同步",
|
||||
low: "后台维护与批处理",
|
||||
multimodal: "多模态(图片)",
|
||||
graph: "图谱抽取",
|
||||
@@ -2523,8 +2537,10 @@ export default {
|
||||
wiki: "Wiki 同步",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "文档解析与通用后处理",
|
||||
low: "摘要、数据源同步、批量重解析与删除清理",
|
||||
default: "文档解析、手工更新与后处理调度",
|
||||
summary: "文档摘要与表格摘要",
|
||||
sync: "手动与定时数据源同步",
|
||||
low: "FAQ 导入、复制移动、批量重解析与删除清理",
|
||||
multimodal: "图片 OCR 与多模态描述",
|
||||
graph: "从文档分块提取知识图谱",
|
||||
question: "基于文档分块生成问题",
|
||||
@@ -2543,7 +2559,8 @@ export default {
|
||||
default_storage_quota_gb: "新租户默认存储配额 (GB)",
|
||||
},
|
||||
asynq: {
|
||||
concurrency: "异步任务并发数",
|
||||
concurrency: "上游 Worker 总并发预算",
|
||||
wiki_concurrency: "Wiki Worker 并发数",
|
||||
},
|
||||
model: {
|
||||
max_concurrency: "模型默认并发上限",
|
||||
@@ -2571,9 +2588,12 @@ export default {
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
"异步任务 worker 并发数(asynq 线程池大小)。" +
|
||||
"文档解析、嵌入等任务多为 I/O 等待,适当提高可缩短批量上传排队时间。" +
|
||||
"修改后需重启服务进程方可生效。",
|
||||
"每个服务实例用于上游后台任务的总并发预算(不含 Wiki)。系统按核心解析 1/2、" +
|
||||
"内容富化 3/8、维护与同步为剩余容量,拆分成相互隔离的 Worker 池。" +
|
||||
"最小值为 3;修改后需重启服务进程方可生效。",
|
||||
wiki_concurrency:
|
||||
"每个服务实例的 Wiki 专用 Worker 并发数,与上游任务池相互隔离。" +
|
||||
"最小值为 1;修改后需重启服务进程方可生效。",
|
||||
},
|
||||
model: {
|
||||
max_concurrency:
|
||||
|
||||
@@ -86,13 +86,29 @@
|
||||
<span class="rq-metric-label">{{ t('system.globalSettings.runtime.summary.retry') }}</span>
|
||||
<strong class="rq-metric-value">{{ totalRetry }}</strong>
|
||||
</div>
|
||||
<div class="rq-metric rq-capacity">
|
||||
<span class="rq-metric-label">{{ t('system.globalSettings.runtime.summary.concurrency') }}</span>
|
||||
<span class="rq-capacity-values">
|
||||
<span><strong>{{ parseConcurrency }}</strong><small>{{ poolLabel('parse') }}</small></span>
|
||||
<i>/</i>
|
||||
<span><strong>{{ wikiConcurrency }}</strong><small>{{ poolLabel('wiki') }}</small></span>
|
||||
</span>
|
||||
<div class="rq-metric" :class="{ 'rq-metric--danger': totalArchived > 0 }">
|
||||
<span class="rq-metric-label">{{ t('system.globalSettings.runtime.summary.archived') }}</span>
|
||||
<strong class="rq-metric-value">{{ totalArchived }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="rq-pools">
|
||||
<div class="rq-pools-header">
|
||||
<div>
|
||||
<h3>{{ t('system.globalSettings.runtime.poolsTitle') }}</h3>
|
||||
<p>{{ t('system.globalSettings.runtime.poolsDescription') }}</p>
|
||||
</div>
|
||||
<span>{{ t('system.globalSettings.runtime.perInstance') }}</span>
|
||||
</div>
|
||||
<div class="rq-pool-strip">
|
||||
<div v-for="pool in pools" :key="pool.name" class="rq-pool-item">
|
||||
<div class="rq-pool-topline">
|
||||
<span>{{ poolLabel(pool.name) }}</span>
|
||||
<strong>{{ pool.concurrency }}</strong>
|
||||
</div>
|
||||
<p>{{ poolDescription(pool.name) }}</p>
|
||||
<small>{{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}</small>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -126,8 +142,10 @@
|
||||
<div class="rq-queue-cell">
|
||||
<span class="rq-queue-name">{{ queueLabel(row.name) }}</span>
|
||||
<span class="rq-queue-meta">
|
||||
{{ queueDescription(row.name) }} · {{ poolLabel(row.pool) }} ·
|
||||
{{ t('system.globalSettings.runtime.weight', { value: row.weight }) }}
|
||||
{{ queueDescription(row.name) }} · {{ poolLabel(row.pool) }}
|
||||
<template v-if="poolQueueCount(row.pool) > 1">
|
||||
· {{ t('system.globalSettings.runtime.weight', { value: row.weight }) }}
|
||||
</template>
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
@@ -168,16 +186,15 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { getRuntimeQueues, type QueueStat } from '@/api/system'
|
||||
import { getRuntimeQueues, type QueueStat, type RuntimeWorkerPool } from '@/api/system'
|
||||
|
||||
const { t, te, locale } = useI18n()
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
|
||||
const queues = ref<QueueStat[]>([])
|
||||
const pools = ref<RuntimeWorkerPool[]>([])
|
||||
const available = ref(true)
|
||||
const parseConcurrency = ref(0)
|
||||
const wikiConcurrency = ref(0)
|
||||
const loading = ref(false)
|
||||
const loadedOnce = ref(false)
|
||||
const error = ref('')
|
||||
@@ -199,6 +216,7 @@ const columns = computed(() => [
|
||||
const totalActive = computed(() => queues.value.reduce((s, q) => s + q.active, 0))
|
||||
const totalPending = computed(() => queues.value.reduce((s, q) => s + q.pending, 0))
|
||||
const totalRetry = computed(() => queues.value.reduce((s, q) => s + q.retry, 0))
|
||||
const totalArchived = computed(() => queues.value.reduce((s, q) => s + q.archived, 0))
|
||||
|
||||
// Friendly per-queue label lives in i18n; falls back to the raw queue
|
||||
// name so a queue added on the backend still renders before translations
|
||||
@@ -218,6 +236,15 @@ function poolLabel(pool: string): string {
|
||||
return te(path) ? (t(path) as string) : pool
|
||||
}
|
||||
|
||||
function poolDescription(pool: string): string {
|
||||
const path = `system.globalSettings.runtime.poolDescriptions.${pool}`
|
||||
return te(path) ? (t(path) as string) : pool
|
||||
}
|
||||
|
||||
function poolQueueCount(pool: string): number {
|
||||
return pools.value.find((item) => item.name === pool)?.queue_count ?? 0
|
||||
}
|
||||
|
||||
function formatLatency(ms: number): string {
|
||||
if (!ms || ms <= 0) return '—'
|
||||
if (ms < 1000) return `${ms} ms`
|
||||
@@ -249,8 +276,7 @@ async function load(showSpinner: boolean) {
|
||||
try {
|
||||
const resp = await getRuntimeQueues()
|
||||
available.value = resp.available
|
||||
parseConcurrency.value = resp.parse_concurrency
|
||||
wikiConcurrency.value = resp.wiki_concurrency
|
||||
pools.value = resp.pools || []
|
||||
queues.value = resp.queues || []
|
||||
updatedAt.value = new Date((resp.timestamp || Date.now() / 1000) * 1000)
|
||||
.toLocaleTimeString(locale.value, { hour12: false })
|
||||
@@ -459,7 +485,7 @@ onUnmounted(() => stopPolling())
|
||||
|
||||
.rq-overview-metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(64px, 1fr)) minmax(190px, 1.8fr);
|
||||
grid-template-columns: repeat(4, minmax(64px, 1fr));
|
||||
align-items: stretch;
|
||||
gap: 24px;
|
||||
flex: 1;
|
||||
@@ -494,42 +520,94 @@ onUnmounted(() => stopPolling())
|
||||
color: var(--td-warning-color);
|
||||
}
|
||||
|
||||
.rq-capacity {
|
||||
padding-left: 28px;
|
||||
border-left: 1px solid var(--td-component-stroke);
|
||||
.rq-metric--danger .rq-metric-value {
|
||||
color: var(--td-error-color);
|
||||
}
|
||||
|
||||
.rq-capacity-values {
|
||||
display: flex;
|
||||
min-height: 22px;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
.rq-pools {
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
|
||||
> span {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 6px;
|
||||
.rq-pools-header {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
margin-bottom: 12px;
|
||||
|
||||
h3 {
|
||||
margin: 0 0 4px;
|
||||
color: var(--td-text-color-primary);
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
i {
|
||||
color: var(--td-text-color-placeholder);
|
||||
p,
|
||||
> span {
|
||||
margin: 0;
|
||||
color: var(--td-text-color-secondary);
|
||||
font-size: 12px;
|
||||
font-style: normal;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
> span {
|
||||
flex-shrink: 0;
|
||||
color: var(--td-text-color-secondary);
|
||||
font-size: 11px;
|
||||
}
|
||||
}
|
||||
|
||||
.rq-pool-strip {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 1px;
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--td-component-stroke);
|
||||
border-radius: 8px;
|
||||
background: var(--td-component-stroke);
|
||||
}
|
||||
|
||||
.rq-pool-item {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
min-height: 106px;
|
||||
flex-direction: column;
|
||||
padding: 14px 16px 13px;
|
||||
background: var(--td-bg-color-container);
|
||||
|
||||
p {
|
||||
margin: 6px 0 8px;
|
||||
color: var(--td-text-color-secondary);
|
||||
font-size: 11px;
|
||||
line-height: 1.45;
|
||||
text-wrap: pretty;
|
||||
}
|
||||
|
||||
small {
|
||||
color: var(--td-text-color-secondary);
|
||||
margin-top: auto;
|
||||
color: var(--td-text-color-placeholder);
|
||||
font-size: 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.rq-pool-topline {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
|
||||
span {
|
||||
color: var(--td-text-color-primary);
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: var(--td-text-color-primary);
|
||||
font-size: 20px;
|
||||
color: var(--td-brand-color);
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
line-height: 1.1;
|
||||
letter-spacing: -0.02em;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,7 +679,6 @@ onUnmounted(() => stopPolling())
|
||||
|
||||
.rq-queue-meta {
|
||||
color: var(--td-text-color-placeholder);
|
||||
font-family: var(--td-font-family-mono, ui-monospace, SFMono-Regular, Menlo, monospace);
|
||||
font-size: 10px;
|
||||
line-height: 1.35;
|
||||
}
|
||||
@@ -706,7 +783,8 @@ onUnmounted(() => stopPolling())
|
||||
|
||||
@media (max-width: 860px) {
|
||||
.rq-header,
|
||||
.rq-details-header {
|
||||
.rq-details-header,
|
||||
.rq-pools-header {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
@@ -727,11 +805,11 @@ onUnmounted(() => stopPolling())
|
||||
|
||||
.rq-overview-metrics {
|
||||
width: 100%;
|
||||
grid-template-columns: repeat(3, minmax(64px, 1fr)) minmax(180px, 1.6fr);
|
||||
grid-template-columns: repeat(4, minmax(64px, 1fr));
|
||||
}
|
||||
|
||||
.rq-capacity {
|
||||
padding-left: 20px;
|
||||
.rq-pool-strip {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -742,16 +820,12 @@ onUnmounted(() => stopPolling())
|
||||
|
||||
.rq-overview-metrics {
|
||||
width: 100%;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.rq-capacity {
|
||||
grid-column: 1 / -1;
|
||||
padding-top: 12px;
|
||||
padding-left: 0;
|
||||
border-top: 1px solid var(--td-component-stroke);
|
||||
border-left: 0;
|
||||
.rq-pool-strip {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.rq-state {
|
||||
|
||||
@@ -234,7 +234,7 @@
|
||||
:disabled="savingKey === item.key"
|
||||
theme="normal"
|
||||
:step="1"
|
||||
:min="0"
|
||||
:min="minimumFor(item)"
|
||||
class="setting-input"
|
||||
@blur="onChange(item)"
|
||||
/>
|
||||
@@ -760,6 +760,12 @@ function placeholderFor(item: SystemSettingItem): string {
|
||||
return String(v)
|
||||
}
|
||||
|
||||
function minimumFor(item: SystemSettingItem): number {
|
||||
if (item.key === 'asynq.concurrency') return 3
|
||||
if (item.key === 'asynq.wiki_concurrency') return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
async function loadSettings() {
|
||||
loading.value = true
|
||||
try {
|
||||
|
||||
@@ -468,9 +468,10 @@ func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types
|
||||
langfuse.InjectTracing(ctx, payload)
|
||||
|
||||
payloadJSON, _ := json.Marshal(payload)
|
||||
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON)
|
||||
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON,
|
||||
asynq.Queue(types.QueueSync), asynq.MaxRetry(5), asynq.Timeout(2*time.Hour))
|
||||
|
||||
_, err = s.taskEnqueuer.Enqueue(task, asynq.Queue("default"))
|
||||
_, err = s.taskEnqueuer.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to enqueue sync task: %v", err)
|
||||
syncLog.Status = types.SyncLogStatusFailed
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent/tools"
|
||||
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
|
||||
@@ -113,7 +114,8 @@ func NewChunkExtractTask(
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.Queue(types.QueueGraph), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeChunkExtract, payload,
|
||||
asynq.Queue(types.QueueGraph), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
info, err := client.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to enqueue task: %v", err)
|
||||
@@ -143,7 +145,8 @@ func NewDataTableSummaryTask(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task := asynq.NewTask(types.TypeDataTableSummary, payload, asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeDataTableSummary, payload,
|
||||
asynq.Queue(types.QueueSummary), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
info, err := client.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to enqueue data table summary task: %v", err)
|
||||
|
||||
@@ -616,7 +616,8 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
|
||||
@@ -1070,7 +1070,8 @@ func (s *knowledgeService) enqueueManualProcessing(ctx context.Context,
|
||||
return fmt.Errorf("failed to marshal manual process payload: %w", err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeManualProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeManualProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to enqueue manual process task: %w", err)
|
||||
|
||||
@@ -196,8 +196,9 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
types.TypeFAQImport,
|
||||
payloadBytes,
|
||||
asynq.TaskID(asynqTaskID),
|
||||
asynq.Queue("default"),
|
||||
asynq.Queue(types.QueueMaintenance),
|
||||
asynq.MaxRetry(maxRetry),
|
||||
asynq.Timeout(2*time.Hour),
|
||||
)
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
|
||||
@@ -433,7 +433,8 @@ func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.C
|
||||
return false
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes,
|
||||
asynq.Queue(types.QueueSummary), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue summary generation for %s: %v", payload.KnowledgeID, err)
|
||||
return false
|
||||
@@ -528,7 +529,8 @@ func (s *KnowledgePostProcessService) enqueueQuestionGenerationTasks(
|
||||
continue
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue(types.QueueQuestion), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes,
|
||||
asynq.Queue(types.QueueQuestion), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation batch %d for %s: %v", batchIndex-1, payload.KnowledgeID, err)
|
||||
continue
|
||||
|
||||
@@ -674,7 +674,8 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
langfuse.InjectTracing(ctx, &postProcessPayload)
|
||||
payloadBytes, err := json.Marshal(postProcessPayload)
|
||||
if err == nil {
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
if _, err := s.task.Enqueue(task); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
|
||||
} else {
|
||||
@@ -3369,7 +3370,8 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
|
||||
continue
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeImageMultimodal, payloadBytes, asynq.Queue(types.QueueMultimodal))
|
||||
task := asynq.NewTask(types.TypeImageMultimodal, payloadBytes,
|
||||
asynq.Queue(types.QueueMultimodal), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
if _, err := s.task.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "Failed to enqueue image multimodal task for %s: %v", img.ServingURL, err)
|
||||
} else {
|
||||
|
||||
@@ -2,13 +2,15 @@ package service
|
||||
|
||||
import (
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asynq.Option {
|
||||
opts := []asynq.Option{
|
||||
asynq.Queue("default"),
|
||||
asynq.Queue(types.QueueDefault),
|
||||
asynq.Timeout(config.DocumentProcessTimeout(cfg)),
|
||||
asynq.MaxRetry(3),
|
||||
}
|
||||
opts = append(opts, extra...)
|
||||
return opts
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -46,9 +47,10 @@ func TestDocumentProcessTaskOptions_defaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
opts := documentProcessTaskOptions(tc.cfg)
|
||||
queue, timeout, maxRetry := parseDocumentProcessOpts(t, opts)
|
||||
assert.Equal(t, "default", queue)
|
||||
assert.Equal(t, types.QueueDefault, queue)
|
||||
assert.Equal(t, config.DefaultDocumentProcessTimeout, timeout)
|
||||
assert.Nil(t, maxRetry)
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -62,16 +64,17 @@ func TestDocumentProcessTaskOptions_configuredTimeout(t *testing.T) {
|
||||
}
|
||||
opts := documentProcessTaskOptions(cfg)
|
||||
queue, timeout, maxRetry := parseDocumentProcessOpts(t, opts)
|
||||
assert.Equal(t, "default", queue)
|
||||
assert.Equal(t, types.QueueDefault, queue)
|
||||
assert.Equal(t, 90*time.Minute, timeout)
|
||||
assert.Nil(t, maxRetry)
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
}
|
||||
|
||||
func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) {
|
||||
t.Parallel()
|
||||
opts := documentProcessTaskOptions(nil, asynq.MaxRetry(3))
|
||||
queue, timeout, maxRetry := parseDocumentProcessOpts(t, opts)
|
||||
assert.Equal(t, "default", queue)
|
||||
assert.Equal(t, types.QueueDefault, queue)
|
||||
assert.Equal(t, config.DefaultDocumentProcessTimeout, timeout)
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
|
||||
@@ -677,7 +677,8 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
return nil
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeKBDelete, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeKBDelete, payloadBytes,
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := s.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to enqueue KB delete task: %v", err)
|
||||
|
||||
@@ -183,19 +183,19 @@ var registry = map[string]settingSpec{
|
||||
"用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" +
|
||||
"每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。",
|
||||
},
|
||||
// asynq.concurrency is the asynq worker pool size (parallel in-flight
|
||||
// tasks). Read once when the asynq server starts — changing it in the
|
||||
// UI requires a process restart to take effect. Mirrors
|
||||
// WEKNORA_ASYNQ_CONCURRENCY (default 32).
|
||||
// asynq.concurrency is the total per-process upstream worker budget.
|
||||
// It is divided deterministically across independent core, enrichment,
|
||||
// and maintenance servers. Read once at startup; changing it requires a
|
||||
// process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32).
|
||||
"asynq.concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
|
||||
Default: int64(32),
|
||||
Default: int64(types.DefaultUpstreamWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "文档解析池的 worker 并发数(asynq 线程池大小,不含 Wiki)。" +
|
||||
"文档解析、嵌入等任务多为 I/O 等待,适当提高可缩短批量上传排队时间。" +
|
||||
"修改后需重启服务进程方可生效。",
|
||||
Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" +
|
||||
"内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" +
|
||||
"最小值为 3;修改后需重启服务进程方可生效。",
|
||||
},
|
||||
// asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool,
|
||||
// separate from asynq.concurrency. Read once when the wiki asynq server
|
||||
@@ -204,7 +204,7 @@ var registry = map[string]settingSpec{
|
||||
"asynq.wiki_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY",
|
||||
Default: int64(16),
|
||||
Default: int64(types.DefaultWikiWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "Wiki 生成专用池的 worker 并发数(与文档解析池相互隔离)。" +
|
||||
@@ -1228,13 +1228,17 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
|
||||
// 400 body verbatim).
|
||||
func validateRegistryEntry(key string, rawValue any) error {
|
||||
switch key {
|
||||
case "asynq.concurrency":
|
||||
case "asynq.concurrency", "asynq.wiki_concurrency":
|
||||
n, err := coerceToPositiveInt64(rawValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n <= 0 {
|
||||
return errors.New("concurrency must be a positive integer")
|
||||
minimum := int64(1)
|
||||
if key == "asynq.concurrency" {
|
||||
minimum = 3
|
||||
}
|
||||
if n < minimum {
|
||||
return fmt.Errorf("concurrency must be at least %d", minimum)
|
||||
}
|
||||
case "ssrf.whitelist":
|
||||
// Coerce into the same shape encodeForType produced. We don't
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package service
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidateWorkerConcurrencyMinimums(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
value any
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true},
|
||||
{name: "upstream minimum", key: "asynq.concurrency", value: 3},
|
||||
{name: "upstream normal", key: "asynq.concurrency", value: 32},
|
||||
{name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true},
|
||||
{name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateRegistryEntry(tt.key, tt.value)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -293,7 +293,8 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
logger.Errorf(ctx, "Failed to marshal knowledge list delete payload: %v", err)
|
||||
return werrors.NewInternalServerError("删除标签下的文档失败")
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes,
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue knowledge list delete task: %v", err)
|
||||
@@ -370,7 +371,8 @@ func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeIndexDelete, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(10))
|
||||
task := asynq.NewTask(types.TypeIndexDelete, payloadBytes,
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(10), asynq.Timeout(time.Hour))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue index delete task: %v", err)
|
||||
|
||||
@@ -264,12 +264,13 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
redisAvailable := os.Getenv("REDIS_ADDR") != ""
|
||||
if redisAvailable {
|
||||
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
|
||||
// Two independent asynq worker pools with separate concurrency budgets:
|
||||
// the parse pool (all queues except QueueWiki) and the wiki pool
|
||||
// (QueueWiki only). Hard capacity isolation so upstream parsing and the
|
||||
// downstream wiki pipeline can't starve each other during concurrent
|
||||
// uploads. Both run the same mux (see router.RunAsynqServer).
|
||||
must(container.Provide(router.NewParseAsynqServer, dig.Name("parseAsynqServer")))
|
||||
// Independent worker pools keep user-facing parsing, high-fanout
|
||||
// enrichment, maintenance/sync, and Wiki generation from consuming one
|
||||
// another's capacity. All pools share one handler mux but subscribe to
|
||||
// disjoint queues from the central topology registry.
|
||||
must(container.Provide(router.NewCoreAsynqServer, dig.Name("coreAsynqServer")))
|
||||
must(container.Provide(router.NewEnrichmentAsynqServer, dig.Name("enrichmentAsynqServer")))
|
||||
must(container.Provide(router.NewMaintenanceAsynqServer, dig.Name("maintenanceAsynqServer")))
|
||||
must(container.Provide(router.NewWikiAsynqServer, dig.Name("wikiAsynqServer")))
|
||||
// Asynq inspector for cancel-by-knowledge-id (best-effort
|
||||
// dequeue of pending/scheduled/retry tasks + active-task cancel).
|
||||
|
||||
@@ -177,7 +177,9 @@ func (s *Scheduler) triggerSync(dataSourceID string, tenantID uint64) {
|
||||
taskID := fmt.Sprintf("dssync:%s:%s", dataSourceID, time.Now().UTC().Truncate(time.Minute).Format("200601021504"))
|
||||
|
||||
_, err = s.taskEnqueuer.Enqueue(task,
|
||||
asynq.Queue("low"),
|
||||
asynq.Queue(types.QueueSync),
|
||||
asynq.MaxRetry(5),
|
||||
asynq.Timeout(2*time.Hour),
|
||||
asynq.TaskID(taskID),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -146,11 +146,19 @@ func (r *fakeSyncLogRepo) HasRunningSync(_ context.Context, dsID string) (bool,
|
||||
|
||||
// fakeTaskEnqueuer counts how many tasks are enqueued.
|
||||
type fakeTaskEnqueuer struct {
|
||||
count atomic.Int64
|
||||
count atomic.Int64
|
||||
lastQueue atomic.Value
|
||||
}
|
||||
|
||||
func (e *fakeTaskEnqueuer) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
e.count.Add(1)
|
||||
for _, opt := range opts {
|
||||
if opt.Type() == asynq.QueueOpt {
|
||||
if queue, ok := opt.Value().(string); ok {
|
||||
e.lastQueue.Store(queue)
|
||||
}
|
||||
}
|
||||
}
|
||||
return &asynq.TaskInfo{ID: "task-fake"}, nil
|
||||
}
|
||||
|
||||
@@ -210,6 +218,9 @@ func TestScheduler_CronFires(t *testing.T) {
|
||||
if enqueuer.count.Load() == 0 {
|
||||
t.Error("expected at least 1 enqueue, got 0")
|
||||
}
|
||||
if queue, _ := enqueuer.lastQueue.Load().(string); queue != types.QueueSync {
|
||||
t.Errorf("scheduled sync queue = %q, want %q", queue, types.QueueSync)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_AddOrUpdate(t *testing.T) {
|
||||
|
||||
@@ -256,7 +256,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
|
||||
return "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes,
|
||||
asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("enqueue task: %w", err)
|
||||
@@ -280,7 +280,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListReparse(
|
||||
return "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListReparse, payloadBytes,
|
||||
asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("enqueue task: %w", err)
|
||||
@@ -2186,7 +2186,8 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) {
|
||||
|
||||
// Enqueue move task
|
||||
task := asynq.NewTask(types.TypeKnowledgeMove, payloadBytes,
|
||||
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
|
||||
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "MoveKnowledge: failed to enqueue task: %v", err)
|
||||
|
||||
@@ -1083,7 +1083,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
|
||||
// Enqueue KB clone task to Asynq
|
||||
task := asynq.NewTask(types.TypeKBClone, payloadBytes,
|
||||
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
|
||||
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
|
||||
|
||||
+45
-33
@@ -1384,26 +1384,24 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
|
||||
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
|
||||
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
|
||||
// UI can render an "unavailable in this deployment" state instead of an
|
||||
// empty table. ParseConcurrency / WikiConcurrency are the *configured*
|
||||
// worker-pool sizes (resolved DB > ENV > default); they are static
|
||||
// capacity, not the live active-worker count (see QueueStat.Active for
|
||||
// what's actually running).
|
||||
type RuntimeQueuesResponse struct {
|
||||
Available bool `json:"available"`
|
||||
ParseConcurrency int `json:"parse_concurrency"`
|
||||
WikiConcurrency int `json:"wiki_concurrency"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
// empty table. Pool concurrency is the configured per-process capacity
|
||||
// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
|
||||
// capacity (see QueueStat.Active for currently running tasks).
|
||||
type RuntimeWorkerPool struct {
|
||||
Name string `json:"name"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
QueueCount int `json:"queue_count"`
|
||||
}
|
||||
|
||||
// defaultParseConcurrency / defaultWikiConcurrency mirror the unexported
|
||||
// router constants (defaultAsynqConcurrency / defaultWikiAsynqConcurrency).
|
||||
// Duplicated here rather than exported from router to avoid an import
|
||||
// cycle (router already imports handler); kept in sync by convention.
|
||||
const (
|
||||
defaultParseConcurrency = 32
|
||||
defaultWikiConcurrency = 16
|
||||
)
|
||||
type RuntimeQueuesResponse struct {
|
||||
Available bool `json:"available"`
|
||||
UpstreamConcurrency int `json:"upstream_concurrency"`
|
||||
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
|
||||
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
|
||||
Pools []RuntimeWorkerPool `json:"pools"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// GetRuntimeQueues godoc
|
||||
// @Summary 获取解析任务队列运行时状态
|
||||
@@ -1414,10 +1412,36 @@ const (
|
||||
// @Router /system/admin/runtime/queues [get]
|
||||
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
ctx := logger.CloneContext(c.Request.Context())
|
||||
upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
|
||||
wikiConcurrency := types.DefaultWikiWorkerConcurrency
|
||||
if h.systemSettingSvc != nil {
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
upstreamConcurrency = int(configured)
|
||||
}
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
wikiConcurrency = int(configured)
|
||||
}
|
||||
}
|
||||
allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
|
||||
queueCounts := make(map[string]int)
|
||||
for _, definition := range types.QueueDefinitions() {
|
||||
queueCounts[definition.Pool]++
|
||||
}
|
||||
resp := RuntimeQueuesResponse{
|
||||
ParseConcurrency: defaultParseConcurrency,
|
||||
WikiConcurrency: defaultWikiConcurrency,
|
||||
Timestamp: time.Now().Unix(),
|
||||
UpstreamConcurrency: allocation.Total,
|
||||
ParseConcurrency: allocation.Total,
|
||||
WikiConcurrency: wikiConcurrency,
|
||||
Pools: []RuntimeWorkerPool{
|
||||
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
|
||||
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
|
||||
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
|
||||
{Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
|
||||
},
|
||||
Timestamp: time.Now().Unix(),
|
||||
}
|
||||
|
||||
if h.taskInspector != nil {
|
||||
@@ -1436,18 +1460,6 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
resp.Queues = []types.QueueStat{}
|
||||
}
|
||||
|
||||
// Resolve configured worker-pool sizes (DB > ENV > default). These
|
||||
// are the same keys NewParseAsynqServer / NewWikiAsynqServer read at
|
||||
// startup, so operators see the value that will apply on next restart.
|
||||
if h.systemSettingSvc != nil {
|
||||
resp.ParseConcurrency = int(h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultParseConcurrency,
|
||||
))
|
||||
resp.WikiConcurrency = int(h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiConcurrency,
|
||||
))
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type runtimeTestSettings struct{}
|
||||
|
||||
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
|
||||
switch key {
|
||||
case "asynq.concurrency":
|
||||
return 32
|
||||
case "asynq.wiki_concurrency":
|
||||
return 16
|
||||
default:
|
||||
return def
|
||||
}
|
||||
}
|
||||
func (runtimeTestSettings) GetString(_ context.Context, _, _, def string) string { return def }
|
||||
func (runtimeTestSettings) GetBool(_ context.Context, _, _ string, def bool) bool { return def }
|
||||
func (runtimeTestSettings) GetStringList(_ context.Context, _, _ string, def []string) []string {
|
||||
return def
|
||||
}
|
||||
func (runtimeTestSettings) List(context.Context) ([]*types.SystemSetting, error) { return nil, nil }
|
||||
func (runtimeTestSettings) Get(context.Context, string) (*types.SystemSetting, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (runtimeTestSettings) Update(context.Context, string, any) (*types.SystemSetting, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (runtimeTestSettings) Reset(context.Context, string) error { return nil }
|
||||
func (runtimeTestSettings) SubscribeRedis(context.Context) error { return nil }
|
||||
|
||||
type runtimeInvalidSettings struct{ runtimeTestSettings }
|
||||
|
||||
func (runtimeInvalidSettings) GetInt(_ context.Context, _ string, _ string, _ int64) int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
type runtimeTestInspector struct{}
|
||||
|
||||
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
|
||||
return []types.QueueStat{}, true, nil
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{
|
||||
systemSettingSvc: runtimeTestSettings{},
|
||||
taskInspector: runtimeTestInspector{},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
|
||||
|
||||
handler.GetRuntimeQueues(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
|
||||
}
|
||||
|
||||
var response RuntimeQueuesResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if !response.Available {
|
||||
t.Fatal("queue inspection should be available")
|
||||
}
|
||||
if response.UpstreamConcurrency != 32 || response.ParseConcurrency != 32 {
|
||||
t.Fatalf("upstream compatibility values are wrong: %+v", response)
|
||||
}
|
||||
want := map[string]struct {
|
||||
concurrency int
|
||||
queueCount int
|
||||
}{
|
||||
types.WorkerPoolCore: {16, 1},
|
||||
types.WorkerPoolEnrichment: {12, 4},
|
||||
types.WorkerPoolMaintenance: {4, 2},
|
||||
types.WorkerPoolWiki: {16, 1},
|
||||
}
|
||||
if len(response.Pools) != len(want) {
|
||||
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
|
||||
}
|
||||
for _, pool := range response.Pools {
|
||||
expected, ok := want[pool.Name]
|
||||
if !ok {
|
||||
t.Fatalf("unexpected pool %q", pool.Name)
|
||||
}
|
||||
if pool.Concurrency != expected.concurrency || pool.QueueCount != expected.queueCount {
|
||||
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
|
||||
pool.Name, pool, expected.concurrency, expected.queueCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesFallsBackFromInvalidHistoricalConcurrency(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{
|
||||
systemSettingSvc: runtimeInvalidSettings{},
|
||||
taskInspector: runtimeTestInspector{},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
|
||||
|
||||
handler.GetRuntimeQueues(ctx)
|
||||
|
||||
var response RuntimeQueuesResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.UpstreamConcurrency != types.DefaultUpstreamWorkerConcurrency ||
|
||||
response.WikiConcurrency != types.DefaultWikiWorkerConcurrency {
|
||||
t.Fatalf("invalid stored values should use worker defaults: %+v", response)
|
||||
}
|
||||
for _, pool := range response.Pools {
|
||||
if pool.Concurrency < 1 {
|
||||
t.Fatalf("pool %q reported non-positive concurrency: %+v", pool.Name, pool)
|
||||
}
|
||||
}
|
||||
}
|
||||
+56
-59
@@ -24,14 +24,13 @@ import (
|
||||
type AsynqTaskParams struct {
|
||||
dig.In
|
||||
|
||||
// ParseServer drains every queue EXCEPT QueueWiki: the user-facing
|
||||
// document parsing / post-processing / enrichment pipeline. WikiServer
|
||||
// drains QueueWiki only. Two independent worker pools give hard capacity
|
||||
// isolation between the upstream parse pipeline and the downstream wiki
|
||||
// pipeline (see NewParseAsynqServer / NewWikiAsynqServer). The same mux is
|
||||
// run on both; asynq routes by the queue each server subscribes to, and
|
||||
// queue↔task-type is 1:1 by convention, so no task is processed twice.
|
||||
ParseServer *asynq.Server `name:"parseAsynqServer"`
|
||||
// Four independent servers provide hard capacity isolation between the
|
||||
// user-facing parse path, high-fanout enrichment, maintenance/sync work,
|
||||
// and Wiki generation. They share one handler mux but subscribe to disjoint
|
||||
// queue sets from types.QueueDefinitions.
|
||||
CoreServer *asynq.Server `name:"coreAsynqServer"`
|
||||
EnrichmentServer *asynq.Server `name:"enrichmentAsynqServer"`
|
||||
MaintenanceServer *asynq.Server `name:"maintenanceAsynqServer"`
|
||||
WikiServer *asynq.Server `name:"wikiAsynqServer"`
|
||||
KnowledgeService interfaces.KnowledgeService
|
||||
KnowledgeBaseService interfaces.KnowledgeBaseService
|
||||
@@ -120,28 +119,14 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
|
||||
return asynq.DefaultRetryDelayFunc(n, e, t)
|
||||
}
|
||||
|
||||
// defaultAsynqConcurrency is the worker pool size used when
|
||||
// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to
|
||||
// runtime.NumCPU(), which under-provisions during batch document uploads:
|
||||
// a single 4-core container can only process 4 documents in parallel even
|
||||
// when 100 are queued, so the queue wait time eats into each task's
|
||||
// DocumentProcessTimeout budget. 32 is a safer default for the I/O-bound
|
||||
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
|
||||
// not on local CPU).
|
||||
const defaultAsynqConcurrency = 32
|
||||
|
||||
// defaultWikiAsynqConcurrency is the worker pool size for the dedicated wiki
|
||||
// pool (QueueWiki) when WEKNORA_WIKI_ASYNQ_CONCURRENCY is unset. Kept
|
||||
// independent from the parse pool so the two can be sized separately for the
|
||||
// host / LLM-provider budget. The wiki pipeline is LLM-bound (synthesis model
|
||||
// calls dominate), so a moderate default avoids overrunning upstream provider
|
||||
// concurrency while still letting many documents' wiki generation run in
|
||||
// parallel across knowledge bases.
|
||||
const defaultWikiAsynqConcurrency = 16
|
||||
// Worker defaults live in types so server construction and runtime reporting
|
||||
// cannot drift. The upstream budget is divided without increasing historical
|
||||
// total capacity; Wiki remains separate because its model-heavy workload has a
|
||||
// different provider-capacity profile.
|
||||
|
||||
// newAsynqServer builds an asynq server bound to a specific queue set and
|
||||
// concurrency. Shared by the parse and wiki pools so both inherit the same
|
||||
// Redis options and retry-delay policy.
|
||||
// concurrency. Every worker pool uses it so Redis options and retry-delay
|
||||
// policy stay consistent across the topology.
|
||||
func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
return asynq.NewServer(
|
||||
@@ -166,47 +151,60 @@ func backgroundTaskMiddleware() asynq.MiddlewareFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki.
|
||||
// This is the user-facing document parse / post-process / enrichment pipeline.
|
||||
// Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from
|
||||
// the historical single-pool behaviour so existing deployments see the same
|
||||
// parse throughput.
|
||||
func NewParseAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
concurrency := defaultAsynqConcurrency
|
||||
func resolveUpstreamPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency {
|
||||
total := types.DefaultUpstreamWorkerConcurrency
|
||||
if svc != nil {
|
||||
n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultAsynqConcurrency)
|
||||
n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency)
|
||||
if n > 0 {
|
||||
concurrency = int(n)
|
||||
total = int(n)
|
||||
}
|
||||
}
|
||||
log.Printf("asynq parse-pool server starting with concurrency=%d redis_op_timeout=%dms",
|
||||
concurrency, readRedisOpTimeoutMs())
|
||||
return newAsynqServer(concurrency, map[string]int{
|
||||
types.QueueDefault: 3, // Default priority queue
|
||||
types.QueueLow: 1, // Lowest priority queue
|
||||
types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks
|
||||
types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks
|
||||
types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks
|
||||
})
|
||||
return types.AllocateWorkerPoolConcurrency(total)
|
||||
}
|
||||
|
||||
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. Runs the
|
||||
// same mux as the parse pool but only ever pulls wiki tasks, so its concurrency
|
||||
// NewCoreAsynqServer runs the latency-sensitive document parse and
|
||||
// post-process orchestration path.
|
||||
func NewCoreAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
log.Printf("asynq core-pool server starting with concurrency=%d total_upstream=%d redis_op_timeout=%dms",
|
||||
allocation.Core, allocation.Total, readRedisOpTimeoutMs())
|
||||
return newAsynqServer(allocation.Core, types.QueueWeightsForPool(types.WorkerPoolCore))
|
||||
}
|
||||
|
||||
// NewEnrichmentAsynqServer runs high-fanout summary, multimodal, graph, and
|
||||
// question generation without consuming core parsing capacity.
|
||||
func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
log.Printf("asynq enrichment-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.Enrichment, allocation.Total)
|
||||
return newAsynqServer(allocation.Enrichment, types.QueueWeightsForPool(types.WorkerPoolEnrichment))
|
||||
}
|
||||
|
||||
// NewMaintenanceAsynqServer runs connector sync, cleanup, deletion, and batch
|
||||
// dispatch work. QueueMaintenance keeps the legacy Redis name "low", so old
|
||||
// tasks are drained safely during rolling upgrades.
|
||||
func NewMaintenanceAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
log.Printf("asynq maintenance-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.Maintenance, allocation.Total)
|
||||
return newAsynqServer(allocation.Maintenance, types.QueueWeightsForPool(types.WorkerPoolMaintenance))
|
||||
}
|
||||
|
||||
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. It runs
|
||||
// the shared handler mux but only ever pulls wiki tasks, so its concurrency
|
||||
// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on
|
||||
// wiki generation. This is the hard capacity isolation that prevents the parse
|
||||
// pipeline from starving wiki (and vice-versa) during concurrent uploads.
|
||||
func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
concurrency := defaultWikiAsynqConcurrency
|
||||
concurrency := types.DefaultWikiWorkerConcurrency
|
||||
if svc != nil {
|
||||
n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiAsynqConcurrency)
|
||||
n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency)
|
||||
if n > 0 {
|
||||
concurrency = int(n)
|
||||
}
|
||||
}
|
||||
log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency)
|
||||
return newAsynqServer(concurrency, map[string]int{
|
||||
types.QueueWiki: 1,
|
||||
})
|
||||
return newAsynqServer(concurrency, types.QueueWeightsForPool(types.WorkerPoolWiki))
|
||||
}
|
||||
|
||||
func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
@@ -295,11 +293,8 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
|
||||
mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle)
|
||||
|
||||
// Run the same mux on both pools. Each server subscribes to a disjoint set
|
||||
// of queues (parse pool = everything except QueueWiki; wiki pool =
|
||||
// QueueWiki only), so a given task is dequeued by exactly one pool. This is
|
||||
// how the two pools get independent concurrency budgets while sharing one
|
||||
// handler registration.
|
||||
// Run the same mux on every pool. Queue definitions are disjoint, so a given
|
||||
// task is dequeued by exactly one pool while handlers stay registered once.
|
||||
runPool := func(name string, srv *asynq.Server) {
|
||||
go func() {
|
||||
if err := srv.Run(mux); err != nil {
|
||||
@@ -307,7 +302,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
}
|
||||
}()
|
||||
}
|
||||
runPool("parse-pool", params.ParseServer)
|
||||
runPool("core-pool", params.CoreServer)
|
||||
runPool("enrichment-pool", params.EnrichmentServer)
|
||||
runPool("maintenance-pool", params.MaintenanceServer)
|
||||
runPool("wiki-pool", params.WikiServer)
|
||||
return mux
|
||||
}
|
||||
|
||||
@@ -50,30 +50,13 @@ type knowledgeIDProbe struct {
|
||||
// MUST include every queue any cancelable task type can land in; the
|
||||
// multimodal queue is required here so cancelling a knowledge also purges
|
||||
// its (potentially hundreds of) pending image:multimodal tasks.
|
||||
var queuesScanned = []string{
|
||||
types.QueueDefault,
|
||||
types.QueueLow,
|
||||
types.QueueMultimodal,
|
||||
types.QueueGraph,
|
||||
types.QueueQuestion,
|
||||
types.QueueWiki,
|
||||
}
|
||||
|
||||
// queueMeta records each queue's worker pool and scheduling weight for
|
||||
// the runtime dashboard. Weights MUST stay in sync with the maps in
|
||||
// NewParseAsynqServer / NewWikiAsynqServer (task.go) — this is display
|
||||
// metadata only, the servers remain the source of truth for scheduling.
|
||||
var queueMeta = map[string]struct {
|
||||
pool string
|
||||
weight int
|
||||
}{
|
||||
types.QueueDefault: {pool: "parse", weight: 3},
|
||||
types.QueueLow: {pool: "parse", weight: 1},
|
||||
types.QueueMultimodal: {pool: "parse", weight: 1},
|
||||
types.QueueGraph: {pool: "parse", weight: 1},
|
||||
types.QueueQuestion: {pool: "parse", weight: 1},
|
||||
types.QueueWiki: {pool: "wiki", weight: 1},
|
||||
}
|
||||
var queuesScanned = func() []string {
|
||||
queues := make([]string, 0, len(types.QueueDefinitions()))
|
||||
for _, definition := range types.QueueDefinitions() {
|
||||
queues = append(queues, definition.Name)
|
||||
}
|
||||
return queues
|
||||
}()
|
||||
|
||||
// taskTypesForKnowledgeCancel lists every asynq task type that carries
|
||||
// a knowledge_id in its payload and should be cancelable. The set is
|
||||
@@ -155,7 +138,7 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
|
||||
// QueueStats returns a depth snapshot for every queue this app enqueues
|
||||
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
|
||||
// the result onto types.QueueStat, attaching static pool/weight metadata
|
||||
// from queueMeta. A queue that has never received a task yields
|
||||
// from the central queue registry. A queue that has never received a task yields
|
||||
// ErrQueueNotFound from asynq; we still surface it as a zeroed row so the
|
||||
// dashboard shows the complete lane set even before a queue receives its
|
||||
// first task.
|
||||
@@ -165,13 +148,14 @@ func (a *asynqTaskInspector) QueueStats(
|
||||
if a == nil || a.inspector == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
stats := make([]types.QueueStat, 0, len(queuesScanned))
|
||||
for _, queue := range queuesScanned {
|
||||
meta := queueMeta[queue]
|
||||
definitions := types.QueueDefinitions()
|
||||
stats := make([]types.QueueStat, 0, len(definitions))
|
||||
for _, definition := range definitions {
|
||||
queue := definition.Name
|
||||
stat := types.QueueStat{
|
||||
Name: queue,
|
||||
Pool: meta.pool,
|
||||
Weight: meta.weight,
|
||||
Pool: definition.Pool,
|
||||
Weight: definition.Weight,
|
||||
}
|
||||
info, err := a.inspector.GetQueueInfo(queue)
|
||||
if err != nil {
|
||||
|
||||
+137
-32
@@ -1,37 +1,143 @@
|
||||
package types
|
||||
|
||||
// Asynq queue names. MUST stay in sync with the Queues weight maps in
|
||||
// router.NewParseAsynqServer / router.NewWikiAsynqServer — a task enqueued to a
|
||||
// queue that no server subscribes to will never be consumed.
|
||||
// Worker-pool names are part of the runtime observability API. Each pool is
|
||||
// backed by an independent asynq.Server, so concurrency is hard-isolated
|
||||
// between pools instead of being only a weighted dequeue preference.
|
||||
const (
|
||||
QueueDefault = "default"
|
||||
QueueLow = "low"
|
||||
// QueueMultimodal isolates high-volume, slow VLM image tasks (OCR + caption)
|
||||
// so a single large scanned PDF (hundreds–thousands of page images) cannot
|
||||
// saturate the shared worker pool and block user-facing document parsing in
|
||||
// the default queue.
|
||||
QueueMultimodal = "multimodal"
|
||||
// QueueGraph isolates high-volume, slow graph-extraction tasks (one per
|
||||
// chunk, LLM-backed, only when Neo4j is enabled). Same rationale as
|
||||
// QueueMultimodal: a large document must not flood the default queue.
|
||||
QueueGraph = "graph"
|
||||
// QueueQuestion isolates high-volume, slow question-generation tasks (one
|
||||
// per 20-chunk batch, LLM-backed). Keeps a large document's hundreds of
|
||||
// question batches from starving the lightweight tasks in the low queue
|
||||
// (summary, deletes, wiki ingest).
|
||||
QueueQuestion = "question"
|
||||
// QueueWiki is the dedicated lane for the Wiki ingest pipeline. Unlike the
|
||||
// other queues above (which all share a single asynq worker pool), this
|
||||
// queue is consumed by a SEPARATE asynq server (router.NewWikiAsynqServer)
|
||||
// with its own concurrency budget. Hard capacity isolation: heavy document
|
||||
// parsing in the upstream pool can never starve wiki generation, and a
|
||||
// wiki-generation burst can never starve user-facing parsing. Sizing the
|
||||
// two pools independently replaces the old weighted-lottery approach where
|
||||
// wiki sat on the low queue (weight 1) and got ~1/13 of the schedule
|
||||
// during upload storms.
|
||||
QueueWiki = "wiki"
|
||||
WorkerPoolCore = "core"
|
||||
WorkerPoolEnrichment = "enrichment"
|
||||
WorkerPoolMaintenance = "maintenance"
|
||||
WorkerPoolWiki = "wiki"
|
||||
|
||||
// DefaultUpstreamWorkerConcurrency is split across core, enrichment,
|
||||
// and maintenance. Wiki stays separately configurable because its model
|
||||
// workload has a different capacity profile.
|
||||
DefaultUpstreamWorkerConcurrency = 32
|
||||
DefaultWikiWorkerConcurrency = 16
|
||||
)
|
||||
|
||||
// Asynq queue names. QueueMaintenance intentionally keeps the physical Redis
|
||||
// name "low" so tasks enqueued by older releases remain consumable during a
|
||||
// rolling deployment. New code uses the business-semantic constant.
|
||||
const (
|
||||
QueueDefault = "default"
|
||||
QueueSummary = "summary"
|
||||
QueueMultimodal = "multimodal"
|
||||
QueueGraph = "graph"
|
||||
QueueQuestion = "question"
|
||||
QueueSync = "sync"
|
||||
QueueMaintenance = "low"
|
||||
QueueWiki = "wiki"
|
||||
)
|
||||
|
||||
// QueueDefinition is the single source of truth for queue topology. Worker
|
||||
// servers and runtime inspection both consume this registry, preventing the
|
||||
// scheduling weights shown to operators from drifting from the actual server
|
||||
// configuration.
|
||||
type QueueDefinition struct {
|
||||
Name string
|
||||
Pool string
|
||||
Weight int
|
||||
TaskTypes []string
|
||||
}
|
||||
|
||||
var queueDefinitions = []QueueDefinition{
|
||||
{Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, TaskTypes: []string{
|
||||
TypeDocumentProcess, TypeManualProcess, TypeKnowledgePostProcess,
|
||||
}},
|
||||
{Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, TaskTypes: []string{
|
||||
TypeSummaryGeneration, TypeDataTableSummary,
|
||||
}},
|
||||
{Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeImageMultimodal}},
|
||||
{Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeChunkExtract}},
|
||||
{Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeQuestionGeneration}},
|
||||
{Name: QueueSync, Pool: WorkerPoolMaintenance, Weight: 2, TaskTypes: []string{TypeDataSourceSync}},
|
||||
{Name: QueueMaintenance, Pool: WorkerPoolMaintenance, Weight: 1, TaskTypes: []string{
|
||||
TypeFAQImport, TypeKBClone, TypeIndexDelete, TypeKBDelete,
|
||||
TypeKnowledgeListDelete, TypeKnowledgeListReparse, TypeKnowledgeMove,
|
||||
}},
|
||||
{Name: QueueWiki, Pool: WorkerPoolWiki, Weight: 1, TaskTypes: []string{TypeWikiIngest, TypeWikiFinalize}},
|
||||
}
|
||||
|
||||
// QueueDefinitions returns a copy so callers cannot mutate global topology.
|
||||
func QueueDefinitions() []QueueDefinition {
|
||||
definitions := make([]QueueDefinition, len(queueDefinitions))
|
||||
for i, definition := range queueDefinitions {
|
||||
definitions[i] = definition
|
||||
definitions[i].TaskTypes = append([]string(nil), definition.TaskTypes...)
|
||||
}
|
||||
return definitions
|
||||
}
|
||||
|
||||
// QueueForTaskType returns the declared queue for a task type. Producers still
|
||||
// pass the queue explicitly to asynq, while tests and observability can use
|
||||
// this mapping to detect drift.
|
||||
func QueueForTaskType(taskType string) (string, bool) {
|
||||
for _, definition := range queueDefinitions {
|
||||
for _, declaredType := range definition.TaskTypes {
|
||||
if declaredType == taskType {
|
||||
return definition.Name, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// QueueWeightsForPool returns the asynq queue configuration for one worker
|
||||
// pool. An empty map indicates a programming error in the pool declaration.
|
||||
func QueueWeightsForPool(pool string) map[string]int {
|
||||
weights := make(map[string]int)
|
||||
for _, definition := range queueDefinitions {
|
||||
if definition.Pool == pool {
|
||||
weights[definition.Name] = definition.Weight
|
||||
}
|
||||
}
|
||||
return weights
|
||||
}
|
||||
|
||||
// WorkerPoolConcurrency is the deterministic split of the total upstream
|
||||
// worker budget. Keeping this pure allocation function in types lets the
|
||||
// server constructors and the System Admin API report identical values.
|
||||
type WorkerPoolConcurrency struct {
|
||||
Total int
|
||||
Core int
|
||||
Enrichment int
|
||||
Maintenance int
|
||||
}
|
||||
|
||||
// AllocateWorkerPoolConcurrency reserves 1/2 of the upstream budget for the
|
||||
// latency-sensitive core pipeline, 3/8 for LLM/VLM enrichment, and the
|
||||
// remainder for sync/cleanup/batch work. A minimum total of three guarantees
|
||||
// every independent server has at least one worker.
|
||||
func AllocateWorkerPoolConcurrency(total int) WorkerPoolConcurrency {
|
||||
if total < 3 {
|
||||
total = 3
|
||||
}
|
||||
core := total / 2
|
||||
enrichment := (total * 3) / 8
|
||||
if core < 1 {
|
||||
core = 1
|
||||
}
|
||||
if enrichment < 1 {
|
||||
enrichment = 1
|
||||
}
|
||||
maintenance := total - core - enrichment
|
||||
if maintenance < 1 {
|
||||
maintenance = 1
|
||||
if core >= enrichment && core > 1 {
|
||||
core--
|
||||
} else if enrichment > 1 {
|
||||
enrichment--
|
||||
}
|
||||
}
|
||||
return WorkerPoolConcurrency{
|
||||
Total: total,
|
||||
Core: core,
|
||||
Enrichment: enrichment,
|
||||
Maintenance: maintenance,
|
||||
}
|
||||
}
|
||||
|
||||
// QueueStat is a read-only depth snapshot of a single asynq queue, used
|
||||
// by the System Admin runtime dashboard. Field names mirror the counts
|
||||
// exposed by asynq.QueueInfo. Pool / Weight are static metadata (which
|
||||
@@ -40,10 +146,9 @@ const (
|
||||
// hard-coding the topology.
|
||||
type QueueStat struct {
|
||||
Name string `json:"name"`
|
||||
// Pool is the worker pool that drains this queue: "parse" or "wiki".
|
||||
// Pool is the independent worker pool that drains this queue.
|
||||
Pool string `json:"pool"`
|
||||
// Weight is the queue's scheduling weight inside its pool. Must match
|
||||
// the maps in router.NewParseAsynqServer / NewWikiAsynqServer.
|
||||
// Weight is the queue's scheduling weight inside its pool.
|
||||
Weight int `json:"weight"`
|
||||
// Size is the total number of tasks in the queue (pending + active +
|
||||
// scheduled + retry + aggregating + archived).
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package types
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) {
|
||||
definitions := QueueDefinitions()
|
||||
if len(definitions) == 0 {
|
||||
t.Fatal("queue registry must not be empty")
|
||||
}
|
||||
|
||||
validPools := map[string]bool{
|
||||
WorkerPoolCore: true,
|
||||
WorkerPoolEnrichment: true,
|
||||
WorkerPoolMaintenance: true,
|
||||
WorkerPoolWiki: true,
|
||||
}
|
||||
seen := make(map[string]bool, len(definitions))
|
||||
seenTaskTypes := make(map[string]string)
|
||||
for _, definition := range definitions {
|
||||
if seen[definition.Name] {
|
||||
t.Fatalf("duplicate queue definition %q", definition.Name)
|
||||
}
|
||||
seen[definition.Name] = true
|
||||
if !validPools[definition.Pool] {
|
||||
t.Fatalf("queue %q references unknown pool %q", definition.Name, definition.Pool)
|
||||
}
|
||||
if definition.Weight <= 0 {
|
||||
t.Fatalf("queue %q has non-positive weight %d", definition.Name, definition.Weight)
|
||||
}
|
||||
if len(definition.TaskTypes) == 0 {
|
||||
t.Fatalf("queue %q declares no task types", definition.Name)
|
||||
}
|
||||
for _, taskType := range definition.TaskTypes {
|
||||
if previousQueue, exists := seenTaskTypes[taskType]; exists {
|
||||
t.Fatalf("task type %q is declared by both %q and %q", taskType, previousQueue, definition.Name)
|
||||
}
|
||||
seenTaskTypes[taskType] = definition.Name
|
||||
queue, ok := QueueForTaskType(taskType)
|
||||
if !ok || queue != definition.Name {
|
||||
t.Fatalf("task type %q resolves to queue %q, want %q", taskType, queue, definition.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for pool := range validPools {
|
||||
if len(QueueWeightsForPool(pool)) == 0 {
|
||||
t.Fatalf("worker pool %q has no queues", pool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueMaintenanceKeepsLegacyPhysicalName(t *testing.T) {
|
||||
if QueueMaintenance != "low" {
|
||||
t.Fatalf("maintenance queue must keep legacy Redis name during rolling upgrades, got %q", QueueMaintenance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryAsynqTaskTypeHasADeclaredQueue(t *testing.T) {
|
||||
taskTypes := []string{
|
||||
TypeChunkExtract, TypeDocumentProcess, TypeFAQImport,
|
||||
TypeQuestionGeneration, TypeSummaryGeneration, TypeKBClone,
|
||||
TypeIndexDelete, TypeKBDelete, TypeKnowledgeListDelete,
|
||||
TypeKnowledgeListReparse, TypeKnowledgeMove, TypeDataTableSummary,
|
||||
TypeImageMultimodal, TypeKnowledgePostProcess, TypeManualProcess,
|
||||
TypeDataSourceSync, TypeWikiIngest, TypeWikiFinalize,
|
||||
}
|
||||
for _, taskType := range taskTypes {
|
||||
if _, ok := QueueForTaskType(taskType); !ok {
|
||||
t.Fatalf("task type %q has no declared queue", taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerPoolConcurrencyPreservesBudget(t *testing.T) {
|
||||
for _, total := range []int{3, 4, 8, 32, 64} {
|
||||
allocation := AllocateWorkerPoolConcurrency(total)
|
||||
if allocation.Total != total {
|
||||
t.Fatalf("total=%d: allocation reported total %d", total, allocation.Total)
|
||||
}
|
||||
if allocation.Core < 1 || allocation.Enrichment < 1 || allocation.Maintenance < 1 {
|
||||
t.Fatalf("total=%d: every pool must receive capacity: %+v", total, allocation)
|
||||
}
|
||||
if allocation.Core+allocation.Enrichment+allocation.Maintenance != total {
|
||||
t.Fatalf("total=%d: split does not preserve budget: %+v", total, allocation)
|
||||
}
|
||||
}
|
||||
|
||||
minimum := AllocateWorkerPoolConcurrency(1)
|
||||
if minimum.Total != 3 || minimum.Core+minimum.Enrichment+minimum.Maintenance != 3 {
|
||||
t.Fatalf("undersized budget should clamp to three workers: %+v", minimum)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user