diff --git a/.env.example b/.env.example
index 4f4c5d35..f492c7a8 100644
--- a/.env.example
+++ b/.env.example
@@ -716,22 +716,35 @@ DOCREADER_TRANSPORT=grpc
# WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h
# --- Async pipeline tuning (optional) -----------------------------------------
-# Worker pool size for the PARSE asynq server (document parse / post-process /
-# embedding / multimodal / graph / question — everything except wiki). Default
-# 32 — raise it on machines that handle many concurrent uploads (default Go
-# runtime.NumCPU() under-provisions for the I/O-bound document pipeline). Can
-# also be set in the management UI under Settings → System settings
-# (asynq.concurrency); UI changes require a process restart.
-# WEKNORA_ASYNQ_CONCURRENCY=32
+# Per-instance guaranteed worker capacity for document parse / manual reparse.
+# Core and enrichment can additionally borrow from the shared elastic pool.
+# WEKNORA_ASYNQ_CORE_CONCURRENCY=8
+
+# Dedicated lightweight post-process/fan-out workers. Keeping this separate
+# prevents completed parses from waiting behind long DocReader jobs.
+# WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=2
+
+# Per-instance guaranteed capacity for summary, image, graph, and question
+# enrichment tasks.
+# WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=12
+
+# Per-instance capacity for datasource sync, batch operations, and cleanup.
+# Maintenance is not eligible for shared burst capacity because its tasks can
+# be long-running.
+# WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=4
+
+# Elastic capacity shared by core parse and enrichment queues. Queue dequeue is
+# atomic, so these workers safely supplement both dedicated pools.
+# WEKNORA_ASYNQ_SHARED_CONCURRENCY=6
# Worker pool size for the DEDICATED wiki asynq server. The wiki generation
# pipeline runs on its own pool with its own concurrency budget, hard-isolated
# from the parse pool above: heavy document parsing can never starve wiki
# generation during upload storms, and a wiki burst can never starve
-# user-facing parsing. Default 16. Wiki work is synthesis-LLM-bound, so size
+# user-facing parsing. Default 8. Wiki work is synthesis-LLM-bound, so size
# this to your LLM provider's concurrency budget. Also settable in the UI
# (asynq.wiki_concurrency); requires a process restart.
-# WEKNORA_WIKI_ASYNQ_CONCURRENCY=16
+# WEKNORA_WIKI_ASYNQ_CONCURRENCY=8
# Default per-model concurrency cap for background tasks (document ingest /
# enrichment). Shared across replicas, keyed by model ID. Only affects
diff --git a/docker-compose.yml b/docker-compose.yml
index bfcec553..33b7090e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -195,8 +195,12 @@ services:
- WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-}
- WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-}
- WEKNORA_TENANT_AUTO_CREATE_API_KEY=${WEKNORA_TENANT_AUTO_CREATE_API_KEY:-}
- - WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32}
- - WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-16}
+ - WEKNORA_ASYNQ_CORE_CONCURRENCY=${WEKNORA_ASYNQ_CORE_CONCURRENCY:-8}
+ - WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=${WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY:-2}
+ - WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=${WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY:-12}
+ - WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=${WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY:-4}
+ - WEKNORA_ASYNQ_SHARED_CONCURRENCY=${WEKNORA_ASYNQ_SHARED_CONCURRENCY:-6}
+ - WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-8}
- WEKNORA_MODEL_MAX_CONCURRENCY=${WEKNORA_MODEL_MAX_CONCURRENCY:-32}
- APK_MIRROR_ARG=${APK_MIRROR_ARG:-}
- WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-}
diff --git a/docs/worker-pool-governance.md b/docs/worker-pool-governance.md
new file mode 100644
index 00000000..c9343452
--- /dev/null
+++ b/docs/worker-pool-governance.md
@@ -0,0 +1,85 @@
+# Worker Pool Governance
+
+WeKnora uses guaranteed per-stage worker pools plus an elastic pool for the
+document ingestion pipeline. Worker concurrency is a scheduling budget, not a
+replacement for model quotas, DocReader capacity, vector-store limits, or
+database connection limits.
+
+## Topology
+
+| Pool | Default per instance | Queues | Purpose |
+| --- | ---: | --- | --- |
+| Core | 8 | `default` | Document parsing and manual reparse guarantee |
+| Post-process | 2 | `postprocess` | Parse finalization and enrichment fan-out |
+| Enrichment | 12 | `summary`, `multimodal`, `graph`, `question` | Model-heavy enrichment guarantee |
+| Maintenance | 4 | `sync`, `low` | Source sync, batch work, move/delete/cleanup |
+| Shared | 6 | Core and enrichment queues | Elastic capacity borrowed by the side with backlog |
+| Wiki | 8 | `wiki` | Independently governed Wiki generation |
+
+The upstream total remains 32 workers per service instance by default. Wiki is
+separate and is not included in that total.
+
+Post-process has its own physical queue so lightweight fan-out work cannot sit
+behind long DocReader calls. Maintenance is intentionally excluded from the
+shared pool because its long-running tasks could pin elastic capacity needed by
+the user-facing pipeline.
+
+Asynq dequeue is atomic. Dedicated and shared servers may safely subscribe to
+the same core/enrichment queues; one task is still processed by one worker.
+
+## Configuration
+
+All settings are available under System settings and require a service restart:
+
+- `asynq.core_concurrency` / `WEKNORA_ASYNQ_CORE_CONCURRENCY`
+- `asynq.postprocess_concurrency` / `WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY`
+- `asynq.enrichment_concurrency` / `WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY`
+- `asynq.maintenance_concurrency` / `WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY`
+- `asynq.shared_concurrency` / `WEKNORA_ASYNQ_SHARED_CONCURRENCY`
+- `asynq.wiki_concurrency` / `WEKNORA_WIKI_ASYNQ_CONCURRENCY`
+
+The old aggregate `asynq.concurrency` / `WEKNORA_ASYNQ_CONCURRENCY` setting is
+retired. Deployments that set it must migrate to the explicit pool settings.
+Persisted old rows are ignored and hidden from the System settings page so they
+cannot be mistaken for an effective runtime control.
+
+## Capacity layers
+
+Tune these layers independently:
+
+1. Worker concurrency controls the number of task handlers admitted per
+ service instance.
+2. Model quota governance controls provider concurrency, RPM, and TPM across
+ replicas and quota groups.
+3. DocReader, vector stores, object storage, Postgres, and local CPU/RAM retain
+ their own resource limits.
+
+If model limiter waiting grows while worker queues are busy, adding workers
+only creates more waiters. Increase the provider quota or reduce worker
+admission instead.
+
+## Sizing
+
+For each pool, estimate peak required workers using:
+
+```text
+required workers = ceil(peak task arrival rate * mean task runtime / 0.70)
+```
+
+Use task arrival rate after fan-out. One parsed document can create one summary,
+multiple question batches, one graph task per chunk, and multiple image tasks.
+Queue count is not a capacity signal.
+
+The runtime dashboard reports:
+
+- configured concurrency per instance;
+- live server instance count;
+- cluster capacity aggregated from active asynq heartbeats;
+- active workers and utilization;
+- queue backlog, retries, dead letters, and oldest pending age;
+- model concurrency and limiter waiting.
+
+Increase a pool only when backlog age grows while its downstream dependency has
+headroom. Reallocate to enrichment when core stays underutilized and fan-out
+queues grow. Reduce core admission when DocReader CPU, memory, or latency is the
+bottleneck.
diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts
index 1c7b5762..5b4d5871 100644
--- a/frontend/src/api/system/index.ts
+++ b/frontend/src/api/system/index.ts
@@ -507,6 +507,10 @@ export interface RuntimeWorkerPool {
name: string
concurrency: number
queue_count: number
+ instances: number
+ cluster_capacity: number
+ active: number
+ utilization: number
}
export interface ModelRuntimeStat {
@@ -520,9 +524,9 @@ export interface ModelRuntimeStat {
/**
* Runtime queue dashboard payload. `available` is false in Lite mode
* (no Redis/asynq) — render an "unavailable in this deployment" state
- * rather than an empty table. Pool concurrency values are CONFIGURED
- * per-process capacity (effective on next restart), not live busy-worker
- * count or cluster-wide capacity.
+ * rather than an empty table. Each pool includes both configured per-process
+ * concurrency and live cluster capacity/active workers aggregated from asynq
+ * server heartbeats.
*/
export interface RuntimeQueuesResponse {
available: boolean
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts
index fc5700b2..bae3de32 100755
--- a/frontend/src/i18n/locales/en-US.ts
+++ b/frontend/src/i18n/locales/en-US.ts
@@ -3496,11 +3496,14 @@ export default {
detailsTitle: "Queue details",
detailsDescription: "Live load and wait time for every processing lane.",
poolsTitle: "Worker pools",
- poolsDescription: "Core parsing, enrichment, maintenance, and Wiki use isolated concurrency budgets.",
- perInstance: "Concurrency is per process",
+ poolsDescription: "Each stage has guaranteed capacity; core and enrichment may also borrow the shared elastic pool.",
+ perInstance: "Card value is cluster active/capacity",
+ poolConfigured: "Per-process {value}",
+ poolInstances: "{value} instances",
+ poolUtilization: "{value}% utilized",
queueCount: "{value} queues",
weight: "Pool weight {value}",
- footnote: "Concurrency is a per-process cap (restart required); weight only controls scheduling within one pool; Active is cluster-wide.",
+ footnote: "Card values show cluster active/live capacity; per-process configuration requires restart. The shared elastic pool only consumes core and enrichment queues.",
updatedAt: "Updated at {value}",
errors: {
generic: "Failed to load queue stats",
@@ -3541,18 +3544,23 @@ export default {
},
pools: {
core: "Core parsing",
+ postprocess: "Post-process orchestration",
enrichment: "Enrichment",
maintenance: "Maintenance & sync",
+ shared: "Shared elastic",
wiki: "Wiki pool",
},
poolDescriptions: {
- core: "Document parsing and post-process orchestration",
+ core: "Guaranteed document and manual parsing capacity",
+ postprocess: "Parse finalization and enrichment fan-out",
enrichment: "Summaries, images, graph, and question generation",
maintenance: "Source sync, batch work, and deletion cleanup",
+ shared: "Borrowed by core or enrichment according to backlog",
wiki: "Wiki content generation and global finalization",
},
queueNames: {
default: "Default (document parse)",
+ postprocess: "Post-process orchestration",
summary: "Summary generation",
sync: "Source sync",
low: "Maintenance & batch jobs",
@@ -3562,7 +3570,8 @@ export default {
wiki: "Wiki ingest",
},
queueDescriptions: {
- default: "Document parsing, manual updates, and post-process orchestration",
+ default: "Document parsing and manual updates",
+ postprocess: "Parse finalization and enrichment task fan-out",
summary: "Document and data-table summaries",
sync: "Manual and scheduled data-source sync",
low: "FAQ import, clone/move, batch reparse, and deletion cleanup",
@@ -3584,7 +3593,11 @@ export default {
default_storage_quota_gb: 'Default storage quota for new tenants (GB)',
},
asynq: {
- concurrency: 'Total upstream worker budget',
+ core_concurrency: 'Guaranteed core parse concurrency',
+ postprocess_concurrency: 'Post-process concurrency',
+ enrichment_concurrency: 'Guaranteed enrichment concurrency',
+ maintenance_concurrency: 'Maintenance concurrency',
+ shared_concurrency: 'Shared elastic concurrency',
wiki_concurrency: 'Wiki worker concurrency',
},
model: {
@@ -3607,8 +3620,11 @@ export default {
'Default storage quota (GB) assigned when a new tenant is created, covering vectors, originals, text, indexes, and related data. Read only at creation time — changes apply to newly created tenants only and do not retroactively update existing tenants. 0 or a negative value uses the built-in default of 10 GB.',
},
asynq: {
- concurrency:
- 'Total per-process concurrency budget for upstream background work (excluding Wiki). It is split into isolated pools: 1/2 core parsing, 3/8 enrichment, and the remainder maintenance and sync. Minimum 3; requires a service restart.',
+ core_concurrency: 'Guaranteed per-process concurrency for document and manual parsing. Core may also borrow the shared elastic pool. Minimum 1; requires a service restart.',
+ postprocess_concurrency: 'Dedicated per-process concurrency for lightweight parse finalization and enrichment fan-out. Minimum 1; requires a service restart.',
+ enrichment_concurrency: 'Guaranteed per-process concurrency for summaries, images, graph, and question generation. Enrichment may also borrow the shared elastic pool. Minimum 1; requires a service restart.',
+ maintenance_concurrency: 'Per-process concurrency for source sync, batch work, and cleanup, hard-isolated from the user-facing pipeline. Minimum 1; requires a service restart.',
+ shared_concurrency: 'Per-process elastic concurrency shared by core parsing and enrichment; whichever side has backlog borrows it automatically. Minimum 1; requires a service restart.',
wiki_concurrency:
'Per-process concurrency for the dedicated Wiki worker pool, isolated from upstream tasks. Minimum 1; requires a service restart.',
},
diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts
index 33b0aa7d..09fe8140 100755
--- a/frontend/src/i18n/locales/ko-KR.ts
+++ b/frontend/src/i18n/locales/ko-KR.ts
@@ -2481,11 +2481,14 @@ export default {
detailsTitle: "큐 세부 정보",
detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.",
poolsTitle: "워커 풀",
- poolsDescription: "핵심 파싱, 보강, 유지 관리 및 Wiki는 독립된 동시성 예산을 사용합니다.",
- perInstance: "동시성은 프로세스별 설정",
+ poolsDescription: "각 단계에 보장 용량이 있으며 핵심 파싱과 보강은 공유 탄력 풀도 사용할 수 있습니다.",
+ perInstance: "카드 값은 클러스터 실행 중/용량",
+ poolConfigured: "프로세스별 {value}",
+ poolInstances: "인스턴스 {value}개",
+ poolUtilization: "사용률 {value}%",
queueCount: "큐 {value}개",
weight: "풀 내 가중치 {value}",
- footnote: "동시성은 프로세스별 상한이며 재시작 후 적용됩니다. 가중치는 같은 풀 안의 스케줄링 비율이며 실행 중 수치는 클러스터 전체입니다.",
+ footnote: "카드는 클러스터 실행 중/실시간 용량을 표시합니다. 프로세스별 설정은 재시작 후 적용되며 공유 풀은 핵심 파싱과 보강 큐만 처리합니다.",
updatedAt: "{value} 업데이트됨",
errors: {
generic: "큐 상태를 불러오지 못했습니다",
@@ -2526,18 +2529,23 @@ export default {
},
pools: {
core: "핵심 파싱",
+ postprocess: "후처리 오케스트레이션",
enrichment: "콘텐츠 보강",
maintenance: "유지 관리 및 동기화",
+ shared: "공유 탄력 풀",
wiki: "Wiki 풀",
},
poolDescriptions: {
- core: "문서 파싱 및 후처리 오케스트레이션",
+ core: "문서 및 수동 파싱 보장 용량",
+ postprocess: "파싱 마무리 및 보강 작업 분배",
enrichment: "요약, 이미지, 그래프 및 질문 생성",
maintenance: "데이터 소스 동기화, 일괄 작업 및 삭제 정리",
+ shared: "적체에 따라 핵심 파싱 또는 보강이 공유",
wiki: "Wiki 콘텐츠 생성 및 전체 마무리",
},
queueNames: {
default: "기본(문서 파싱)",
+ postprocess: "후처리 오케스트레이션",
summary: "요약 생성",
sync: "데이터 소스 동기화",
low: "백그라운드 유지 관리 및 일괄 작업",
@@ -2547,7 +2555,8 @@ export default {
wiki: "Wiki 동기화",
},
queueDescriptions: {
- default: "문서 파싱, 수동 업데이트 및 후처리 오케스트레이션",
+ default: "문서 파싱 및 수동 업데이트",
+ postprocess: "파싱 마무리 및 보강 작업 분배",
summary: "문서 및 데이터 테이블 요약",
sync: "수동 및 예약 데이터 소스 동기화",
low: "FAQ 가져오기, 복제/이동, 일괄 재파싱 및 삭제 정리",
@@ -2569,7 +2578,11 @@ export default {
default_storage_quota_gb: "신규 테넌트 기본 저장 용량 (GB)",
},
asynq: {
- concurrency: "상위 워커 총 동시성 예산",
+ core_concurrency: "핵심 파싱 보장 동시성",
+ postprocess_concurrency: "후처리 동시성",
+ enrichment_concurrency: "콘텐츠 보강 보장 동시성",
+ maintenance_concurrency: "유지 관리 동시성",
+ shared_concurrency: "공유 탄력 동시성",
wiki_concurrency: "Wiki 워커 동시성",
},
model: {
@@ -2592,8 +2605,11 @@ export default {
"신규 테넌트 생성 시 기본으로 할당되는 저장 용량(GB)으로, 벡터·원본·텍스트·인덱스 등을 포함합니다. 생성 시에만 읽으며, 변경은 이후 생성되는 테넌트에만 적용되고 기존 테넌트에는 소급되지 않습니다. 0 또는 음수는 내장 기본값 10GB를 사용합니다.",
},
asynq: {
- concurrency:
- "Wiki를 제외한 상위 백그라운드 작업의 프로세스별 총 동시성 예산입니다. 핵심 파싱 1/2, 콘텐츠 보강 3/8, 나머지는 유지 관리 및 동기화로 분리됩니다. 최소값은 3이며 적용하려면 서비스를 재시작해야 합니다.",
+ core_concurrency: "문서 및 수동 파싱의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.",
+ postprocess_concurrency: "파싱 마무리 및 보강 작업 분배 전용 프로세스별 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.",
+ enrichment_concurrency: "요약, 이미지, 그래프 및 질문 생성의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.",
+ maintenance_concurrency: "동기화, 일괄 작업 및 정리의 프로세스별 동시성으로 사용자 파이프라인과 격리됩니다. 최소 1, 서비스 재시작이 필요합니다.",
+ shared_concurrency: "핵심 파싱과 콘텐츠 보강이 적체에 따라 공유하는 프로세스별 탄력 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.",
wiki_concurrency:
"상위 작업과 분리된 전용 Wiki 워커 풀의 프로세스별 동시성입니다. 최소값은 1이며 적용하려면 서비스를 재시작해야 합니다.",
},
diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts
index e54543cc..096459f1 100755
--- a/frontend/src/i18n/locales/ru-RU.ts
+++ b/frontend/src/i18n/locales/ru-RU.ts
@@ -2179,11 +2179,14 @@ export default {
detailsTitle: "Сведения об очередях",
detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.",
poolsTitle: "Пулы воркеров",
- poolsDescription: "Основной разбор, обогащение, обслуживание и Wiki используют независимые бюджеты параллелизма.",
- perInstance: "Параллелизм указан на процесс",
+ poolsDescription: "Каждый этап имеет гарантированную ёмкость; разбор и обогащение также используют общий эластичный пул.",
+ perInstance: "На карточке активно/ёмкость кластера",
+ poolConfigured: "На процесс: {value}",
+ poolInstances: "Экземпляров: {value}",
+ poolUtilization: "Загрузка {value}%",
queueCount: "Очередей: {value}",
weight: "Вес в пуле {value}",
- footnote: "Параллелизм указан на процесс и требует перезапуска; вес действует только внутри пула; активные задачи показаны по всему кластеру.",
+ footnote: "Карточки показывают активные задачи/живую ёмкость кластера. Настройки на процесс требуют перезапуска; общий пул обслуживает только разбор и обогащение.",
updatedAt: "Обновлено в {value}",
errors: {
generic: "Не удалось загрузить состояние очередей",
@@ -2224,18 +2227,23 @@ export default {
},
pools: {
core: "Основной разбор",
+ postprocess: "Оркестрация постобработки",
enrichment: "Обогащение",
maintenance: "Обслуживание и синхронизация",
+ shared: "Общий эластичный пул",
wiki: "Пул Wiki",
},
poolDescriptions: {
- core: "Разбор документов и оркестрация постобработки",
+ core: "Гарантированная ёмкость разбора документов",
+ postprocess: "Завершение разбора и запуск обогащения",
enrichment: "Сводки, изображения, граф и генерация вопросов",
maintenance: "Синхронизация источников, пакетные задачи и удаление",
+ shared: "Используется разбором или обогащением по мере очереди",
wiki: "Создание содержимого Wiki и финальная обработка",
},
queueNames: {
default: "По умолчанию (парсинг документов)",
+ postprocess: "Оркестрация постобработки",
summary: "Создание сводок",
sync: "Синхронизация источников",
low: "Фоновое обслуживание и пакетные задачи",
@@ -2245,7 +2253,8 @@ export default {
wiki: "Синхронизация Wiki",
},
queueDescriptions: {
- default: "Разбор документов, ручные обновления и оркестрация постобработки",
+ default: "Разбор документов и ручные обновления",
+ postprocess: "Завершение разбора и запуск задач обогащения",
summary: "Сводки документов и таблиц данных",
sync: "Ручная и плановая синхронизация источников",
low: "Импорт FAQ, клонирование/перемещение, пакетный повторный разбор и удаление",
@@ -2267,7 +2276,11 @@ export default {
default_storage_quota_gb: 'Квота хранилища для новых тенантов по умолчанию (ГБ)',
},
asynq: {
- concurrency: 'Общий бюджет фоновых воркеров',
+ core_concurrency: 'Гарантированный параллелизм разбора',
+ postprocess_concurrency: 'Параллелизм постобработки',
+ enrichment_concurrency: 'Гарантированный параллелизм обогащения',
+ maintenance_concurrency: 'Параллелизм обслуживания',
+ shared_concurrency: 'Общий эластичный параллелизм',
wiki_concurrency: 'Параллелизм воркеров Wiki',
},
model: {
@@ -2290,8 +2303,11 @@ export default {
'Квота хранилища по умолчанию (ГБ) для нового тенанта: векторы, оригиналы, текст, индексы и т.д. Читается только при создании — изменения применяются только к новым тенантам и не перезаписывают существующие. 0 или отрицательное значение — встроенное значение по умолчанию 10 ГБ.',
},
asynq: {
- concurrency:
- 'Общий бюджет параллелизма фоновых задач на процесс (без Wiki). Он делится на изолированные пулы: 1/2 основной разбор, 3/8 обогащение, остаток — обслуживание и синхронизация. Минимум 3; требуется перезапуск сервиса.',
+ core_concurrency: 'Гарантированный параллелизм разбора документов на процесс; также может использовать общий эластичный пул. Минимум 1; требуется перезапуск.',
+ postprocess_concurrency: 'Выделенный параллелизм завершения разбора и запуска обогащения на процесс. Минимум 1; требуется перезапуск.',
+ enrichment_concurrency: 'Гарантированный параллелизм сводок, изображений, графа и вопросов на процесс; также может использовать общий пул. Минимум 1; требуется перезапуск.',
+ maintenance_concurrency: 'Параллелизм синхронизации, пакетных операций и очистки на процесс, изолированный от пользовательского конвейера. Минимум 1; требуется перезапуск.',
+ shared_concurrency: 'Эластичный параллелизм на процесс, общий для разбора и обогащения и используемый стороной с очередью. Минимум 1; требуется перезапуск.',
wiki_concurrency:
'Параллелизм выделенного пула Wiki на процесс, изолированного от остальных задач. Минимум 1; требуется перезапуск сервиса.',
},
diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts
index c51fd234..65c3779f 100755
--- a/frontend/src/i18n/locales/zh-CN.ts
+++ b/frontend/src/i18n/locales/zh-CN.ts
@@ -2484,11 +2484,14 @@ export default {
detailsTitle: "队列明细",
detailsDescription: "各处理通道的实时负载与等待情况。",
poolsTitle: "Worker 池",
- poolsDescription: "核心解析、内容富化、维护同步和 Wiki 使用相互隔离的并发预算。",
- perInstance: "并发数为单实例配置",
+ poolsDescription: "各阶段拥有保底容量,核心解析与内容富化还可借用共享弹性池。",
+ perInstance: "卡片主值为集群运行中/容量",
+ poolConfigured: "单实例配置 {value}",
+ poolInstances: "{value} 个实例",
+ poolUtilization: "利用率 {value}%",
queueCount: "{value} 个队列",
weight: "池内权重 {value}",
- footnote: "Worker 并发数为单实例上限(修改后需重启);权重只决定同一池内的调度份额;「运行中」为集群当前任务数。",
+ footnote: "卡片主值为集群运行中/实时容量;单实例配置修改后需重启。共享弹性池只消费核心解析和内容富化队列。",
updatedAt: "更新于 {value}",
errors: {
generic: "获取队列状态失败",
@@ -2529,18 +2532,23 @@ export default {
},
pools: {
core: "核心解析",
+ postprocess: "后处理编排",
enrichment: "内容富化",
maintenance: "维护与同步",
+ shared: "共享弹性",
wiki: "Wiki 池",
},
poolDescriptions: {
- core: "文档解析与后处理调度",
+ core: "文档解析与手工重解析的保底容量",
+ postprocess: "解析完成后的收尾与富化扇出",
enrichment: "摘要、图片、图谱与问题生成",
maintenance: "数据源同步、批处理与删除清理",
+ shared: "由核心解析与内容富化按积压借用",
wiki: "Wiki 内容生成与全局收尾",
},
queueNames: {
default: "默认(文档解析)",
+ postprocess: "后处理编排",
summary: "摘要生成",
sync: "数据源同步",
low: "后台维护与批处理",
@@ -2550,7 +2558,8 @@ export default {
wiki: "Wiki 同步",
},
queueDescriptions: {
- default: "文档解析、手工更新与后处理调度",
+ default: "文档解析与手工更新",
+ postprocess: "解析完成后的状态收尾与富化任务扇出",
summary: "文档摘要与表格摘要",
sync: "手动与定时数据源同步",
low: "FAQ 导入、复制移动、批量重解析与删除清理",
@@ -2572,7 +2581,11 @@ export default {
default_storage_quota_gb: "新租户默认存储配额 (GB)",
},
asynq: {
- concurrency: "上游 Worker 总并发预算",
+ core_concurrency: "核心解析保底并发数",
+ postprocess_concurrency: "后处理编排并发数",
+ enrichment_concurrency: "内容富化保底并发数",
+ maintenance_concurrency: "维护与同步并发数",
+ shared_concurrency: "共享弹性并发数",
wiki_concurrency: "Wiki Worker 并发数",
},
model: {
@@ -2600,10 +2613,11 @@ export default {
"0 或负数表示使用内置默认值 10GB。",
},
asynq: {
- concurrency:
- "每个服务实例用于上游后台任务的总并发预算(不含 Wiki)。系统按核心解析 1/2、" +
- "内容富化 3/8、维护与同步为剩余容量,拆分成相互隔离的 Worker 池。" +
- "最小值为 3;修改后需重启服务进程方可生效。",
+ core_concurrency: "文档解析与手工重解析的每实例保底并发,可额外借用共享弹性池;修改后需重启。",
+ postprocess_concurrency: "解析完成后的轻量收尾和富化扇出专用并发;修改后需重启。",
+ enrichment_concurrency: "摘要、图片、图谱与问题生成的每实例保底并发,可额外借用共享弹性池;修改后需重启。",
+ maintenance_concurrency: "数据源同步、批处理和清理任务的每实例并发,与用户面流水线硬隔离;修改后需重启。",
+ shared_concurrency: "核心解析与内容富化共同使用的每实例弹性并发,由有积压的一侧自动借用;修改后需重启。",
wiki_concurrency:
"每个服务实例的 Wiki 专用 Worker 并发数,与上游任务池相互隔离。" +
"最小值为 1;修改后需重启服务进程方可生效。",
diff --git a/frontend/src/views/system/RuntimeQueues.vue b/frontend/src/views/system/RuntimeQueues.vue
index 9dfbf7e7..f11bb155 100644
--- a/frontend/src/views/system/RuntimeQueues.vue
+++ b/frontend/src/views/system/RuntimeQueues.vue
@@ -106,12 +106,19 @@
{{ poolLabel(pool.name) }}
- {{ pool.concurrency }}
+
+ {{ pool.instances > 0 ? `${pool.active}/${pool.cluster_capacity}` : pool.concurrency }}
+
{{ poolDescription(pool.name) }}
- {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}
+ {{ t('system.globalSettings.runtime.poolConfigured', { value: pool.concurrency }) }}
+
+ · {{ t('system.globalSettings.runtime.poolInstances', { value: pool.instances }) }}
+ · {{ t('system.globalSettings.runtime.poolUtilization', { value: poolUtilization(pool) }) }}
+
+ · {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}
@@ -310,6 +317,10 @@ function poolQueueCount(pool: string): number {
return pools.value.find((item) => item.name === pool)?.queue_count ?? 0
}
+function poolUtilization(pool: RuntimeWorkerPool): number {
+ return Math.round(Math.max(0, Math.min(1, pool.utilization || 0)) * 100)
+}
+
function formatLatency(ms: number): string {
if (!ms || ms <= 0) return '—'
if (ms < 1000) return `${ms} ms`
diff --git a/frontend/src/views/system/SystemSettings.vue b/frontend/src/views/system/SystemSettings.vue
index ea110765..adc7b8e5 100644
--- a/frontend/src/views/system/SystemSettings.vue
+++ b/frontend/src/views/system/SystemSettings.vue
@@ -761,8 +761,7 @@ function placeholderFor(item: SystemSettingItem): string {
}
function minimumFor(item: SystemSettingItem): number {
- if (item.key === 'asynq.concurrency') return 3
- if (item.key === 'asynq.wiki_concurrency') return 1
+ if (item.key.startsWith('asynq.') && item.key.endsWith('_concurrency')) return 1
return 0
}
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index 47adc444..fa187e71 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -626,7 +626,7 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con
}
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
- asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
+ knowledgePostProcessTaskOptions()...)
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
} else {
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index 386efb64..02995b63 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -186,8 +186,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
if queueSkipped > 0 {
// Visibility into "stale span heartbeat but tasks still queued"
// — i.e. backpressure, not a stuck row. Persistent counts here
- // mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY
- // or document_process_timeout), not that housekeeping misfires.
+ // mean the queue is the bottleneck (raise the matching per-pool or
+ // shared asynq concurrency, or document_process_timeout), not that
+ // housekeeping misfires.
logger.Infof(ctx,
"[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)",
queueSkipped)
diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go
index 368b565a..f2c483c7 100644
--- a/internal/application/service/knowledge_housekeeping_test.go
+++ b/internal/application/service/knowledge_housekeeping_test.go
@@ -126,6 +126,12 @@ func (f fakeTaskInspector) QueueStats(
return nil, false, nil
}
+func (f fakeTaskInspector) WorkerServerStats(
+ _ context.Context,
+) ([]types.WorkerServerStat, bool, error) {
+ return nil, false, nil
+}
+
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
return newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index ebe3feda..ccbc192a 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -675,7 +675,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
payloadBytes, err := json.Marshal(postProcessPayload)
if err == nil {
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
- asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
+ knowledgePostProcessTaskOptions()...)
if _, err := s.task.Enqueue(task); err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
} else {
diff --git a/internal/application/service/knowledge_task_options.go b/internal/application/service/knowledge_task_options.go
index f65aca3c..9d7a1ae3 100644
--- a/internal/application/service/knowledge_task_options.go
+++ b/internal/application/service/knowledge_task_options.go
@@ -1,6 +1,8 @@
package service
import (
+ "time"
+
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
@@ -15,3 +17,11 @@ func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asy
opts = append(opts, extra...)
return opts
}
+
+func knowledgePostProcessTaskOptions() []asynq.Option {
+ return []asynq.Option{
+ asynq.Queue(types.QueuePostProcess),
+ asynq.MaxRetry(3),
+ asynq.Timeout(30 * time.Minute),
+ }
+}
diff --git a/internal/application/service/knowledge_task_options_test.go b/internal/application/service/knowledge_task_options_test.go
index 1b049c55..8a60543c 100644
--- a/internal/application/service/knowledge_task_options_test.go
+++ b/internal/application/service/knowledge_task_options_test.go
@@ -79,3 +79,12 @@ func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) {
require.NotNil(t, maxRetry)
assert.Equal(t, 3, *maxRetry)
}
+
+func TestKnowledgePostProcessTaskOptionsUseDedicatedQueue(t *testing.T) {
+ t.Parallel()
+ queue, timeout, maxRetry := parseDocumentProcessOpts(t, knowledgePostProcessTaskOptions())
+ assert.Equal(t, types.QueuePostProcess, queue)
+ assert.Equal(t, 30*time.Minute, timeout)
+ require.NotNil(t, maxRetry)
+ assert.Equal(t, 3, *maxRetry)
+}
diff --git a/internal/application/service/system_setting.go b/internal/application/service/system_setting.go
index 5b2779ee..d596d0d6 100644
--- a/internal/application/service/system_setting.go
+++ b/internal/application/service/system_setting.go
@@ -183,24 +183,50 @@ var registry = map[string]settingSpec{
"用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" +
"每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。",
},
- // asynq.concurrency is the total per-process upstream worker budget.
- // It is divided deterministically across independent core, enrichment,
- // and maintenance servers. Read once at startup; changing it requires a
- // process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32).
- "asynq.concurrency": {
+ "asynq.core_concurrency": {
Type: "int",
- EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
- Default: int64(types.DefaultUpstreamWorkerConcurrency),
+ EnvName: "WEKNORA_ASYNQ_CORE_CONCURRENCY",
+ Default: int64(types.DefaultCoreWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
- Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" +
- "内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" +
- "最小值为 3;修改后需重启服务进程方可生效。",
+ Description: "文档解析、手工重解析等核心任务的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
+ },
+ "asynq.postprocess_concurrency": {
+ Type: "int",
+ EnvName: "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY",
+ Default: int64(types.DefaultPostProcessWorkerConcurrency),
+ Category: "worker",
+ RequiresRestart: true,
+ Description: "解析完成后的轻量编排与富化扇出专用并发,避免被长时间文档解析阻塞;修改后需重启。",
+ },
+ "asynq.enrichment_concurrency": {
+ Type: "int",
+ EnvName: "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY",
+ Default: int64(types.DefaultEnrichmentWorkerConcurrency),
+ Category: "worker",
+ RequiresRestart: true,
+ Description: "摘要、图片、图谱和问题生成的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
+ },
+ "asynq.maintenance_concurrency": {
+ Type: "int",
+ EnvName: "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY",
+ Default: int64(types.DefaultMaintenanceWorkerConcurrency),
+ Category: "worker",
+ RequiresRestart: true,
+ Description: "数据源同步、批处理、移动和删除清理的每实例保底并发,与用户面流水线硬隔离;修改后需重启。",
+ },
+ "asynq.shared_concurrency": {
+ Type: "int",
+ EnvName: "WEKNORA_ASYNQ_SHARED_CONCURRENCY",
+ Default: int64(types.DefaultSharedWorkerConcurrency),
+ Category: "worker",
+ RequiresRestart: true,
+ Description: "核心解析与内容富化共用的每实例弹性并发。空闲容量由有积压的一侧借用;修改后需重启。",
},
// asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool,
- // separate from asynq.concurrency. Read once when the wiki asynq server
+ // separate from the upstream pools. Read once when the wiki asynq server
// starts — changing it in the UI requires a process restart. Mirrors
- // WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16).
+ // WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 8).
"asynq.wiki_concurrency": {
Type: "int",
EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY",
@@ -669,6 +695,10 @@ func (s *systemSettingService) List(ctx context.Context) ([]*types.SystemSetting
for _, row := range rows {
byKey[row.Key] = row
}
+ // asynq.concurrency was the old fixed-ratio aggregate. Keeping an old DB
+ // row visible would suggest it still controls runtime capacity, so retire it
+ // explicitly while preserving all genuinely unknown rows for diagnostics.
+ delete(byKey, "asynq.concurrency")
keys := make([]string, 0, len(registry))
for key := range registry {
@@ -1228,17 +1258,15 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
// 400 body verbatim).
func validateRegistryEntry(key string, rawValue any) error {
switch key {
- case "asynq.concurrency", "asynq.wiki_concurrency":
+ case "asynq.core_concurrency", "asynq.postprocess_concurrency",
+ "asynq.enrichment_concurrency", "asynq.maintenance_concurrency",
+ "asynq.shared_concurrency", "asynq.wiki_concurrency":
n, err := coerceToPositiveInt64(rawValue)
if err != nil {
return err
}
- minimum := int64(1)
- if key == "asynq.concurrency" {
- minimum = 3
- }
- if n < minimum {
- return fmt.Errorf("concurrency must be at least %d", minimum)
+ if n < 1 {
+ return errors.New("concurrency must be at least 1")
}
case "ssrf.whitelist":
// Coerce into the same shape encodeForType produced. We don't
diff --git a/internal/application/service/system_setting_validation_test.go b/internal/application/service/system_setting_validation_test.go
index 17ea5232..0523e6b2 100644
--- a/internal/application/service/system_setting_validation_test.go
+++ b/internal/application/service/system_setting_validation_test.go
@@ -9,9 +9,12 @@ func TestValidateWorkerConcurrencyMinimums(t *testing.T) {
value any
wantErr bool
}{
- {name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true},
- {name: "upstream minimum", key: "asynq.concurrency", value: 3},
- {name: "upstream normal", key: "asynq.concurrency", value: 32},
+ {name: "core zero", key: "asynq.core_concurrency", value: 0, wantErr: true},
+ {name: "core minimum", key: "asynq.core_concurrency", value: 1},
+ {name: "postprocess minimum", key: "asynq.postprocess_concurrency", value: 1},
+ {name: "enrichment minimum", key: "asynq.enrichment_concurrency", value: 1},
+ {name: "maintenance minimum", key: "asynq.maintenance_concurrency", value: 1},
+ {name: "shared minimum", key: "asynq.shared_concurrency", value: 1},
{name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true},
{name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1},
}
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index b30ae0a3..e69346b2 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -79,7 +79,7 @@ const (
// wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}.
wikiInflightPrefix = "wiki:inflight:"
// wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight
- // is unset. 4 leaves headroom in a default 16-worker wiki pool for other
+ // is unset. 4 leaves half of the default 8-worker wiki pool for other
// KBs while still giving one KB solid parallelism.
wikiInflightDefault = 4
// wikiInflightTTL is how long a reserved slot survives without renewal.
diff --git a/internal/container/container.go b/internal/container/container.go
index 0d5c9052..558b8677 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -264,13 +264,14 @@ func BuildContainer(container *dig.Container) *dig.Container {
redisAvailable := os.Getenv("REDIS_ADDR") != ""
if redisAvailable {
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
- // Independent worker pools keep user-facing parsing, high-fanout
- // enrichment, maintenance/sync, and Wiki generation from consuming one
- // another's capacity. All pools share one handler mux but subscribe to
- // disjoint queues from the central topology registry.
+ // Dedicated pools guarantee capacity for each stage. The shared pool
+ // additionally subscribes to core/enrichment queues to provide elastic
+ // borrowing while post-process and maintenance remain hard-isolated.
must(container.Provide(router.NewCoreAsynqServer, dig.Name("coreAsynqServer")))
+ must(container.Provide(router.NewPostProcessAsynqServer, dig.Name("postProcessAsynqServer")))
must(container.Provide(router.NewEnrichmentAsynqServer, dig.Name("enrichmentAsynqServer")))
must(container.Provide(router.NewMaintenanceAsynqServer, dig.Name("maintenanceAsynqServer")))
+ must(container.Provide(router.NewSharedAsynqServer, dig.Name("sharedAsynqServer")))
must(container.Provide(router.NewWikiAsynqServer, dig.Name("wikiAsynqServer")))
// Asynq inspector for cancel-by-knowledge-id (best-effort
// dequeue of pending/scheduled/retry tasks + active-task cancel).
diff --git a/internal/handler/system.go b/internal/handler/system.go
index d57fac3f..32de90cd 100644
--- a/internal/handler/system.go
+++ b/internal/handler/system.go
@@ -1385,13 +1385,16 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
// UI can render an "unavailable in this deployment" state instead of an
-// empty table. Pool concurrency is the configured per-process capacity
-// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
-// capacity (see QueueStat.Active for currently running tasks).
+// empty table. Each pool reports configured per-process concurrency plus live
+// cluster capacity/active workers aggregated from asynq server heartbeats.
type RuntimeWorkerPool struct {
- Name string `json:"name"`
- Concurrency int `json:"concurrency"`
- QueueCount int `json:"queue_count"`
+ Name string `json:"name"`
+ Concurrency int `json:"concurrency"` // configured per instance
+ QueueCount int `json:"queue_count"`
+ Instances int `json:"instances"` // live asynq server heartbeats
+ ClusterCapacity int `json:"cluster_capacity"` // sum of live server concurrency
+ Active int `json:"active"` // live workers assigned to this pool
+ Utilization float64 `json:"utilization"` // Active / ClusterCapacity
}
type RuntimeQueuesResponse struct {
@@ -1406,6 +1409,56 @@ type RuntimeQueuesResponse struct {
Timestamp int64 `json:"timestamp"`
}
+func aggregateRuntimeWorkerPools(pools []RuntimeWorkerPool, servers []types.WorkerServerStat) {
+ queueConfigs := map[string]map[string]int{
+ types.WorkerPoolCore: types.QueueWeightsForPool(types.WorkerPoolCore),
+ types.WorkerPoolPostProcess: types.QueueWeightsForPool(types.WorkerPoolPostProcess),
+ types.WorkerPoolEnrichment: types.QueueWeightsForPool(types.WorkerPoolEnrichment),
+ types.WorkerPoolMaintenance: types.QueueWeightsForPool(types.WorkerPoolMaintenance),
+ types.WorkerPoolShared: types.QueueWeightsForSharedPool(),
+ types.WorkerPoolWiki: types.QueueWeightsForPool(types.WorkerPoolWiki),
+ }
+ indexes := make(map[string]int, len(pools))
+ for i := range pools {
+ indexes[pools[i].Name] = i
+ }
+ for _, server := range servers {
+ if server.Status != "active" {
+ continue
+ }
+ for poolName, expectedQueues := range queueConfigs {
+ if !sameQueueWeights(server.Queues, expectedQueues) {
+ continue
+ }
+ idx, ok := indexes[poolName]
+ if !ok {
+ break
+ }
+ pools[idx].Instances++
+ pools[idx].ClusterCapacity += server.Concurrency
+ pools[idx].Active += server.Active
+ break
+ }
+ }
+ for i := range pools {
+ if pools[i].ClusterCapacity > 0 {
+ pools[i].Utilization = float64(pools[i].Active) / float64(pools[i].ClusterCapacity)
+ }
+ }
+}
+
+func sameQueueWeights(left, right map[string]int) bool {
+ if len(left) != len(right) {
+ return false
+ }
+ for name, weight := range left {
+ if right[name] != weight {
+ return false
+ }
+ }
+ return true
+}
+
// GetRuntimeQueues godoc
// @Summary 获取解析任务队列运行时状态
// @Description 返回各 asynq 队列的实时深度(pending/active/scheduled/retry 等)与 worker 并发配置,仅系统管理员可见
@@ -1415,34 +1468,30 @@ type RuntimeQueuesResponse struct {
// @Router /system/admin/runtime/queues [get]
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
ctx := logger.CloneContext(c.Request.Context())
- upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
- wikiConcurrency := types.DefaultWikiWorkerConcurrency
+ var allocation types.WorkerPoolConcurrency
if h.systemSettingSvc != nil {
- if configured := h.systemSettingSvc.GetInt(
- ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
- ); configured > 0 {
- upstreamConcurrency = int(configured)
- }
- if configured := h.systemSettingSvc.GetInt(
- ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
- ); configured > 0 {
- wikiConcurrency = int(configured)
- }
+ allocation = types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int {
+ configured := h.systemSettingSvc.GetInt(ctx, key, env, int64(fallback))
+ return int(configured)
+ })
+ } else {
+ allocation = types.DefaultWorkerPoolConcurrency()
}
- allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
queueCounts := make(map[string]int)
for _, definition := range types.QueueDefinitions() {
queueCounts[definition.Pool]++
}
resp := RuntimeQueuesResponse{
- UpstreamConcurrency: allocation.Total,
- ParseConcurrency: allocation.Total,
- WikiConcurrency: wikiConcurrency,
+ UpstreamConcurrency: allocation.UpstreamTotal(),
+ ParseConcurrency: allocation.UpstreamTotal(),
+ WikiConcurrency: allocation.Wiki,
Pools: []RuntimeWorkerPool{
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
+ {Name: types.WorkerPoolPostProcess, Concurrency: allocation.PostProcess, QueueCount: queueCounts[types.WorkerPoolPostProcess]},
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
- {Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
+ {Name: types.WorkerPoolShared, Concurrency: allocation.Shared, QueueCount: len(types.QueueWeightsForSharedPool())},
+ {Name: types.WorkerPoolWiki, Concurrency: allocation.Wiki, QueueCount: queueCounts[types.WorkerPoolWiki]},
},
Timestamp: time.Now().Unix(),
}
@@ -1456,6 +1505,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
if stats != nil {
resp.Queues = stats
}
+ serverStats, _, serverErr := h.taskInspector.WorkerServerStats(ctx)
+ if serverErr != nil {
+ logger.Errorf(ctx, "get worker server stats failed: %v", serverErr)
+ } else {
+ aggregateRuntimeWorkerPools(resp.Pools, serverStats)
+ }
}
// Always emit a non-nil array so the JSON serialises to `[]` rather
// than `null`, keeping the frontend iteration simple.
diff --git a/internal/handler/system_runtime_test.go b/internal/handler/system_runtime_test.go
index 8b145f34..2ddd91c5 100644
--- a/internal/handler/system_runtime_test.go
+++ b/internal/handler/system_runtime_test.go
@@ -15,10 +15,18 @@ type runtimeTestSettings struct{}
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
switch key {
- case "asynq.concurrency":
- return 32
+ case "asynq.core_concurrency":
+ return 8
+ case "asynq.postprocess_concurrency":
+ return 2
+ case "asynq.enrichment_concurrency":
+ return 12
+ case "asynq.maintenance_concurrency":
+ return 4
+ case "asynq.shared_concurrency":
+ return 6
case "asynq.wiki_concurrency":
- return 16
+ return 8
default:
return def
}
@@ -55,6 +63,17 @@ func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string)
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
return []types.QueueStat{}, true, nil
}
+func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
+ return []types.WorkerServerStat{
+ {Concurrency: 8, Active: 4, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
+ {Concurrency: 2, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolPostProcess)},
+ {Concurrency: 12, Active: 6, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolEnrichment)},
+ {Concurrency: 4, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolMaintenance)},
+ {Concurrency: 6, Active: 3, Status: "active", Queues: types.QueueWeightsForSharedPool()},
+ {Concurrency: 8, Active: 2, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolWiki)},
+ {Concurrency: 99, Active: 0, Status: "stopped", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
+ }, true, nil
+}
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -85,10 +104,12 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
concurrency int
queueCount int
}{
- types.WorkerPoolCore: {16, 1},
+ types.WorkerPoolCore: {8, 1},
+ types.WorkerPoolPostProcess: {2, 1},
types.WorkerPoolEnrichment: {12, 4},
types.WorkerPoolMaintenance: {4, 2},
- types.WorkerPoolWiki: {16, 1},
+ types.WorkerPoolShared: {6, 5},
+ types.WorkerPoolWiki: {8, 1},
}
if len(response.Pools) != len(want) {
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
@@ -102,6 +123,9 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
pool.Name, pool, expected.concurrency, expected.queueCount)
}
+ if pool.Instances != 1 || pool.ClusterCapacity != expected.concurrency {
+ t.Fatalf("pool %q live capacity = %+v", pool.Name, pool)
+ }
}
}
diff --git a/internal/router/task.go b/internal/router/task.go
index 5ec1cd73..333807d4 100644
--- a/internal/router/task.go
+++ b/internal/router/task.go
@@ -24,13 +24,15 @@ import (
type AsynqTaskParams struct {
dig.In
- // Four independent servers provide hard capacity isolation between the
- // user-facing parse path, high-fanout enrichment, maintenance/sync work,
- // and Wiki generation. They share one handler mux but subscribe to disjoint
- // queue sets from types.QueueDefinitions.
+ // Dedicated servers provide minimum capacity for parse, post-process,
+ // enrichment, maintenance, and Wiki work. SharedServer is the elastic tier:
+ // it also subscribes to core/enrichment queues so either stage can borrow
+ // otherwise-idle capacity without consuming the other stage's guarantee.
CoreServer *asynq.Server `name:"coreAsynqServer"`
+ PostProcessServer *asynq.Server `name:"postProcessAsynqServer"`
EnrichmentServer *asynq.Server `name:"enrichmentAsynqServer"`
MaintenanceServer *asynq.Server `name:"maintenanceAsynqServer"`
+ SharedServer *asynq.Server `name:"sharedAsynqServer"`
WikiServer *asynq.Server `name:"wikiAsynqServer"`
KnowledgeService interfaces.KnowledgeService
KnowledgeBaseService interfaces.KnowledgeBaseService
@@ -151,32 +153,41 @@ func backgroundTaskMiddleware() asynq.MiddlewareFunc {
}
}
-func resolveUpstreamPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency {
- total := types.DefaultUpstreamWorkerConcurrency
- if svc != nil {
- n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency)
- if n > 0 {
- total = int(n)
- }
+func resolveWorkerPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency {
+ if svc == nil {
+ return types.DefaultWorkerPoolConcurrency()
}
- return types.AllocateWorkerPoolConcurrency(total)
+ ctx := context.Background()
+ return types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int {
+ value := svc.GetInt(ctx, key, env, int64(fallback))
+ return int(value)
+ })
}
-// NewCoreAsynqServer runs the latency-sensitive document parse and
-// post-process orchestration path.
+// NewCoreAsynqServer runs document and manual parsing with guaranteed capacity.
func NewCoreAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
- allocation := resolveUpstreamPoolConcurrency(svc)
+ allocation := resolveWorkerPoolConcurrency(svc)
log.Printf("asynq core-pool server starting with concurrency=%d total_upstream=%d redis_op_timeout=%dms",
- allocation.Core, allocation.Total, readRedisOpTimeoutMs())
+ allocation.Core, allocation.UpstreamTotal(), readRedisOpTimeoutMs())
return newAsynqServer(allocation.Core, types.QueueWeightsForPool(types.WorkerPoolCore))
}
+// NewPostProcessAsynqServer reserves capacity for the lightweight but
+// latency-sensitive fan-out stage. It cannot be trapped behind a burst of
+// long-running document parses in QueueDefault.
+func NewPostProcessAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
+ allocation := resolveWorkerPoolConcurrency(svc)
+ log.Printf("asynq postprocess-pool server starting with concurrency=%d total_upstream=%d",
+ allocation.PostProcess, allocation.UpstreamTotal())
+ return newAsynqServer(allocation.PostProcess, types.QueueWeightsForPool(types.WorkerPoolPostProcess))
+}
+
// NewEnrichmentAsynqServer runs high-fanout summary, multimodal, graph, and
// question generation without consuming core parsing capacity.
func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
- allocation := resolveUpstreamPoolConcurrency(svc)
+ allocation := resolveWorkerPoolConcurrency(svc)
log.Printf("asynq enrichment-pool server starting with concurrency=%d total_upstream=%d",
- allocation.Enrichment, allocation.Total)
+ allocation.Enrichment, allocation.UpstreamTotal())
return newAsynqServer(allocation.Enrichment, types.QueueWeightsForPool(types.WorkerPoolEnrichment))
}
@@ -184,25 +195,29 @@ func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server
// dispatch work. QueueMaintenance keeps the legacy Redis name "low", so old
// tasks are drained safely during rolling upgrades.
func NewMaintenanceAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
- allocation := resolveUpstreamPoolConcurrency(svc)
+ allocation := resolveWorkerPoolConcurrency(svc)
log.Printf("asynq maintenance-pool server starting with concurrency=%d total_upstream=%d",
- allocation.Maintenance, allocation.Total)
+ allocation.Maintenance, allocation.UpstreamTotal())
return newAsynqServer(allocation.Maintenance, types.QueueWeightsForPool(types.WorkerPoolMaintenance))
}
+// NewSharedAsynqServer is the elastic tier. Asynq dequeue is atomic, so it is
+// safe for this server and the dedicated servers to subscribe to the same
+// queues: every task is still executed by exactly one worker.
+func NewSharedAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
+ allocation := resolveWorkerPoolConcurrency(svc)
+ log.Printf("asynq shared-pool server starting with concurrency=%d total_upstream=%d",
+ allocation.Shared, allocation.UpstreamTotal())
+ return newAsynqServer(allocation.Shared, types.QueueWeightsForSharedPool())
+}
+
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. It runs
// the shared handler mux but only ever pulls wiki tasks, so its concurrency
-// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on
+// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 8) is spent exclusively on
// wiki generation. This is the hard capacity isolation that prevents the parse
// pipeline from starving wiki (and vice-versa) during concurrent uploads.
func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
- concurrency := types.DefaultWikiWorkerConcurrency
- if svc != nil {
- n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency)
- if n > 0 {
- concurrency = int(n)
- }
- }
+ concurrency := resolveWorkerPoolConcurrency(svc).Wiki
log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency)
return newAsynqServer(concurrency, types.QueueWeightsForPool(types.WorkerPoolWiki))
}
@@ -293,8 +308,8 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle)
- // Run the same mux on every pool. Queue definitions are disjoint, so a given
- // task is dequeued by exactly one pool while handlers stay registered once.
+ // Run the same mux on every pool. Shared and dedicated servers intentionally
+ // overlap, but Redis dequeue is atomic, so each task still executes once.
runPool := func(name string, srv *asynq.Server) {
go func() {
if err := srv.Run(mux); err != nil {
@@ -303,8 +318,10 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
}()
}
runPool("core-pool", params.CoreServer)
+ runPool("postprocess-pool", params.PostProcessServer)
runPool("enrichment-pool", params.EnrichmentServer)
runPool("maintenance-pool", params.MaintenanceServer)
+ runPool("shared-pool", params.SharedServer)
runPool("wiki-pool", params.WikiServer)
return mux
}
diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go
index a2e4dd4d..3ace346a 100644
--- a/internal/router/task_inspector.go
+++ b/internal/router/task_inspector.go
@@ -183,6 +183,35 @@ func (a *asynqTaskInspector) QueueStats(
return stats, true, nil
}
+func (a *asynqTaskInspector) WorkerServerStats(
+ ctx context.Context,
+) ([]types.WorkerServerStat, bool, error) {
+ if a == nil || a.inspector == nil {
+ return nil, false, nil
+ }
+ servers, err := a.inspector.Servers()
+ if err != nil {
+ return nil, true, err
+ }
+ stats := make([]types.WorkerServerStat, 0, len(servers))
+ for _, server := range servers {
+ if server == nil {
+ continue
+ }
+ queues := make(map[string]int, len(server.Queues))
+ for name, weight := range server.Queues {
+ queues[name] = weight
+ }
+ stats = append(stats, types.WorkerServerStat{
+ Concurrency: server.Concurrency,
+ Active: len(server.ActiveWorkers),
+ Status: server.Status,
+ Queues: queues,
+ })
+ }
+ return stats, true, nil
+}
+
// queueStateHasMatch pages through one (queue, state) list looking for a
// task that references knowledgeID. Mirrors the delete* scanners but is
// strictly read-only and returns early on the first hit. A backend error
@@ -390,3 +419,9 @@ func (noopTaskInspector) QueueStats(
) ([]types.QueueStat, bool, error) {
return nil, false, nil
}
+
+func (noopTaskInspector) WorkerServerStats(
+ ctx context.Context,
+) ([]types.WorkerServerStat, bool, error) {
+ return nil, false, nil
+}
diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go
index 7ca1a901..87da0f5c 100644
--- a/internal/types/interfaces/task_inspector.go
+++ b/internal/types/interfaces/task_inspector.go
@@ -57,4 +57,9 @@ type TaskInspector interface {
// surfaced as a zeroed row (so the full lane set stays visible) rather
// than failing the whole call.
QueueStats(ctx context.Context) (stats []types.QueueStat, supported bool, err error)
+
+ // WorkerServerStats returns live asynq server heartbeats across all
+ // replicas. The runtime dashboard uses them to aggregate actual cluster
+ // capacity and busy workers for each configured pool.
+ WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error)
}
diff --git a/internal/types/task.go b/internal/types/task.go
index b9385ac2..3d7426d4 100644
--- a/internal/types/task.go
+++ b/internal/types/task.go
@@ -5,15 +5,24 @@ package types
// between pools instead of being only a weighted dequeue preference.
const (
WorkerPoolCore = "core"
+ WorkerPoolPostProcess = "postprocess"
WorkerPoolEnrichment = "enrichment"
WorkerPoolMaintenance = "maintenance"
+ WorkerPoolShared = "shared"
WorkerPoolWiki = "wiki"
- // DefaultUpstreamWorkerConcurrency is split across core, enrichment,
- // and maintenance. Wiki stays separately configurable because its model
- // workload has a different capacity profile.
- DefaultUpstreamWorkerConcurrency = 32
- DefaultWikiWorkerConcurrency = 16
+ // Upstream defaults are explicit guarantees plus an elastic pool. The
+ // shared pool may consume core and enrichment queues, so idle capacity in
+ // either stage can be borrowed without sacrificing the dedicated minimums.
+ DefaultCoreWorkerConcurrency = 8
+ DefaultPostProcessWorkerConcurrency = 2
+ DefaultEnrichmentWorkerConcurrency = 12
+ DefaultMaintenanceWorkerConcurrency = 4
+ DefaultSharedWorkerConcurrency = 6
+ DefaultWikiWorkerConcurrency = 8
+ DefaultUpstreamWorkerConcurrency = DefaultCoreWorkerConcurrency +
+ DefaultPostProcessWorkerConcurrency + DefaultEnrichmentWorkerConcurrency +
+ DefaultMaintenanceWorkerConcurrency + DefaultSharedWorkerConcurrency
)
// Asynq queue names. QueueMaintenance intentionally keeps the physical Redis
@@ -21,6 +30,7 @@ const (
// rolling deployment. New code uses the business-semantic constant.
const (
QueueDefault = "default"
+ QueuePostProcess = "postprocess"
QueueSummary = "summary"
QueueMultimodal = "multimodal"
QueueGraph = "graph"
@@ -35,22 +45,26 @@ const (
// scheduling weights shown to operators from drifting from the actual server
// configuration.
type QueueDefinition struct {
- Name string
- Pool string
- Weight int
- TaskTypes []string
+ Name string
+ Pool string
+ Weight int
+ SharedWeight int
+ TaskTypes []string
}
var queueDefinitions = []QueueDefinition{
- {Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, TaskTypes: []string{
- TypeDocumentProcess, TypeManualProcess, TypeKnowledgePostProcess,
+ {Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, SharedWeight: 3, TaskTypes: []string{
+ TypeDocumentProcess, TypeManualProcess,
}},
- {Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, TaskTypes: []string{
+ {Name: QueuePostProcess, Pool: WorkerPoolPostProcess, Weight: 1, TaskTypes: []string{
+ TypeKnowledgePostProcess,
+ }},
+ {Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, SharedWeight: 2, TaskTypes: []string{
TypeSummaryGeneration, TypeDataTableSummary,
}},
- {Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeImageMultimodal}},
- {Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeChunkExtract}},
- {Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeQuestionGeneration}},
+ {Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeImageMultimodal}},
+ {Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeChunkExtract}},
+ {Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeQuestionGeneration}},
{Name: QueueSync, Pool: WorkerPoolMaintenance, Weight: 2, TaskTypes: []string{TypeDataSourceSync}},
{Name: QueueMaintenance, Pool: WorkerPoolMaintenance, Weight: 1, TaskTypes: []string{
TypeFAQImport, TypeKBClone, TypeIndexDelete, TypeKBDelete,
@@ -95,49 +109,71 @@ func QueueWeightsForPool(pool string) map[string]int {
return weights
}
-// WorkerPoolConcurrency is the deterministic split of the total upstream
-// worker budget. Keeping this pure allocation function in types lets the
-// server constructors and the System Admin API report identical values.
-type WorkerPoolConcurrency struct {
- Total int
- Core int
- Enrichment int
- Maintenance int
-}
-
-// AllocateWorkerPoolConcurrency reserves 1/2 of the upstream budget for the
-// latency-sensitive core pipeline, 3/8 for LLM/VLM enrichment, and the
-// remainder for sync/cleanup/batch work. A minimum total of three guarantees
-// every independent server has at least one worker.
-func AllocateWorkerPoolConcurrency(total int) WorkerPoolConcurrency {
- if total < 3 {
- total = 3
- }
- core := total / 2
- enrichment := (total * 3) / 8
- if core < 1 {
- core = 1
- }
- if enrichment < 1 {
- enrichment = 1
- }
- maintenance := total - core - enrichment
- if maintenance < 1 {
- maintenance = 1
- if core >= enrichment && core > 1 {
- core--
- } else if enrichment > 1 {
- enrichment--
+// QueueWeightsForSharedPool returns the core/enrichment queues eligible for
+// elastic capacity. Post-process and maintenance are deliberately excluded:
+// post-process needs a small latency guarantee, while long maintenance tasks
+// must not pin burst capacity intended for the user-facing pipeline.
+func QueueWeightsForSharedPool() map[string]int {
+ weights := make(map[string]int)
+ for _, definition := range queueDefinitions {
+ if definition.SharedWeight > 0 {
+ weights[definition.Name] = definition.SharedWeight
}
}
+ return weights
+}
+
+// WorkerPoolConcurrency contains the explicit per-instance pool capacities.
+// Unlike the old ratio allocator, each field is independently configurable.
+type WorkerPoolConcurrency struct {
+ Core int
+ PostProcess int
+ Enrichment int
+ Maintenance int
+ Shared int
+ Wiki int
+}
+
+func DefaultWorkerPoolConcurrency() WorkerPoolConcurrency {
return WorkerPoolConcurrency{
- Total: total,
- Core: core,
- Enrichment: enrichment,
- Maintenance: maintenance,
+ Core: DefaultCoreWorkerConcurrency,
+ PostProcess: DefaultPostProcessWorkerConcurrency,
+ Enrichment: DefaultEnrichmentWorkerConcurrency,
+ Maintenance: DefaultMaintenanceWorkerConcurrency,
+ Shared: DefaultSharedWorkerConcurrency,
+ Wiki: DefaultWikiWorkerConcurrency,
}
}
+// ResolveWorkerPoolConcurrency centralizes setting keys, environment names,
+// defaults, and invalid-value fallback for both server construction and the
+// runtime API. The callback lets this package stay independent of the system
+// setting service interface.
+func ResolveWorkerPoolConcurrency(read func(key, env string, fallback int) int) WorkerPoolConcurrency {
+ allocation := DefaultWorkerPoolConcurrency()
+ if read == nil {
+ return allocation
+ }
+ positive := func(key, env string, fallback int) int {
+ value := read(key, env, fallback)
+ if value < 1 {
+ return fallback
+ }
+ return value
+ }
+ allocation.Core = positive("asynq.core_concurrency", "WEKNORA_ASYNQ_CORE_CONCURRENCY", allocation.Core)
+ allocation.PostProcess = positive("asynq.postprocess_concurrency", "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY", allocation.PostProcess)
+ allocation.Enrichment = positive("asynq.enrichment_concurrency", "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY", allocation.Enrichment)
+ allocation.Maintenance = positive("asynq.maintenance_concurrency", "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY", allocation.Maintenance)
+ allocation.Shared = positive("asynq.shared_concurrency", "WEKNORA_ASYNQ_SHARED_CONCURRENCY", allocation.Shared)
+ allocation.Wiki = positive("asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", allocation.Wiki)
+ return allocation
+}
+
+func (c WorkerPoolConcurrency) UpstreamTotal() int {
+ return c.Core + c.PostProcess + c.Enrichment + c.Maintenance + c.Shared
+}
+
// QueueStat is a read-only depth snapshot of a single asynq queue, used
// by the System Admin runtime dashboard. Field names mirror the counts
// exposed by asynq.QueueInfo. Pool / Weight are static metadata (which
@@ -170,6 +206,17 @@ type QueueStat struct {
MemoryUsageBytes int64 `json:"memory_usage_bytes"`
}
+// WorkerServerStat is one live asynq server heartbeat. Queue weights identify
+// which logical pool the server belongs to; the runtime handler aggregates
+// these records across replicas so configured per-instance capacity is not
+// confused with actual cluster capacity.
+type WorkerServerStat struct {
+ Concurrency int
+ Active int
+ Status string
+ Queues map[string]int
+}
+
const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
diff --git a/internal/types/task_queue_test.go b/internal/types/task_queue_test.go
index 9922f164..c13472b8 100644
--- a/internal/types/task_queue_test.go
+++ b/internal/types/task_queue_test.go
@@ -10,6 +10,7 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) {
validPools := map[string]bool{
WorkerPoolCore: true,
+ WorkerPoolPostProcess: true,
WorkerPoolEnrichment: true,
WorkerPoolMaintenance: true,
WorkerPoolWiki: true,
@@ -47,6 +48,13 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) {
t.Fatalf("worker pool %q has no queues", pool)
}
}
+ shared := QueueWeightsForSharedPool()
+ if len(shared) == 0 || shared[QueueDefault] <= 0 || shared[QueueSummary] <= 0 {
+ t.Fatalf("shared pool must cover core and enrichment queues: %+v", shared)
+ }
+ if shared[QueuePostProcess] != 0 || shared[QueueMaintenance] != 0 {
+ t.Fatalf("shared pool must not consume post-process or maintenance: %+v", shared)
+ }
}
func TestQueueMaintenanceKeepsLegacyPhysicalName(t *testing.T) {
@@ -71,22 +79,31 @@ func TestEveryAsynqTaskTypeHasADeclaredQueue(t *testing.T) {
}
}
-func TestAllocateWorkerPoolConcurrencyPreservesBudget(t *testing.T) {
- for _, total := range []int{3, 4, 8, 32, 64} {
- allocation := AllocateWorkerPoolConcurrency(total)
- if allocation.Total != total {
- t.Fatalf("total=%d: allocation reported total %d", total, allocation.Total)
- }
- if allocation.Core < 1 || allocation.Enrichment < 1 || allocation.Maintenance < 1 {
- t.Fatalf("total=%d: every pool must receive capacity: %+v", total, allocation)
- }
- if allocation.Core+allocation.Enrichment+allocation.Maintenance != total {
- t.Fatalf("total=%d: split does not preserve budget: %+v", total, allocation)
- }
+func TestDefaultWorkerPoolConcurrencyIsExplicitBudget(t *testing.T) {
+ allocation := DefaultWorkerPoolConcurrency()
+ if allocation.UpstreamTotal() != DefaultUpstreamWorkerConcurrency {
+ t.Fatalf("upstream total = %d, want %d", allocation.UpstreamTotal(), DefaultUpstreamWorkerConcurrency)
}
-
- minimum := AllocateWorkerPoolConcurrency(1)
- if minimum.Total != 3 || minimum.Core+minimum.Enrichment+minimum.Maintenance != 3 {
- t.Fatalf("undersized budget should clamp to three workers: %+v", minimum)
+ if allocation.Core < 1 || allocation.PostProcess < 1 || allocation.Enrichment < 1 ||
+ allocation.Maintenance < 1 || allocation.Shared < 1 || allocation.Wiki < 1 {
+ t.Fatalf("every pool must have positive explicit capacity: %+v", allocation)
+ }
+}
+
+func TestResolveWorkerPoolConcurrencyFallsBackPerPool(t *testing.T) {
+ allocation := ResolveWorkerPoolConcurrency(func(key, _ string, fallback int) int {
+ if key == "asynq.core_concurrency" {
+ return 15
+ }
+ if key == "asynq.shared_concurrency" {
+ return 0
+ }
+ return fallback
+ })
+ if allocation.Core != 15 {
+ t.Fatalf("core = %d, want override 15", allocation.Core)
+ }
+ if allocation.Shared != DefaultSharedWorkerConcurrency {
+ t.Fatalf("shared = %d, want fallback %d", allocation.Shared, DefaultSharedWorkerConcurrency)
}
}