feat(worker): replace aggregate asynq concurrency with per-pool governance
Split the monolithic upstream worker budget into explicit core, post-process, enrichment, maintenance, and shared elastic pools, expose live cluster capacity on the runtime dashboard, and retire the legacy asynq.concurrency setting.
This commit is contained in:
+22
-9
@@ -716,22 +716,35 @@ DOCREADER_TRANSPORT=grpc
|
||||
# WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h
|
||||
|
||||
# --- Async pipeline tuning (optional) -----------------------------------------
|
||||
# Worker pool size for the PARSE asynq server (document parse / post-process /
|
||||
# embedding / multimodal / graph / question — everything except wiki). Default
|
||||
# 32 — raise it on machines that handle many concurrent uploads (default Go
|
||||
# runtime.NumCPU() under-provisions for the I/O-bound document pipeline). Can
|
||||
# also be set in the management UI under Settings → System settings
|
||||
# (asynq.concurrency); UI changes require a process restart.
|
||||
# WEKNORA_ASYNQ_CONCURRENCY=32
|
||||
# Per-instance guaranteed worker capacity for document parse / manual reparse.
|
||||
# Core and enrichment can additionally borrow from the shared elastic pool.
|
||||
# WEKNORA_ASYNQ_CORE_CONCURRENCY=8
|
||||
|
||||
# Dedicated lightweight post-process/fan-out workers. Keeping this separate
|
||||
# prevents completed parses from waiting behind long DocReader jobs.
|
||||
# WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=2
|
||||
|
||||
# Per-instance guaranteed capacity for summary, image, graph, and question
|
||||
# enrichment tasks.
|
||||
# WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=12
|
||||
|
||||
# Per-instance capacity for datasource sync, batch operations, and cleanup.
|
||||
# Maintenance is not eligible for shared burst capacity because its tasks can
|
||||
# be long-running.
|
||||
# WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=4
|
||||
|
||||
# Elastic capacity shared by core parse and enrichment queues. Queue dequeue is
|
||||
# atomic, so these workers safely supplement both dedicated pools.
|
||||
# WEKNORA_ASYNQ_SHARED_CONCURRENCY=6
|
||||
|
||||
# Worker pool size for the DEDICATED wiki asynq server. The wiki generation
|
||||
# pipeline runs on its own pool with its own concurrency budget, hard-isolated
|
||||
# from the parse pool above: heavy document parsing can never starve wiki
|
||||
# generation during upload storms, and a wiki burst can never starve
|
||||
# user-facing parsing. Default 16. Wiki work is synthesis-LLM-bound, so size
|
||||
# user-facing parsing. Default 8. Wiki work is synthesis-LLM-bound, so size
|
||||
# this to your LLM provider's concurrency budget. Also settable in the UI
|
||||
# (asynq.wiki_concurrency); requires a process restart.
|
||||
# WEKNORA_WIKI_ASYNQ_CONCURRENCY=16
|
||||
# WEKNORA_WIKI_ASYNQ_CONCURRENCY=8
|
||||
|
||||
# Default per-model concurrency cap for background tasks (document ingest /
|
||||
# enrichment). Shared across replicas, keyed by model ID. Only affects
|
||||
|
||||
+6
-2
@@ -195,8 +195,12 @@ services:
|
||||
- WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-}
|
||||
- WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-}
|
||||
- WEKNORA_TENANT_AUTO_CREATE_API_KEY=${WEKNORA_TENANT_AUTO_CREATE_API_KEY:-}
|
||||
- WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32}
|
||||
- WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-16}
|
||||
- WEKNORA_ASYNQ_CORE_CONCURRENCY=${WEKNORA_ASYNQ_CORE_CONCURRENCY:-8}
|
||||
- WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=${WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY:-2}
|
||||
- WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=${WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY:-12}
|
||||
- WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=${WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY:-4}
|
||||
- WEKNORA_ASYNQ_SHARED_CONCURRENCY=${WEKNORA_ASYNQ_SHARED_CONCURRENCY:-6}
|
||||
- WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-8}
|
||||
- WEKNORA_MODEL_MAX_CONCURRENCY=${WEKNORA_MODEL_MAX_CONCURRENCY:-32}
|
||||
- APK_MIRROR_ARG=${APK_MIRROR_ARG:-}
|
||||
- WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# Worker Pool Governance
|
||||
|
||||
WeKnora uses guaranteed per-stage worker pools plus an elastic pool for the
|
||||
document ingestion pipeline. Worker concurrency is a scheduling budget, not a
|
||||
replacement for model quotas, DocReader capacity, vector-store limits, or
|
||||
database connection limits.
|
||||
|
||||
## Topology
|
||||
|
||||
| Pool | Default per instance | Queues | Purpose |
|
||||
| --- | ---: | --- | --- |
|
||||
| Core | 8 | `default` | Document parsing and manual reparse guarantee |
|
||||
| Post-process | 2 | `postprocess` | Parse finalization and enrichment fan-out |
|
||||
| Enrichment | 12 | `summary`, `multimodal`, `graph`, `question` | Model-heavy enrichment guarantee |
|
||||
| Maintenance | 4 | `sync`, `low` | Source sync, batch work, move/delete/cleanup |
|
||||
| Shared | 6 | Core and enrichment queues | Elastic capacity borrowed by the side with backlog |
|
||||
| Wiki | 8 | `wiki` | Independently governed Wiki generation |
|
||||
|
||||
The upstream total remains 32 workers per service instance by default. Wiki is
|
||||
separate and is not included in that total.
|
||||
|
||||
Post-process has its own physical queue so lightweight fan-out work cannot sit
|
||||
behind long DocReader calls. Maintenance is intentionally excluded from the
|
||||
shared pool because its long-running tasks could pin elastic capacity needed by
|
||||
the user-facing pipeline.
|
||||
|
||||
Asynq dequeue is atomic. Dedicated and shared servers may safely subscribe to
|
||||
the same core/enrichment queues; one task is still processed by one worker.
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings are available under System settings and require a service restart:
|
||||
|
||||
- `asynq.core_concurrency` / `WEKNORA_ASYNQ_CORE_CONCURRENCY`
|
||||
- `asynq.postprocess_concurrency` / `WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY`
|
||||
- `asynq.enrichment_concurrency` / `WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY`
|
||||
- `asynq.maintenance_concurrency` / `WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY`
|
||||
- `asynq.shared_concurrency` / `WEKNORA_ASYNQ_SHARED_CONCURRENCY`
|
||||
- `asynq.wiki_concurrency` / `WEKNORA_WIKI_ASYNQ_CONCURRENCY`
|
||||
|
||||
The old aggregate `asynq.concurrency` / `WEKNORA_ASYNQ_CONCURRENCY` setting is
|
||||
retired. Deployments that set it must migrate to the explicit pool settings.
|
||||
Persisted old rows are ignored and hidden from the System settings page so they
|
||||
cannot be mistaken for an effective runtime control.
|
||||
|
||||
## Capacity layers
|
||||
|
||||
Tune these layers independently:
|
||||
|
||||
1. Worker concurrency controls the number of task handlers admitted per
|
||||
service instance.
|
||||
2. Model quota governance controls provider concurrency, RPM, and TPM across
|
||||
replicas and quota groups.
|
||||
3. DocReader, vector stores, object storage, Postgres, and local CPU/RAM retain
|
||||
their own resource limits.
|
||||
|
||||
If model limiter waiting grows while worker queues are busy, adding workers
|
||||
only creates more waiters. Increase the provider quota or reduce worker
|
||||
admission instead.
|
||||
|
||||
## Sizing
|
||||
|
||||
For each pool, estimate peak required workers using:
|
||||
|
||||
```text
|
||||
required workers = ceil(peak task arrival rate * mean task runtime / 0.70)
|
||||
```
|
||||
|
||||
Use task arrival rate after fan-out. One parsed document can create one summary,
|
||||
multiple question batches, one graph task per chunk, and multiple image tasks.
|
||||
Queue count is not a capacity signal.
|
||||
|
||||
The runtime dashboard reports:
|
||||
|
||||
- configured concurrency per instance;
|
||||
- live server instance count;
|
||||
- cluster capacity aggregated from active asynq heartbeats;
|
||||
- active workers and utilization;
|
||||
- queue backlog, retries, dead letters, and oldest pending age;
|
||||
- model concurrency and limiter waiting.
|
||||
|
||||
Increase a pool only when backlog age grows while its downstream dependency has
|
||||
headroom. Reallocate to enrichment when core stays underutilized and fan-out
|
||||
queues grow. Reduce core admission when DocReader CPU, memory, or latency is the
|
||||
bottleneck.
|
||||
@@ -507,6 +507,10 @@ export interface RuntimeWorkerPool {
|
||||
name: string
|
||||
concurrency: number
|
||||
queue_count: number
|
||||
instances: number
|
||||
cluster_capacity: number
|
||||
active: number
|
||||
utilization: number
|
||||
}
|
||||
|
||||
export interface ModelRuntimeStat {
|
||||
@@ -520,9 +524,9 @@ export interface ModelRuntimeStat {
|
||||
/**
|
||||
* Runtime queue dashboard payload. `available` is false in Lite mode
|
||||
* (no Redis/asynq) — render an "unavailable in this deployment" state
|
||||
* rather than an empty table. Pool concurrency values are CONFIGURED
|
||||
* per-process capacity (effective on next restart), not live busy-worker
|
||||
* count or cluster-wide capacity.
|
||||
* rather than an empty table. Each pool includes both configured per-process
|
||||
* concurrency and live cluster capacity/active workers aggregated from asynq
|
||||
* server heartbeats.
|
||||
*/
|
||||
export interface RuntimeQueuesResponse {
|
||||
available: boolean
|
||||
|
||||
@@ -3496,11 +3496,14 @@ export default {
|
||||
detailsTitle: "Queue details",
|
||||
detailsDescription: "Live load and wait time for every processing lane.",
|
||||
poolsTitle: "Worker pools",
|
||||
poolsDescription: "Core parsing, enrichment, maintenance, and Wiki use isolated concurrency budgets.",
|
||||
perInstance: "Concurrency is per process",
|
||||
poolsDescription: "Each stage has guaranteed capacity; core and enrichment may also borrow the shared elastic pool.",
|
||||
perInstance: "Card value is cluster active/capacity",
|
||||
poolConfigured: "Per-process {value}",
|
||||
poolInstances: "{value} instances",
|
||||
poolUtilization: "{value}% utilized",
|
||||
queueCount: "{value} queues",
|
||||
weight: "Pool weight {value}",
|
||||
footnote: "Concurrency is a per-process cap (restart required); weight only controls scheduling within one pool; Active is cluster-wide.",
|
||||
footnote: "Card values show cluster active/live capacity; per-process configuration requires restart. The shared elastic pool only consumes core and enrichment queues.",
|
||||
updatedAt: "Updated at {value}",
|
||||
errors: {
|
||||
generic: "Failed to load queue stats",
|
||||
@@ -3541,18 +3544,23 @@ export default {
|
||||
},
|
||||
pools: {
|
||||
core: "Core parsing",
|
||||
postprocess: "Post-process orchestration",
|
||||
enrichment: "Enrichment",
|
||||
maintenance: "Maintenance & sync",
|
||||
shared: "Shared elastic",
|
||||
wiki: "Wiki pool",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "Document parsing and post-process orchestration",
|
||||
core: "Guaranteed document and manual parsing capacity",
|
||||
postprocess: "Parse finalization and enrichment fan-out",
|
||||
enrichment: "Summaries, images, graph, and question generation",
|
||||
maintenance: "Source sync, batch work, and deletion cleanup",
|
||||
shared: "Borrowed by core or enrichment according to backlog",
|
||||
wiki: "Wiki content generation and global finalization",
|
||||
},
|
||||
queueNames: {
|
||||
default: "Default (document parse)",
|
||||
postprocess: "Post-process orchestration",
|
||||
summary: "Summary generation",
|
||||
sync: "Source sync",
|
||||
low: "Maintenance & batch jobs",
|
||||
@@ -3562,7 +3570,8 @@ export default {
|
||||
wiki: "Wiki ingest",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "Document parsing, manual updates, and post-process orchestration",
|
||||
default: "Document parsing and manual updates",
|
||||
postprocess: "Parse finalization and enrichment task fan-out",
|
||||
summary: "Document and data-table summaries",
|
||||
sync: "Manual and scheduled data-source sync",
|
||||
low: "FAQ import, clone/move, batch reparse, and deletion cleanup",
|
||||
@@ -3584,7 +3593,11 @@ export default {
|
||||
default_storage_quota_gb: 'Default storage quota for new tenants (GB)',
|
||||
},
|
||||
asynq: {
|
||||
concurrency: 'Total upstream worker budget',
|
||||
core_concurrency: 'Guaranteed core parse concurrency',
|
||||
postprocess_concurrency: 'Post-process concurrency',
|
||||
enrichment_concurrency: 'Guaranteed enrichment concurrency',
|
||||
maintenance_concurrency: 'Maintenance concurrency',
|
||||
shared_concurrency: 'Shared elastic concurrency',
|
||||
wiki_concurrency: 'Wiki worker concurrency',
|
||||
},
|
||||
model: {
|
||||
@@ -3607,8 +3620,11 @@ export default {
|
||||
'Default storage quota (GB) assigned when a new tenant is created, covering vectors, originals, text, indexes, and related data. Read only at creation time — changes apply to newly created tenants only and do not retroactively update existing tenants. 0 or a negative value uses the built-in default of 10 GB.',
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
'Total per-process concurrency budget for upstream background work (excluding Wiki). It is split into isolated pools: 1/2 core parsing, 3/8 enrichment, and the remainder maintenance and sync. Minimum 3; requires a service restart.',
|
||||
core_concurrency: 'Guaranteed per-process concurrency for document and manual parsing. Core may also borrow the shared elastic pool. Minimum 1; requires a service restart.',
|
||||
postprocess_concurrency: 'Dedicated per-process concurrency for lightweight parse finalization and enrichment fan-out. Minimum 1; requires a service restart.',
|
||||
enrichment_concurrency: 'Guaranteed per-process concurrency for summaries, images, graph, and question generation. Enrichment may also borrow the shared elastic pool. Minimum 1; requires a service restart.',
|
||||
maintenance_concurrency: 'Per-process concurrency for source sync, batch work, and cleanup, hard-isolated from the user-facing pipeline. Minimum 1; requires a service restart.',
|
||||
shared_concurrency: 'Per-process elastic concurrency shared by core parsing and enrichment; whichever side has backlog borrows it automatically. Minimum 1; requires a service restart.',
|
||||
wiki_concurrency:
|
||||
'Per-process concurrency for the dedicated Wiki worker pool, isolated from upstream tasks. Minimum 1; requires a service restart.',
|
||||
},
|
||||
|
||||
@@ -2481,11 +2481,14 @@ export default {
|
||||
detailsTitle: "큐 세부 정보",
|
||||
detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.",
|
||||
poolsTitle: "워커 풀",
|
||||
poolsDescription: "핵심 파싱, 보강, 유지 관리 및 Wiki는 독립된 동시성 예산을 사용합니다.",
|
||||
perInstance: "동시성은 프로세스별 설정",
|
||||
poolsDescription: "각 단계에 보장 용량이 있으며 핵심 파싱과 보강은 공유 탄력 풀도 사용할 수 있습니다.",
|
||||
perInstance: "카드 값은 클러스터 실행 중/용량",
|
||||
poolConfigured: "프로세스별 {value}",
|
||||
poolInstances: "인스턴스 {value}개",
|
||||
poolUtilization: "사용률 {value}%",
|
||||
queueCount: "큐 {value}개",
|
||||
weight: "풀 내 가중치 {value}",
|
||||
footnote: "동시성은 프로세스별 상한이며 재시작 후 적용됩니다. 가중치는 같은 풀 안의 스케줄링 비율이며 실행 중 수치는 클러스터 전체입니다.",
|
||||
footnote: "카드는 클러스터 실행 중/실시간 용량을 표시합니다. 프로세스별 설정은 재시작 후 적용되며 공유 풀은 핵심 파싱과 보강 큐만 처리합니다.",
|
||||
updatedAt: "{value} 업데이트됨",
|
||||
errors: {
|
||||
generic: "큐 상태를 불러오지 못했습니다",
|
||||
@@ -2526,18 +2529,23 @@ export default {
|
||||
},
|
||||
pools: {
|
||||
core: "핵심 파싱",
|
||||
postprocess: "후처리 오케스트레이션",
|
||||
enrichment: "콘텐츠 보강",
|
||||
maintenance: "유지 관리 및 동기화",
|
||||
shared: "공유 탄력 풀",
|
||||
wiki: "Wiki 풀",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "문서 파싱 및 후처리 오케스트레이션",
|
||||
core: "문서 및 수동 파싱 보장 용량",
|
||||
postprocess: "파싱 마무리 및 보강 작업 분배",
|
||||
enrichment: "요약, 이미지, 그래프 및 질문 생성",
|
||||
maintenance: "데이터 소스 동기화, 일괄 작업 및 삭제 정리",
|
||||
shared: "적체에 따라 핵심 파싱 또는 보강이 공유",
|
||||
wiki: "Wiki 콘텐츠 생성 및 전체 마무리",
|
||||
},
|
||||
queueNames: {
|
||||
default: "기본(문서 파싱)",
|
||||
postprocess: "후처리 오케스트레이션",
|
||||
summary: "요약 생성",
|
||||
sync: "데이터 소스 동기화",
|
||||
low: "백그라운드 유지 관리 및 일괄 작업",
|
||||
@@ -2547,7 +2555,8 @@ export default {
|
||||
wiki: "Wiki 동기화",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "문서 파싱, 수동 업데이트 및 후처리 오케스트레이션",
|
||||
default: "문서 파싱 및 수동 업데이트",
|
||||
postprocess: "파싱 마무리 및 보강 작업 분배",
|
||||
summary: "문서 및 데이터 테이블 요약",
|
||||
sync: "수동 및 예약 데이터 소스 동기화",
|
||||
low: "FAQ 가져오기, 복제/이동, 일괄 재파싱 및 삭제 정리",
|
||||
@@ -2569,7 +2578,11 @@ export default {
|
||||
default_storage_quota_gb: "신규 테넌트 기본 저장 용량 (GB)",
|
||||
},
|
||||
asynq: {
|
||||
concurrency: "상위 워커 총 동시성 예산",
|
||||
core_concurrency: "핵심 파싱 보장 동시성",
|
||||
postprocess_concurrency: "후처리 동시성",
|
||||
enrichment_concurrency: "콘텐츠 보강 보장 동시성",
|
||||
maintenance_concurrency: "유지 관리 동시성",
|
||||
shared_concurrency: "공유 탄력 동시성",
|
||||
wiki_concurrency: "Wiki 워커 동시성",
|
||||
},
|
||||
model: {
|
||||
@@ -2592,8 +2605,11 @@ export default {
|
||||
"신규 테넌트 생성 시 기본으로 할당되는 저장 용량(GB)으로, 벡터·원본·텍스트·인덱스 등을 포함합니다. 생성 시에만 읽으며, 변경은 이후 생성되는 테넌트에만 적용되고 기존 테넌트에는 소급되지 않습니다. 0 또는 음수는 내장 기본값 10GB를 사용합니다.",
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
"Wiki를 제외한 상위 백그라운드 작업의 프로세스별 총 동시성 예산입니다. 핵심 파싱 1/2, 콘텐츠 보강 3/8, 나머지는 유지 관리 및 동기화로 분리됩니다. 최소값은 3이며 적용하려면 서비스를 재시작해야 합니다.",
|
||||
core_concurrency: "문서 및 수동 파싱의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.",
|
||||
postprocess_concurrency: "파싱 마무리 및 보강 작업 분배 전용 프로세스별 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.",
|
||||
enrichment_concurrency: "요약, 이미지, 그래프 및 질문 생성의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.",
|
||||
maintenance_concurrency: "동기화, 일괄 작업 및 정리의 프로세스별 동시성으로 사용자 파이프라인과 격리됩니다. 최소 1, 서비스 재시작이 필요합니다.",
|
||||
shared_concurrency: "핵심 파싱과 콘텐츠 보강이 적체에 따라 공유하는 프로세스별 탄력 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.",
|
||||
wiki_concurrency:
|
||||
"상위 작업과 분리된 전용 Wiki 워커 풀의 프로세스별 동시성입니다. 최소값은 1이며 적용하려면 서비스를 재시작해야 합니다.",
|
||||
},
|
||||
|
||||
@@ -2179,11 +2179,14 @@ export default {
|
||||
detailsTitle: "Сведения об очередях",
|
||||
detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.",
|
||||
poolsTitle: "Пулы воркеров",
|
||||
poolsDescription: "Основной разбор, обогащение, обслуживание и Wiki используют независимые бюджеты параллелизма.",
|
||||
perInstance: "Параллелизм указан на процесс",
|
||||
poolsDescription: "Каждый этап имеет гарантированную ёмкость; разбор и обогащение также используют общий эластичный пул.",
|
||||
perInstance: "На карточке активно/ёмкость кластера",
|
||||
poolConfigured: "На процесс: {value}",
|
||||
poolInstances: "Экземпляров: {value}",
|
||||
poolUtilization: "Загрузка {value}%",
|
||||
queueCount: "Очередей: {value}",
|
||||
weight: "Вес в пуле {value}",
|
||||
footnote: "Параллелизм указан на процесс и требует перезапуска; вес действует только внутри пула; активные задачи показаны по всему кластеру.",
|
||||
footnote: "Карточки показывают активные задачи/живую ёмкость кластера. Настройки на процесс требуют перезапуска; общий пул обслуживает только разбор и обогащение.",
|
||||
updatedAt: "Обновлено в {value}",
|
||||
errors: {
|
||||
generic: "Не удалось загрузить состояние очередей",
|
||||
@@ -2224,18 +2227,23 @@ export default {
|
||||
},
|
||||
pools: {
|
||||
core: "Основной разбор",
|
||||
postprocess: "Оркестрация постобработки",
|
||||
enrichment: "Обогащение",
|
||||
maintenance: "Обслуживание и синхронизация",
|
||||
shared: "Общий эластичный пул",
|
||||
wiki: "Пул Wiki",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "Разбор документов и оркестрация постобработки",
|
||||
core: "Гарантированная ёмкость разбора документов",
|
||||
postprocess: "Завершение разбора и запуск обогащения",
|
||||
enrichment: "Сводки, изображения, граф и генерация вопросов",
|
||||
maintenance: "Синхронизация источников, пакетные задачи и удаление",
|
||||
shared: "Используется разбором или обогащением по мере очереди",
|
||||
wiki: "Создание содержимого Wiki и финальная обработка",
|
||||
},
|
||||
queueNames: {
|
||||
default: "По умолчанию (парсинг документов)",
|
||||
postprocess: "Оркестрация постобработки",
|
||||
summary: "Создание сводок",
|
||||
sync: "Синхронизация источников",
|
||||
low: "Фоновое обслуживание и пакетные задачи",
|
||||
@@ -2245,7 +2253,8 @@ export default {
|
||||
wiki: "Синхронизация Wiki",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "Разбор документов, ручные обновления и оркестрация постобработки",
|
||||
default: "Разбор документов и ручные обновления",
|
||||
postprocess: "Завершение разбора и запуск задач обогащения",
|
||||
summary: "Сводки документов и таблиц данных",
|
||||
sync: "Ручная и плановая синхронизация источников",
|
||||
low: "Импорт FAQ, клонирование/перемещение, пакетный повторный разбор и удаление",
|
||||
@@ -2267,7 +2276,11 @@ export default {
|
||||
default_storage_quota_gb: 'Квота хранилища для новых тенантов по умолчанию (ГБ)',
|
||||
},
|
||||
asynq: {
|
||||
concurrency: 'Общий бюджет фоновых воркеров',
|
||||
core_concurrency: 'Гарантированный параллелизм разбора',
|
||||
postprocess_concurrency: 'Параллелизм постобработки',
|
||||
enrichment_concurrency: 'Гарантированный параллелизм обогащения',
|
||||
maintenance_concurrency: 'Параллелизм обслуживания',
|
||||
shared_concurrency: 'Общий эластичный параллелизм',
|
||||
wiki_concurrency: 'Параллелизм воркеров Wiki',
|
||||
},
|
||||
model: {
|
||||
@@ -2290,8 +2303,11 @@ export default {
|
||||
'Квота хранилища по умолчанию (ГБ) для нового тенанта: векторы, оригиналы, текст, индексы и т.д. Читается только при создании — изменения применяются только к новым тенантам и не перезаписывают существующие. 0 или отрицательное значение — встроенное значение по умолчанию 10 ГБ.',
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
'Общий бюджет параллелизма фоновых задач на процесс (без Wiki). Он делится на изолированные пулы: 1/2 основной разбор, 3/8 обогащение, остаток — обслуживание и синхронизация. Минимум 3; требуется перезапуск сервиса.',
|
||||
core_concurrency: 'Гарантированный параллелизм разбора документов на процесс; также может использовать общий эластичный пул. Минимум 1; требуется перезапуск.',
|
||||
postprocess_concurrency: 'Выделенный параллелизм завершения разбора и запуска обогащения на процесс. Минимум 1; требуется перезапуск.',
|
||||
enrichment_concurrency: 'Гарантированный параллелизм сводок, изображений, графа и вопросов на процесс; также может использовать общий пул. Минимум 1; требуется перезапуск.',
|
||||
maintenance_concurrency: 'Параллелизм синхронизации, пакетных операций и очистки на процесс, изолированный от пользовательского конвейера. Минимум 1; требуется перезапуск.',
|
||||
shared_concurrency: 'Эластичный параллелизм на процесс, общий для разбора и обогащения и используемый стороной с очередью. Минимум 1; требуется перезапуск.',
|
||||
wiki_concurrency:
|
||||
'Параллелизм выделенного пула Wiki на процесс, изолированного от остальных задач. Минимум 1; требуется перезапуск сервиса.',
|
||||
},
|
||||
|
||||
@@ -2484,11 +2484,14 @@ export default {
|
||||
detailsTitle: "队列明细",
|
||||
detailsDescription: "各处理通道的实时负载与等待情况。",
|
||||
poolsTitle: "Worker 池",
|
||||
poolsDescription: "核心解析、内容富化、维护同步和 Wiki 使用相互隔离的并发预算。",
|
||||
perInstance: "并发数为单实例配置",
|
||||
poolsDescription: "各阶段拥有保底容量,核心解析与内容富化还可借用共享弹性池。",
|
||||
perInstance: "卡片主值为集群运行中/容量",
|
||||
poolConfigured: "单实例配置 {value}",
|
||||
poolInstances: "{value} 个实例",
|
||||
poolUtilization: "利用率 {value}%",
|
||||
queueCount: "{value} 个队列",
|
||||
weight: "池内权重 {value}",
|
||||
footnote: "Worker 并发数为单实例上限(修改后需重启);权重只决定同一池内的调度份额;「运行中」为集群当前任务数。",
|
||||
footnote: "卡片主值为集群运行中/实时容量;单实例配置修改后需重启。共享弹性池只消费核心解析和内容富化队列。",
|
||||
updatedAt: "更新于 {value}",
|
||||
errors: {
|
||||
generic: "获取队列状态失败",
|
||||
@@ -2529,18 +2532,23 @@ export default {
|
||||
},
|
||||
pools: {
|
||||
core: "核心解析",
|
||||
postprocess: "后处理编排",
|
||||
enrichment: "内容富化",
|
||||
maintenance: "维护与同步",
|
||||
shared: "共享弹性",
|
||||
wiki: "Wiki 池",
|
||||
},
|
||||
poolDescriptions: {
|
||||
core: "文档解析与后处理调度",
|
||||
core: "文档解析与手工重解析的保底容量",
|
||||
postprocess: "解析完成后的收尾与富化扇出",
|
||||
enrichment: "摘要、图片、图谱与问题生成",
|
||||
maintenance: "数据源同步、批处理与删除清理",
|
||||
shared: "由核心解析与内容富化按积压借用",
|
||||
wiki: "Wiki 内容生成与全局收尾",
|
||||
},
|
||||
queueNames: {
|
||||
default: "默认(文档解析)",
|
||||
postprocess: "后处理编排",
|
||||
summary: "摘要生成",
|
||||
sync: "数据源同步",
|
||||
low: "后台维护与批处理",
|
||||
@@ -2550,7 +2558,8 @@ export default {
|
||||
wiki: "Wiki 同步",
|
||||
},
|
||||
queueDescriptions: {
|
||||
default: "文档解析、手工更新与后处理调度",
|
||||
default: "文档解析与手工更新",
|
||||
postprocess: "解析完成后的状态收尾与富化任务扇出",
|
||||
summary: "文档摘要与表格摘要",
|
||||
sync: "手动与定时数据源同步",
|
||||
low: "FAQ 导入、复制移动、批量重解析与删除清理",
|
||||
@@ -2572,7 +2581,11 @@ export default {
|
||||
default_storage_quota_gb: "新租户默认存储配额 (GB)",
|
||||
},
|
||||
asynq: {
|
||||
concurrency: "上游 Worker 总并发预算",
|
||||
core_concurrency: "核心解析保底并发数",
|
||||
postprocess_concurrency: "后处理编排并发数",
|
||||
enrichment_concurrency: "内容富化保底并发数",
|
||||
maintenance_concurrency: "维护与同步并发数",
|
||||
shared_concurrency: "共享弹性并发数",
|
||||
wiki_concurrency: "Wiki Worker 并发数",
|
||||
},
|
||||
model: {
|
||||
@@ -2600,10 +2613,11 @@ export default {
|
||||
"0 或负数表示使用内置默认值 10GB。",
|
||||
},
|
||||
asynq: {
|
||||
concurrency:
|
||||
"每个服务实例用于上游后台任务的总并发预算(不含 Wiki)。系统按核心解析 1/2、" +
|
||||
"内容富化 3/8、维护与同步为剩余容量,拆分成相互隔离的 Worker 池。" +
|
||||
"最小值为 3;修改后需重启服务进程方可生效。",
|
||||
core_concurrency: "文档解析与手工重解析的每实例保底并发,可额外借用共享弹性池;修改后需重启。",
|
||||
postprocess_concurrency: "解析完成后的轻量收尾和富化扇出专用并发;修改后需重启。",
|
||||
enrichment_concurrency: "摘要、图片、图谱与问题生成的每实例保底并发,可额外借用共享弹性池;修改后需重启。",
|
||||
maintenance_concurrency: "数据源同步、批处理和清理任务的每实例并发,与用户面流水线硬隔离;修改后需重启。",
|
||||
shared_concurrency: "核心解析与内容富化共同使用的每实例弹性并发,由有积压的一侧自动借用;修改后需重启。",
|
||||
wiki_concurrency:
|
||||
"每个服务实例的 Wiki 专用 Worker 并发数,与上游任务池相互隔离。" +
|
||||
"最小值为 1;修改后需重启服务进程方可生效。",
|
||||
|
||||
@@ -106,12 +106,19 @@
|
||||
<div v-for="pool in pools" :key="pool.name" class="rq-pool-card">
|
||||
<div class="rq-pool-topline">
|
||||
<span class="rq-pool-name">{{ poolLabel(pool.name) }}</span>
|
||||
<strong class="rq-pool-value">{{ pool.concurrency }}</strong>
|
||||
<strong class="rq-pool-value">
|
||||
{{ pool.instances > 0 ? `${pool.active}/${pool.cluster_capacity}` : pool.concurrency }}
|
||||
</strong>
|
||||
</div>
|
||||
<p class="rq-pool-desc">
|
||||
{{ poolDescription(pool.name) }}
|
||||
<span class="rq-pool-meta">
|
||||
{{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}
|
||||
{{ t('system.globalSettings.runtime.poolConfigured', { value: pool.concurrency }) }}
|
||||
<template v-if="pool.instances > 0">
|
||||
· {{ t('system.globalSettings.runtime.poolInstances', { value: pool.instances }) }}
|
||||
· {{ t('system.globalSettings.runtime.poolUtilization', { value: poolUtilization(pool) }) }}
|
||||
</template>
|
||||
· {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
@@ -310,6 +317,10 @@ function poolQueueCount(pool: string): number {
|
||||
return pools.value.find((item) => item.name === pool)?.queue_count ?? 0
|
||||
}
|
||||
|
||||
function poolUtilization(pool: RuntimeWorkerPool): number {
|
||||
return Math.round(Math.max(0, Math.min(1, pool.utilization || 0)) * 100)
|
||||
}
|
||||
|
||||
function formatLatency(ms: number): string {
|
||||
if (!ms || ms <= 0) return '—'
|
||||
if (ms < 1000) return `${ms} ms`
|
||||
|
||||
@@ -761,8 +761,7 @@ function placeholderFor(item: SystemSettingItem): string {
|
||||
}
|
||||
|
||||
function minimumFor(item: SystemSettingItem): number {
|
||||
if (item.key === 'asynq.concurrency') return 3
|
||||
if (item.key === 'asynq.wiki_concurrency') return 1
|
||||
if (item.key.startsWith('asynq.') && item.key.endsWith('_concurrency')) return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -626,7 +626,7 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
knowledgePostProcessTaskOptions()...)
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
|
||||
@@ -186,8 +186,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
if queueSkipped > 0 {
|
||||
// Visibility into "stale span heartbeat but tasks still queued"
|
||||
// — i.e. backpressure, not a stuck row. Persistent counts here
|
||||
// mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY
|
||||
// or document_process_timeout), not that housekeeping misfires.
|
||||
// mean the queue is the bottleneck (raise the matching per-pool or
|
||||
// shared asynq concurrency, or document_process_timeout), not that
|
||||
// housekeeping misfires.
|
||||
logger.Infof(ctx,
|
||||
"[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)",
|
||||
queueSkipped)
|
||||
|
||||
@@ -126,6 +126,12 @@ func (f fakeTaskInspector) QueueStats(
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) WorkerServerStats(
|
||||
_ context.Context,
|
||||
) ([]types.WorkerServerStat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
|
||||
return newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
|
||||
}
|
||||
|
||||
@@ -675,7 +675,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
payloadBytes, err := json.Marshal(postProcessPayload)
|
||||
if err == nil {
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
knowledgePostProcessTaskOptions()...)
|
||||
if _, err := s.task.Enqueue(task); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
@@ -15,3 +17,11 @@ func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asy
|
||||
opts = append(opts, extra...)
|
||||
return opts
|
||||
}
|
||||
|
||||
func knowledgePostProcessTaskOptions() []asynq.Option {
|
||||
return []asynq.Option{
|
||||
asynq.Queue(types.QueuePostProcess),
|
||||
asynq.MaxRetry(3),
|
||||
asynq.Timeout(30 * time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,3 +79,12 @@ func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) {
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
}
|
||||
|
||||
func TestKnowledgePostProcessTaskOptionsUseDedicatedQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
queue, timeout, maxRetry := parseDocumentProcessOpts(t, knowledgePostProcessTaskOptions())
|
||||
assert.Equal(t, types.QueuePostProcess, queue)
|
||||
assert.Equal(t, 30*time.Minute, timeout)
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
}
|
||||
|
||||
@@ -183,24 +183,50 @@ var registry = map[string]settingSpec{
|
||||
"用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" +
|
||||
"每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。",
|
||||
},
|
||||
// asynq.concurrency is the total per-process upstream worker budget.
|
||||
// It is divided deterministically across independent core, enrichment,
|
||||
// and maintenance servers. Read once at startup; changing it requires a
|
||||
// process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32).
|
||||
"asynq.concurrency": {
|
||||
"asynq.core_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
|
||||
Default: int64(types.DefaultUpstreamWorkerConcurrency),
|
||||
EnvName: "WEKNORA_ASYNQ_CORE_CONCURRENCY",
|
||||
Default: int64(types.DefaultCoreWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" +
|
||||
"内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" +
|
||||
"最小值为 3;修改后需重启服务进程方可生效。",
|
||||
Description: "文档解析、手工重解析等核心任务的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
|
||||
},
|
||||
"asynq.postprocess_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY",
|
||||
Default: int64(types.DefaultPostProcessWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "解析完成后的轻量编排与富化扇出专用并发,避免被长时间文档解析阻塞;修改后需重启。",
|
||||
},
|
||||
"asynq.enrichment_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY",
|
||||
Default: int64(types.DefaultEnrichmentWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "摘要、图片、图谱和问题生成的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
|
||||
},
|
||||
"asynq.maintenance_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY",
|
||||
Default: int64(types.DefaultMaintenanceWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "数据源同步、批处理、移动和删除清理的每实例保底并发,与用户面流水线硬隔离;修改后需重启。",
|
||||
},
|
||||
"asynq.shared_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_SHARED_CONCURRENCY",
|
||||
Default: int64(types.DefaultSharedWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "核心解析与内容富化共用的每实例弹性并发。空闲容量由有积压的一侧借用;修改后需重启。",
|
||||
},
|
||||
// asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool,
|
||||
// separate from asynq.concurrency. Read once when the wiki asynq server
|
||||
// separate from the upstream pools. Read once when the wiki asynq server
|
||||
// starts — changing it in the UI requires a process restart. Mirrors
|
||||
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16).
|
||||
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 8).
|
||||
"asynq.wiki_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY",
|
||||
@@ -669,6 +695,10 @@ func (s *systemSettingService) List(ctx context.Context) ([]*types.SystemSetting
|
||||
for _, row := range rows {
|
||||
byKey[row.Key] = row
|
||||
}
|
||||
// asynq.concurrency was the old fixed-ratio aggregate. Keeping an old DB
|
||||
// row visible would suggest it still controls runtime capacity, so retire it
|
||||
// explicitly while preserving all genuinely unknown rows for diagnostics.
|
||||
delete(byKey, "asynq.concurrency")
|
||||
|
||||
keys := make([]string, 0, len(registry))
|
||||
for key := range registry {
|
||||
@@ -1228,17 +1258,15 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
|
||||
// 400 body verbatim).
|
||||
func validateRegistryEntry(key string, rawValue any) error {
|
||||
switch key {
|
||||
case "asynq.concurrency", "asynq.wiki_concurrency":
|
||||
case "asynq.core_concurrency", "asynq.postprocess_concurrency",
|
||||
"asynq.enrichment_concurrency", "asynq.maintenance_concurrency",
|
||||
"asynq.shared_concurrency", "asynq.wiki_concurrency":
|
||||
n, err := coerceToPositiveInt64(rawValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
minimum := int64(1)
|
||||
if key == "asynq.concurrency" {
|
||||
minimum = 3
|
||||
}
|
||||
if n < minimum {
|
||||
return fmt.Errorf("concurrency must be at least %d", minimum)
|
||||
if n < 1 {
|
||||
return errors.New("concurrency must be at least 1")
|
||||
}
|
||||
case "ssrf.whitelist":
|
||||
// Coerce into the same shape encodeForType produced. We don't
|
||||
|
||||
@@ -9,9 +9,12 @@ func TestValidateWorkerConcurrencyMinimums(t *testing.T) {
|
||||
value any
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true},
|
||||
{name: "upstream minimum", key: "asynq.concurrency", value: 3},
|
||||
{name: "upstream normal", key: "asynq.concurrency", value: 32},
|
||||
{name: "core zero", key: "asynq.core_concurrency", value: 0, wantErr: true},
|
||||
{name: "core minimum", key: "asynq.core_concurrency", value: 1},
|
||||
{name: "postprocess minimum", key: "asynq.postprocess_concurrency", value: 1},
|
||||
{name: "enrichment minimum", key: "asynq.enrichment_concurrency", value: 1},
|
||||
{name: "maintenance minimum", key: "asynq.maintenance_concurrency", value: 1},
|
||||
{name: "shared minimum", key: "asynq.shared_concurrency", value: 1},
|
||||
{name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true},
|
||||
{name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1},
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ const (
|
||||
// wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}.
|
||||
wikiInflightPrefix = "wiki:inflight:"
|
||||
// wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight
|
||||
// is unset. 4 leaves headroom in a default 16-worker wiki pool for other
|
||||
// is unset. 4 leaves half of the default 8-worker wiki pool for other
|
||||
// KBs while still giving one KB solid parallelism.
|
||||
wikiInflightDefault = 4
|
||||
// wikiInflightTTL is how long a reserved slot survives without renewal.
|
||||
|
||||
@@ -264,13 +264,14 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
redisAvailable := os.Getenv("REDIS_ADDR") != ""
|
||||
if redisAvailable {
|
||||
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
|
||||
// Independent worker pools keep user-facing parsing, high-fanout
|
||||
// enrichment, maintenance/sync, and Wiki generation from consuming one
|
||||
// another's capacity. All pools share one handler mux but subscribe to
|
||||
// disjoint queues from the central topology registry.
|
||||
// Dedicated pools guarantee capacity for each stage. The shared pool
|
||||
// additionally subscribes to core/enrichment queues to provide elastic
|
||||
// borrowing while post-process and maintenance remain hard-isolated.
|
||||
must(container.Provide(router.NewCoreAsynqServer, dig.Name("coreAsynqServer")))
|
||||
must(container.Provide(router.NewPostProcessAsynqServer, dig.Name("postProcessAsynqServer")))
|
||||
must(container.Provide(router.NewEnrichmentAsynqServer, dig.Name("enrichmentAsynqServer")))
|
||||
must(container.Provide(router.NewMaintenanceAsynqServer, dig.Name("maintenanceAsynqServer")))
|
||||
must(container.Provide(router.NewSharedAsynqServer, dig.Name("sharedAsynqServer")))
|
||||
must(container.Provide(router.NewWikiAsynqServer, dig.Name("wikiAsynqServer")))
|
||||
// Asynq inspector for cancel-by-knowledge-id (best-effort
|
||||
// dequeue of pending/scheduled/retry tasks + active-task cancel).
|
||||
|
||||
+78
-23
@@ -1385,13 +1385,16 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
|
||||
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
|
||||
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
|
||||
// UI can render an "unavailable in this deployment" state instead of an
|
||||
// empty table. Pool concurrency is the configured per-process capacity
|
||||
// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
|
||||
// capacity (see QueueStat.Active for currently running tasks).
|
||||
// empty table. Each pool reports configured per-process concurrency plus live
|
||||
// cluster capacity/active workers aggregated from asynq server heartbeats.
|
||||
type RuntimeWorkerPool struct {
|
||||
Name string `json:"name"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
QueueCount int `json:"queue_count"`
|
||||
Name string `json:"name"`
|
||||
Concurrency int `json:"concurrency"` // configured per instance
|
||||
QueueCount int `json:"queue_count"`
|
||||
Instances int `json:"instances"` // live asynq server heartbeats
|
||||
ClusterCapacity int `json:"cluster_capacity"` // sum of live server concurrency
|
||||
Active int `json:"active"` // live workers assigned to this pool
|
||||
Utilization float64 `json:"utilization"` // Active / ClusterCapacity
|
||||
}
|
||||
|
||||
type RuntimeQueuesResponse struct {
|
||||
@@ -1406,6 +1409,56 @@ type RuntimeQueuesResponse struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
func aggregateRuntimeWorkerPools(pools []RuntimeWorkerPool, servers []types.WorkerServerStat) {
|
||||
queueConfigs := map[string]map[string]int{
|
||||
types.WorkerPoolCore: types.QueueWeightsForPool(types.WorkerPoolCore),
|
||||
types.WorkerPoolPostProcess: types.QueueWeightsForPool(types.WorkerPoolPostProcess),
|
||||
types.WorkerPoolEnrichment: types.QueueWeightsForPool(types.WorkerPoolEnrichment),
|
||||
types.WorkerPoolMaintenance: types.QueueWeightsForPool(types.WorkerPoolMaintenance),
|
||||
types.WorkerPoolShared: types.QueueWeightsForSharedPool(),
|
||||
types.WorkerPoolWiki: types.QueueWeightsForPool(types.WorkerPoolWiki),
|
||||
}
|
||||
indexes := make(map[string]int, len(pools))
|
||||
for i := range pools {
|
||||
indexes[pools[i].Name] = i
|
||||
}
|
||||
for _, server := range servers {
|
||||
if server.Status != "active" {
|
||||
continue
|
||||
}
|
||||
for poolName, expectedQueues := range queueConfigs {
|
||||
if !sameQueueWeights(server.Queues, expectedQueues) {
|
||||
continue
|
||||
}
|
||||
idx, ok := indexes[poolName]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
pools[idx].Instances++
|
||||
pools[idx].ClusterCapacity += server.Concurrency
|
||||
pools[idx].Active += server.Active
|
||||
break
|
||||
}
|
||||
}
|
||||
for i := range pools {
|
||||
if pools[i].ClusterCapacity > 0 {
|
||||
pools[i].Utilization = float64(pools[i].Active) / float64(pools[i].ClusterCapacity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sameQueueWeights(left, right map[string]int) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for name, weight := range left {
|
||||
if right[name] != weight {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// GetRuntimeQueues godoc
|
||||
// @Summary 获取解析任务队列运行时状态
|
||||
// @Description 返回各 asynq 队列的实时深度(pending/active/scheduled/retry 等)与 worker 并发配置,仅系统管理员可见
|
||||
@@ -1415,34 +1468,30 @@ type RuntimeQueuesResponse struct {
|
||||
// @Router /system/admin/runtime/queues [get]
|
||||
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
ctx := logger.CloneContext(c.Request.Context())
|
||||
upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
|
||||
wikiConcurrency := types.DefaultWikiWorkerConcurrency
|
||||
var allocation types.WorkerPoolConcurrency
|
||||
if h.systemSettingSvc != nil {
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
upstreamConcurrency = int(configured)
|
||||
}
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
wikiConcurrency = int(configured)
|
||||
}
|
||||
allocation = types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int {
|
||||
configured := h.systemSettingSvc.GetInt(ctx, key, env, int64(fallback))
|
||||
return int(configured)
|
||||
})
|
||||
} else {
|
||||
allocation = types.DefaultWorkerPoolConcurrency()
|
||||
}
|
||||
allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
|
||||
queueCounts := make(map[string]int)
|
||||
for _, definition := range types.QueueDefinitions() {
|
||||
queueCounts[definition.Pool]++
|
||||
}
|
||||
resp := RuntimeQueuesResponse{
|
||||
UpstreamConcurrency: allocation.Total,
|
||||
ParseConcurrency: allocation.Total,
|
||||
WikiConcurrency: wikiConcurrency,
|
||||
UpstreamConcurrency: allocation.UpstreamTotal(),
|
||||
ParseConcurrency: allocation.UpstreamTotal(),
|
||||
WikiConcurrency: allocation.Wiki,
|
||||
Pools: []RuntimeWorkerPool{
|
||||
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
|
||||
{Name: types.WorkerPoolPostProcess, Concurrency: allocation.PostProcess, QueueCount: queueCounts[types.WorkerPoolPostProcess]},
|
||||
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
|
||||
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
|
||||
{Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
|
||||
{Name: types.WorkerPoolShared, Concurrency: allocation.Shared, QueueCount: len(types.QueueWeightsForSharedPool())},
|
||||
{Name: types.WorkerPoolWiki, Concurrency: allocation.Wiki, QueueCount: queueCounts[types.WorkerPoolWiki]},
|
||||
},
|
||||
Timestamp: time.Now().Unix(),
|
||||
}
|
||||
@@ -1456,6 +1505,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
if stats != nil {
|
||||
resp.Queues = stats
|
||||
}
|
||||
serverStats, _, serverErr := h.taskInspector.WorkerServerStats(ctx)
|
||||
if serverErr != nil {
|
||||
logger.Errorf(ctx, "get worker server stats failed: %v", serverErr)
|
||||
} else {
|
||||
aggregateRuntimeWorkerPools(resp.Pools, serverStats)
|
||||
}
|
||||
}
|
||||
// Always emit a non-nil array so the JSON serialises to `[]` rather
|
||||
// than `null`, keeping the frontend iteration simple.
|
||||
|
||||
@@ -15,10 +15,18 @@ type runtimeTestSettings struct{}
|
||||
|
||||
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
|
||||
switch key {
|
||||
case "asynq.concurrency":
|
||||
return 32
|
||||
case "asynq.core_concurrency":
|
||||
return 8
|
||||
case "asynq.postprocess_concurrency":
|
||||
return 2
|
||||
case "asynq.enrichment_concurrency":
|
||||
return 12
|
||||
case "asynq.maintenance_concurrency":
|
||||
return 4
|
||||
case "asynq.shared_concurrency":
|
||||
return 6
|
||||
case "asynq.wiki_concurrency":
|
||||
return 16
|
||||
return 8
|
||||
default:
|
||||
return def
|
||||
}
|
||||
@@ -55,6 +63,17 @@ func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string)
|
||||
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
|
||||
return []types.QueueStat{}, true, nil
|
||||
}
|
||||
func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
|
||||
return []types.WorkerServerStat{
|
||||
{Concurrency: 8, Active: 4, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
|
||||
{Concurrency: 2, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolPostProcess)},
|
||||
{Concurrency: 12, Active: 6, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolEnrichment)},
|
||||
{Concurrency: 4, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolMaintenance)},
|
||||
{Concurrency: 6, Active: 3, Status: "active", Queues: types.QueueWeightsForSharedPool()},
|
||||
{Concurrency: 8, Active: 2, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolWiki)},
|
||||
{Concurrency: 99, Active: 0, Status: "stopped", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
@@ -85,10 +104,12 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
concurrency int
|
||||
queueCount int
|
||||
}{
|
||||
types.WorkerPoolCore: {16, 1},
|
||||
types.WorkerPoolCore: {8, 1},
|
||||
types.WorkerPoolPostProcess: {2, 1},
|
||||
types.WorkerPoolEnrichment: {12, 4},
|
||||
types.WorkerPoolMaintenance: {4, 2},
|
||||
types.WorkerPoolWiki: {16, 1},
|
||||
types.WorkerPoolShared: {6, 5},
|
||||
types.WorkerPoolWiki: {8, 1},
|
||||
}
|
||||
if len(response.Pools) != len(want) {
|
||||
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
|
||||
@@ -102,6 +123,9 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
|
||||
pool.Name, pool, expected.concurrency, expected.queueCount)
|
||||
}
|
||||
if pool.Instances != 1 || pool.ClusterCapacity != expected.concurrency {
|
||||
t.Fatalf("pool %q live capacity = %+v", pool.Name, pool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-30
@@ -24,13 +24,15 @@ import (
|
||||
type AsynqTaskParams struct {
|
||||
dig.In
|
||||
|
||||
// Four independent servers provide hard capacity isolation between the
|
||||
// user-facing parse path, high-fanout enrichment, maintenance/sync work,
|
||||
// and Wiki generation. They share one handler mux but subscribe to disjoint
|
||||
// queue sets from types.QueueDefinitions.
|
||||
// Dedicated servers provide minimum capacity for parse, post-process,
|
||||
// enrichment, maintenance, and Wiki work. SharedServer is the elastic tier:
|
||||
// it also subscribes to core/enrichment queues so either stage can borrow
|
||||
// otherwise-idle capacity without consuming the other stage's guarantee.
|
||||
CoreServer *asynq.Server `name:"coreAsynqServer"`
|
||||
PostProcessServer *asynq.Server `name:"postProcessAsynqServer"`
|
||||
EnrichmentServer *asynq.Server `name:"enrichmentAsynqServer"`
|
||||
MaintenanceServer *asynq.Server `name:"maintenanceAsynqServer"`
|
||||
SharedServer *asynq.Server `name:"sharedAsynqServer"`
|
||||
WikiServer *asynq.Server `name:"wikiAsynqServer"`
|
||||
KnowledgeService interfaces.KnowledgeService
|
||||
KnowledgeBaseService interfaces.KnowledgeBaseService
|
||||
@@ -151,32 +153,41 @@ func backgroundTaskMiddleware() asynq.MiddlewareFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func resolveUpstreamPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency {
|
||||
total := types.DefaultUpstreamWorkerConcurrency
|
||||
if svc != nil {
|
||||
n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency)
|
||||
if n > 0 {
|
||||
total = int(n)
|
||||
}
|
||||
func resolveWorkerPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency {
|
||||
if svc == nil {
|
||||
return types.DefaultWorkerPoolConcurrency()
|
||||
}
|
||||
return types.AllocateWorkerPoolConcurrency(total)
|
||||
ctx := context.Background()
|
||||
return types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int {
|
||||
value := svc.GetInt(ctx, key, env, int64(fallback))
|
||||
return int(value)
|
||||
})
|
||||
}
|
||||
|
||||
// NewCoreAsynqServer runs the latency-sensitive document parse and
|
||||
// post-process orchestration path.
|
||||
// NewCoreAsynqServer runs document and manual parsing with guaranteed capacity.
|
||||
func NewCoreAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
allocation := resolveWorkerPoolConcurrency(svc)
|
||||
log.Printf("asynq core-pool server starting with concurrency=%d total_upstream=%d redis_op_timeout=%dms",
|
||||
allocation.Core, allocation.Total, readRedisOpTimeoutMs())
|
||||
allocation.Core, allocation.UpstreamTotal(), readRedisOpTimeoutMs())
|
||||
return newAsynqServer(allocation.Core, types.QueueWeightsForPool(types.WorkerPoolCore))
|
||||
}
|
||||
|
||||
// NewPostProcessAsynqServer reserves capacity for the lightweight but
|
||||
// latency-sensitive fan-out stage. It cannot be trapped behind a burst of
|
||||
// long-running document parses in QueueDefault.
|
||||
func NewPostProcessAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveWorkerPoolConcurrency(svc)
|
||||
log.Printf("asynq postprocess-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.PostProcess, allocation.UpstreamTotal())
|
||||
return newAsynqServer(allocation.PostProcess, types.QueueWeightsForPool(types.WorkerPoolPostProcess))
|
||||
}
|
||||
|
||||
// NewEnrichmentAsynqServer runs high-fanout summary, multimodal, graph, and
|
||||
// question generation without consuming core parsing capacity.
|
||||
func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
allocation := resolveWorkerPoolConcurrency(svc)
|
||||
log.Printf("asynq enrichment-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.Enrichment, allocation.Total)
|
||||
allocation.Enrichment, allocation.UpstreamTotal())
|
||||
return newAsynqServer(allocation.Enrichment, types.QueueWeightsForPool(types.WorkerPoolEnrichment))
|
||||
}
|
||||
|
||||
@@ -184,25 +195,29 @@ func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server
|
||||
// dispatch work. QueueMaintenance keeps the legacy Redis name "low", so old
|
||||
// tasks are drained safely during rolling upgrades.
|
||||
func NewMaintenanceAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveUpstreamPoolConcurrency(svc)
|
||||
allocation := resolveWorkerPoolConcurrency(svc)
|
||||
log.Printf("asynq maintenance-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.Maintenance, allocation.Total)
|
||||
allocation.Maintenance, allocation.UpstreamTotal())
|
||||
return newAsynqServer(allocation.Maintenance, types.QueueWeightsForPool(types.WorkerPoolMaintenance))
|
||||
}
|
||||
|
||||
// NewSharedAsynqServer is the elastic tier. Asynq dequeue is atomic, so it is
|
||||
// safe for this server and the dedicated servers to subscribe to the same
|
||||
// queues: every task is still executed by exactly one worker.
|
||||
func NewSharedAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
allocation := resolveWorkerPoolConcurrency(svc)
|
||||
log.Printf("asynq shared-pool server starting with concurrency=%d total_upstream=%d",
|
||||
allocation.Shared, allocation.UpstreamTotal())
|
||||
return newAsynqServer(allocation.Shared, types.QueueWeightsForSharedPool())
|
||||
}
|
||||
|
||||
// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. It runs
|
||||
// the shared handler mux but only ever pulls wiki tasks, so its concurrency
|
||||
// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on
|
||||
// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 8) is spent exclusively on
|
||||
// wiki generation. This is the hard capacity isolation that prevents the parse
|
||||
// pipeline from starving wiki (and vice-versa) during concurrent uploads.
|
||||
func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server {
|
||||
concurrency := types.DefaultWikiWorkerConcurrency
|
||||
if svc != nil {
|
||||
n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency)
|
||||
if n > 0 {
|
||||
concurrency = int(n)
|
||||
}
|
||||
}
|
||||
concurrency := resolveWorkerPoolConcurrency(svc).Wiki
|
||||
log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency)
|
||||
return newAsynqServer(concurrency, types.QueueWeightsForPool(types.WorkerPoolWiki))
|
||||
}
|
||||
@@ -293,8 +308,8 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
|
||||
mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle)
|
||||
|
||||
// Run the same mux on every pool. Queue definitions are disjoint, so a given
|
||||
// task is dequeued by exactly one pool while handlers stay registered once.
|
||||
// Run the same mux on every pool. Shared and dedicated servers intentionally
|
||||
// overlap, but Redis dequeue is atomic, so each task still executes once.
|
||||
runPool := func(name string, srv *asynq.Server) {
|
||||
go func() {
|
||||
if err := srv.Run(mux); err != nil {
|
||||
@@ -303,8 +318,10 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
}()
|
||||
}
|
||||
runPool("core-pool", params.CoreServer)
|
||||
runPool("postprocess-pool", params.PostProcessServer)
|
||||
runPool("enrichment-pool", params.EnrichmentServer)
|
||||
runPool("maintenance-pool", params.MaintenanceServer)
|
||||
runPool("shared-pool", params.SharedServer)
|
||||
runPool("wiki-pool", params.WikiServer)
|
||||
return mux
|
||||
}
|
||||
|
||||
@@ -183,6 +183,35 @@ func (a *asynqTaskInspector) QueueStats(
|
||||
return stats, true, nil
|
||||
}
|
||||
|
||||
func (a *asynqTaskInspector) WorkerServerStats(
|
||||
ctx context.Context,
|
||||
) ([]types.WorkerServerStat, bool, error) {
|
||||
if a == nil || a.inspector == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
servers, err := a.inspector.Servers()
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
stats := make([]types.WorkerServerStat, 0, len(servers))
|
||||
for _, server := range servers {
|
||||
if server == nil {
|
||||
continue
|
||||
}
|
||||
queues := make(map[string]int, len(server.Queues))
|
||||
for name, weight := range server.Queues {
|
||||
queues[name] = weight
|
||||
}
|
||||
stats = append(stats, types.WorkerServerStat{
|
||||
Concurrency: server.Concurrency,
|
||||
Active: len(server.ActiveWorkers),
|
||||
Status: server.Status,
|
||||
Queues: queues,
|
||||
})
|
||||
}
|
||||
return stats, true, nil
|
||||
}
|
||||
|
||||
// queueStateHasMatch pages through one (queue, state) list looking for a
|
||||
// task that references knowledgeID. Mirrors the delete* scanners but is
|
||||
// strictly read-only and returns early on the first hit. A backend error
|
||||
@@ -390,3 +419,9 @@ func (noopTaskInspector) QueueStats(
|
||||
) ([]types.QueueStat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func (noopTaskInspector) WorkerServerStats(
|
||||
ctx context.Context,
|
||||
) ([]types.WorkerServerStat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
@@ -57,4 +57,9 @@ type TaskInspector interface {
|
||||
// surfaced as a zeroed row (so the full lane set stays visible) rather
|
||||
// than failing the whole call.
|
||||
QueueStats(ctx context.Context) (stats []types.QueueStat, supported bool, err error)
|
||||
|
||||
// WorkerServerStats returns live asynq server heartbeats across all
|
||||
// replicas. The runtime dashboard uses them to aggregate actual cluster
|
||||
// capacity and busy workers for each configured pool.
|
||||
WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error)
|
||||
}
|
||||
|
||||
+99
-52
@@ -5,15 +5,24 @@ package types
|
||||
// between pools instead of being only a weighted dequeue preference.
|
||||
const (
|
||||
WorkerPoolCore = "core"
|
||||
WorkerPoolPostProcess = "postprocess"
|
||||
WorkerPoolEnrichment = "enrichment"
|
||||
WorkerPoolMaintenance = "maintenance"
|
||||
WorkerPoolShared = "shared"
|
||||
WorkerPoolWiki = "wiki"
|
||||
|
||||
// DefaultUpstreamWorkerConcurrency is split across core, enrichment,
|
||||
// and maintenance. Wiki stays separately configurable because its model
|
||||
// workload has a different capacity profile.
|
||||
DefaultUpstreamWorkerConcurrency = 32
|
||||
DefaultWikiWorkerConcurrency = 16
|
||||
// Upstream defaults are explicit guarantees plus an elastic pool. The
|
||||
// shared pool may consume core and enrichment queues, so idle capacity in
|
||||
// either stage can be borrowed without sacrificing the dedicated minimums.
|
||||
DefaultCoreWorkerConcurrency = 8
|
||||
DefaultPostProcessWorkerConcurrency = 2
|
||||
DefaultEnrichmentWorkerConcurrency = 12
|
||||
DefaultMaintenanceWorkerConcurrency = 4
|
||||
DefaultSharedWorkerConcurrency = 6
|
||||
DefaultWikiWorkerConcurrency = 8
|
||||
DefaultUpstreamWorkerConcurrency = DefaultCoreWorkerConcurrency +
|
||||
DefaultPostProcessWorkerConcurrency + DefaultEnrichmentWorkerConcurrency +
|
||||
DefaultMaintenanceWorkerConcurrency + DefaultSharedWorkerConcurrency
|
||||
)
|
||||
|
||||
// Asynq queue names. QueueMaintenance intentionally keeps the physical Redis
|
||||
@@ -21,6 +30,7 @@ const (
|
||||
// rolling deployment. New code uses the business-semantic constant.
|
||||
const (
|
||||
QueueDefault = "default"
|
||||
QueuePostProcess = "postprocess"
|
||||
QueueSummary = "summary"
|
||||
QueueMultimodal = "multimodal"
|
||||
QueueGraph = "graph"
|
||||
@@ -35,22 +45,26 @@ const (
|
||||
// scheduling weights shown to operators from drifting from the actual server
|
||||
// configuration.
|
||||
type QueueDefinition struct {
|
||||
Name string
|
||||
Pool string
|
||||
Weight int
|
||||
TaskTypes []string
|
||||
Name string
|
||||
Pool string
|
||||
Weight int
|
||||
SharedWeight int
|
||||
TaskTypes []string
|
||||
}
|
||||
|
||||
var queueDefinitions = []QueueDefinition{
|
||||
{Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, TaskTypes: []string{
|
||||
TypeDocumentProcess, TypeManualProcess, TypeKnowledgePostProcess,
|
||||
{Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, SharedWeight: 3, TaskTypes: []string{
|
||||
TypeDocumentProcess, TypeManualProcess,
|
||||
}},
|
||||
{Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, TaskTypes: []string{
|
||||
{Name: QueuePostProcess, Pool: WorkerPoolPostProcess, Weight: 1, TaskTypes: []string{
|
||||
TypeKnowledgePostProcess,
|
||||
}},
|
||||
{Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, SharedWeight: 2, TaskTypes: []string{
|
||||
TypeSummaryGeneration, TypeDataTableSummary,
|
||||
}},
|
||||
{Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeImageMultimodal}},
|
||||
{Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeChunkExtract}},
|
||||
{Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeQuestionGeneration}},
|
||||
{Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeImageMultimodal}},
|
||||
{Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeChunkExtract}},
|
||||
{Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeQuestionGeneration}},
|
||||
{Name: QueueSync, Pool: WorkerPoolMaintenance, Weight: 2, TaskTypes: []string{TypeDataSourceSync}},
|
||||
{Name: QueueMaintenance, Pool: WorkerPoolMaintenance, Weight: 1, TaskTypes: []string{
|
||||
TypeFAQImport, TypeKBClone, TypeIndexDelete, TypeKBDelete,
|
||||
@@ -95,49 +109,71 @@ func QueueWeightsForPool(pool string) map[string]int {
|
||||
return weights
|
||||
}
|
||||
|
||||
// WorkerPoolConcurrency is the deterministic split of the total upstream
|
||||
// worker budget. Keeping this pure allocation function in types lets the
|
||||
// server constructors and the System Admin API report identical values.
|
||||
type WorkerPoolConcurrency struct {
|
||||
Total int
|
||||
Core int
|
||||
Enrichment int
|
||||
Maintenance int
|
||||
}
|
||||
|
||||
// AllocateWorkerPoolConcurrency reserves 1/2 of the upstream budget for the
|
||||
// latency-sensitive core pipeline, 3/8 for LLM/VLM enrichment, and the
|
||||
// remainder for sync/cleanup/batch work. A minimum total of three guarantees
|
||||
// every independent server has at least one worker.
|
||||
func AllocateWorkerPoolConcurrency(total int) WorkerPoolConcurrency {
|
||||
if total < 3 {
|
||||
total = 3
|
||||
}
|
||||
core := total / 2
|
||||
enrichment := (total * 3) / 8
|
||||
if core < 1 {
|
||||
core = 1
|
||||
}
|
||||
if enrichment < 1 {
|
||||
enrichment = 1
|
||||
}
|
||||
maintenance := total - core - enrichment
|
||||
if maintenance < 1 {
|
||||
maintenance = 1
|
||||
if core >= enrichment && core > 1 {
|
||||
core--
|
||||
} else if enrichment > 1 {
|
||||
enrichment--
|
||||
// QueueWeightsForSharedPool returns the core/enrichment queues eligible for
|
||||
// elastic capacity. Post-process and maintenance are deliberately excluded:
|
||||
// post-process needs a small latency guarantee, while long maintenance tasks
|
||||
// must not pin burst capacity intended for the user-facing pipeline.
|
||||
func QueueWeightsForSharedPool() map[string]int {
|
||||
weights := make(map[string]int)
|
||||
for _, definition := range queueDefinitions {
|
||||
if definition.SharedWeight > 0 {
|
||||
weights[definition.Name] = definition.SharedWeight
|
||||
}
|
||||
}
|
||||
return weights
|
||||
}
|
||||
|
||||
// WorkerPoolConcurrency contains the explicit per-instance pool capacities.
|
||||
// Unlike the old ratio allocator, each field is independently configurable.
|
||||
type WorkerPoolConcurrency struct {
|
||||
Core int
|
||||
PostProcess int
|
||||
Enrichment int
|
||||
Maintenance int
|
||||
Shared int
|
||||
Wiki int
|
||||
}
|
||||
|
||||
func DefaultWorkerPoolConcurrency() WorkerPoolConcurrency {
|
||||
return WorkerPoolConcurrency{
|
||||
Total: total,
|
||||
Core: core,
|
||||
Enrichment: enrichment,
|
||||
Maintenance: maintenance,
|
||||
Core: DefaultCoreWorkerConcurrency,
|
||||
PostProcess: DefaultPostProcessWorkerConcurrency,
|
||||
Enrichment: DefaultEnrichmentWorkerConcurrency,
|
||||
Maintenance: DefaultMaintenanceWorkerConcurrency,
|
||||
Shared: DefaultSharedWorkerConcurrency,
|
||||
Wiki: DefaultWikiWorkerConcurrency,
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveWorkerPoolConcurrency centralizes setting keys, environment names,
|
||||
// defaults, and invalid-value fallback for both server construction and the
|
||||
// runtime API. The callback lets this package stay independent of the system
|
||||
// setting service interface.
|
||||
func ResolveWorkerPoolConcurrency(read func(key, env string, fallback int) int) WorkerPoolConcurrency {
|
||||
allocation := DefaultWorkerPoolConcurrency()
|
||||
if read == nil {
|
||||
return allocation
|
||||
}
|
||||
positive := func(key, env string, fallback int) int {
|
||||
value := read(key, env, fallback)
|
||||
if value < 1 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
allocation.Core = positive("asynq.core_concurrency", "WEKNORA_ASYNQ_CORE_CONCURRENCY", allocation.Core)
|
||||
allocation.PostProcess = positive("asynq.postprocess_concurrency", "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY", allocation.PostProcess)
|
||||
allocation.Enrichment = positive("asynq.enrichment_concurrency", "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY", allocation.Enrichment)
|
||||
allocation.Maintenance = positive("asynq.maintenance_concurrency", "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY", allocation.Maintenance)
|
||||
allocation.Shared = positive("asynq.shared_concurrency", "WEKNORA_ASYNQ_SHARED_CONCURRENCY", allocation.Shared)
|
||||
allocation.Wiki = positive("asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", allocation.Wiki)
|
||||
return allocation
|
||||
}
|
||||
|
||||
func (c WorkerPoolConcurrency) UpstreamTotal() int {
|
||||
return c.Core + c.PostProcess + c.Enrichment + c.Maintenance + c.Shared
|
||||
}
|
||||
|
||||
// QueueStat is a read-only depth snapshot of a single asynq queue, used
|
||||
// by the System Admin runtime dashboard. Field names mirror the counts
|
||||
// exposed by asynq.QueueInfo. Pool / Weight are static metadata (which
|
||||
@@ -170,6 +206,17 @@ type QueueStat struct {
|
||||
MemoryUsageBytes int64 `json:"memory_usage_bytes"`
|
||||
}
|
||||
|
||||
// WorkerServerStat is one live asynq server heartbeat. Queue weights identify
|
||||
// which logical pool the server belongs to; the runtime handler aggregates
|
||||
// these records across replicas so configured per-instance capacity is not
|
||||
// confused with actual cluster capacity.
|
||||
type WorkerServerStat struct {
|
||||
Concurrency int
|
||||
Active int
|
||||
Status string
|
||||
Queues map[string]int
|
||||
}
|
||||
|
||||
const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
|
||||
@@ -10,6 +10,7 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) {
|
||||
|
||||
validPools := map[string]bool{
|
||||
WorkerPoolCore: true,
|
||||
WorkerPoolPostProcess: true,
|
||||
WorkerPoolEnrichment: true,
|
||||
WorkerPoolMaintenance: true,
|
||||
WorkerPoolWiki: true,
|
||||
@@ -47,6 +48,13 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) {
|
||||
t.Fatalf("worker pool %q has no queues", pool)
|
||||
}
|
||||
}
|
||||
shared := QueueWeightsForSharedPool()
|
||||
if len(shared) == 0 || shared[QueueDefault] <= 0 || shared[QueueSummary] <= 0 {
|
||||
t.Fatalf("shared pool must cover core and enrichment queues: %+v", shared)
|
||||
}
|
||||
if shared[QueuePostProcess] != 0 || shared[QueueMaintenance] != 0 {
|
||||
t.Fatalf("shared pool must not consume post-process or maintenance: %+v", shared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueMaintenanceKeepsLegacyPhysicalName(t *testing.T) {
|
||||
@@ -71,22 +79,31 @@ func TestEveryAsynqTaskTypeHasADeclaredQueue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerPoolConcurrencyPreservesBudget(t *testing.T) {
|
||||
for _, total := range []int{3, 4, 8, 32, 64} {
|
||||
allocation := AllocateWorkerPoolConcurrency(total)
|
||||
if allocation.Total != total {
|
||||
t.Fatalf("total=%d: allocation reported total %d", total, allocation.Total)
|
||||
}
|
||||
if allocation.Core < 1 || allocation.Enrichment < 1 || allocation.Maintenance < 1 {
|
||||
t.Fatalf("total=%d: every pool must receive capacity: %+v", total, allocation)
|
||||
}
|
||||
if allocation.Core+allocation.Enrichment+allocation.Maintenance != total {
|
||||
t.Fatalf("total=%d: split does not preserve budget: %+v", total, allocation)
|
||||
}
|
||||
func TestDefaultWorkerPoolConcurrencyIsExplicitBudget(t *testing.T) {
|
||||
allocation := DefaultWorkerPoolConcurrency()
|
||||
if allocation.UpstreamTotal() != DefaultUpstreamWorkerConcurrency {
|
||||
t.Fatalf("upstream total = %d, want %d", allocation.UpstreamTotal(), DefaultUpstreamWorkerConcurrency)
|
||||
}
|
||||
|
||||
minimum := AllocateWorkerPoolConcurrency(1)
|
||||
if minimum.Total != 3 || minimum.Core+minimum.Enrichment+minimum.Maintenance != 3 {
|
||||
t.Fatalf("undersized budget should clamp to three workers: %+v", minimum)
|
||||
if allocation.Core < 1 || allocation.PostProcess < 1 || allocation.Enrichment < 1 ||
|
||||
allocation.Maintenance < 1 || allocation.Shared < 1 || allocation.Wiki < 1 {
|
||||
t.Fatalf("every pool must have positive explicit capacity: %+v", allocation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWorkerPoolConcurrencyFallsBackPerPool(t *testing.T) {
|
||||
allocation := ResolveWorkerPoolConcurrency(func(key, _ string, fallback int) int {
|
||||
if key == "asynq.core_concurrency" {
|
||||
return 15
|
||||
}
|
||||
if key == "asynq.shared_concurrency" {
|
||||
return 0
|
||||
}
|
||||
return fallback
|
||||
})
|
||||
if allocation.Core != 15 {
|
||||
t.Fatalf("core = %d, want override 15", allocation.Core)
|
||||
}
|
||||
if allocation.Shared != DefaultSharedWorkerConcurrency {
|
||||
t.Fatalf("shared = %d, want fallback %d", allocation.Shared, DefaultSharedWorkerConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user