From 07ba01ec93129eb8e1d005c1ba1bffe1201f7718 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 13 Jul 2026 14:55:59 +0800 Subject: [PATCH] feat(worker): replace aggregate asynq concurrency with per-pool governance Split the monolithic upstream worker budget into explicit core, post-process, enrichment, maintenance, and shared elastic pools, expose live cluster capacity on the runtime dashboard, and retire the legacy asynq.concurrency setting. --- .env.example | 31 ++-- docker-compose.yml | 8 +- docs/worker-pool-governance.md | 85 ++++++++++ frontend/src/api/system/index.ts | 10 +- frontend/src/i18n/locales/en-US.ts | 32 +++- frontend/src/i18n/locales/ko-KR.ts | 32 +++- frontend/src/i18n/locales/ru-RU.ts | 32 +++- frontend/src/i18n/locales/zh-CN.ts | 34 ++-- frontend/src/views/system/RuntimeQueues.vue | 15 +- frontend/src/views/system/SystemSettings.vue | 3 +- .../application/service/image_multimodal.go | 2 +- .../service/knowledge_housekeeping.go | 5 +- .../service/knowledge_housekeeping_test.go | 6 + .../application/service/knowledge_process.go | 2 +- .../service/knowledge_task_options.go | 10 ++ .../service/knowledge_task_options_test.go | 9 ++ .../application/service/system_setting.go | 66 +++++--- .../service/system_setting_validation_test.go | 9 +- internal/application/service/wiki_ingest.go | 2 +- internal/container/container.go | 9 +- internal/handler/system.go | 101 +++++++++--- internal/handler/system_runtime_test.go | 34 +++- internal/router/task.go | 77 +++++---- internal/router/task_inspector.go | 35 ++++ internal/types/interfaces/task_inspector.go | 5 + internal/types/task.go | 151 ++++++++++++------ internal/types/task_queue_test.go | 49 ++++-- 27 files changed, 645 insertions(+), 209 deletions(-) create mode 100644 docs/worker-pool-governance.md diff --git a/.env.example b/.env.example index 4f4c5d35..f492c7a8 100644 --- a/.env.example +++ b/.env.example @@ -716,22 +716,35 @@ DOCREADER_TRANSPORT=grpc # WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h # --- Async pipeline tuning (optional) ----------------------------------------- -# Worker pool size for the PARSE asynq server (document parse / post-process / -# embedding / multimodal / graph / question — everything except wiki). Default -# 32 — raise it on machines that handle many concurrent uploads (default Go -# runtime.NumCPU() under-provisions for the I/O-bound document pipeline). Can -# also be set in the management UI under Settings → System settings -# (asynq.concurrency); UI changes require a process restart. -# WEKNORA_ASYNQ_CONCURRENCY=32 +# Per-instance guaranteed worker capacity for document parse / manual reparse. +# Core and enrichment can additionally borrow from the shared elastic pool. +# WEKNORA_ASYNQ_CORE_CONCURRENCY=8 + +# Dedicated lightweight post-process/fan-out workers. Keeping this separate +# prevents completed parses from waiting behind long DocReader jobs. +# WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=2 + +# Per-instance guaranteed capacity for summary, image, graph, and question +# enrichment tasks. +# WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=12 + +# Per-instance capacity for datasource sync, batch operations, and cleanup. +# Maintenance is not eligible for shared burst capacity because its tasks can +# be long-running. +# WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=4 + +# Elastic capacity shared by core parse and enrichment queues. Queue dequeue is +# atomic, so these workers safely supplement both dedicated pools. +# WEKNORA_ASYNQ_SHARED_CONCURRENCY=6 # Worker pool size for the DEDICATED wiki asynq server. The wiki generation # pipeline runs on its own pool with its own concurrency budget, hard-isolated # from the parse pool above: heavy document parsing can never starve wiki # generation during upload storms, and a wiki burst can never starve -# user-facing parsing. Default 16. Wiki work is synthesis-LLM-bound, so size +# user-facing parsing. Default 8. Wiki work is synthesis-LLM-bound, so size # this to your LLM provider's concurrency budget. Also settable in the UI # (asynq.wiki_concurrency); requires a process restart. -# WEKNORA_WIKI_ASYNQ_CONCURRENCY=16 +# WEKNORA_WIKI_ASYNQ_CONCURRENCY=8 # Default per-model concurrency cap for background tasks (document ingest / # enrichment). Shared across replicas, keyed by model ID. Only affects diff --git a/docker-compose.yml b/docker-compose.yml index bfcec553..33b7090e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,8 +195,12 @@ services: - WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-} - WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-} - WEKNORA_TENANT_AUTO_CREATE_API_KEY=${WEKNORA_TENANT_AUTO_CREATE_API_KEY:-} - - WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32} - - WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-16} + - WEKNORA_ASYNQ_CORE_CONCURRENCY=${WEKNORA_ASYNQ_CORE_CONCURRENCY:-8} + - WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY=${WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY:-2} + - WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY=${WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY:-12} + - WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY=${WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY:-4} + - WEKNORA_ASYNQ_SHARED_CONCURRENCY=${WEKNORA_ASYNQ_SHARED_CONCURRENCY:-6} + - WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-8} - WEKNORA_MODEL_MAX_CONCURRENCY=${WEKNORA_MODEL_MAX_CONCURRENCY:-32} - APK_MIRROR_ARG=${APK_MIRROR_ARG:-} - WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-} diff --git a/docs/worker-pool-governance.md b/docs/worker-pool-governance.md new file mode 100644 index 00000000..c9343452 --- /dev/null +++ b/docs/worker-pool-governance.md @@ -0,0 +1,85 @@ +# Worker Pool Governance + +WeKnora uses guaranteed per-stage worker pools plus an elastic pool for the +document ingestion pipeline. Worker concurrency is a scheduling budget, not a +replacement for model quotas, DocReader capacity, vector-store limits, or +database connection limits. + +## Topology + +| Pool | Default per instance | Queues | Purpose | +| --- | ---: | --- | --- | +| Core | 8 | `default` | Document parsing and manual reparse guarantee | +| Post-process | 2 | `postprocess` | Parse finalization and enrichment fan-out | +| Enrichment | 12 | `summary`, `multimodal`, `graph`, `question` | Model-heavy enrichment guarantee | +| Maintenance | 4 | `sync`, `low` | Source sync, batch work, move/delete/cleanup | +| Shared | 6 | Core and enrichment queues | Elastic capacity borrowed by the side with backlog | +| Wiki | 8 | `wiki` | Independently governed Wiki generation | + +The upstream total remains 32 workers per service instance by default. Wiki is +separate and is not included in that total. + +Post-process has its own physical queue so lightweight fan-out work cannot sit +behind long DocReader calls. Maintenance is intentionally excluded from the +shared pool because its long-running tasks could pin elastic capacity needed by +the user-facing pipeline. + +Asynq dequeue is atomic. Dedicated and shared servers may safely subscribe to +the same core/enrichment queues; one task is still processed by one worker. + +## Configuration + +All settings are available under System settings and require a service restart: + +- `asynq.core_concurrency` / `WEKNORA_ASYNQ_CORE_CONCURRENCY` +- `asynq.postprocess_concurrency` / `WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY` +- `asynq.enrichment_concurrency` / `WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY` +- `asynq.maintenance_concurrency` / `WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY` +- `asynq.shared_concurrency` / `WEKNORA_ASYNQ_SHARED_CONCURRENCY` +- `asynq.wiki_concurrency` / `WEKNORA_WIKI_ASYNQ_CONCURRENCY` + +The old aggregate `asynq.concurrency` / `WEKNORA_ASYNQ_CONCURRENCY` setting is +retired. Deployments that set it must migrate to the explicit pool settings. +Persisted old rows are ignored and hidden from the System settings page so they +cannot be mistaken for an effective runtime control. + +## Capacity layers + +Tune these layers independently: + +1. Worker concurrency controls the number of task handlers admitted per + service instance. +2. Model quota governance controls provider concurrency, RPM, and TPM across + replicas and quota groups. +3. DocReader, vector stores, object storage, Postgres, and local CPU/RAM retain + their own resource limits. + +If model limiter waiting grows while worker queues are busy, adding workers +only creates more waiters. Increase the provider quota or reduce worker +admission instead. + +## Sizing + +For each pool, estimate peak required workers using: + +```text +required workers = ceil(peak task arrival rate * mean task runtime / 0.70) +``` + +Use task arrival rate after fan-out. One parsed document can create one summary, +multiple question batches, one graph task per chunk, and multiple image tasks. +Queue count is not a capacity signal. + +The runtime dashboard reports: + +- configured concurrency per instance; +- live server instance count; +- cluster capacity aggregated from active asynq heartbeats; +- active workers and utilization; +- queue backlog, retries, dead letters, and oldest pending age; +- model concurrency and limiter waiting. + +Increase a pool only when backlog age grows while its downstream dependency has +headroom. Reallocate to enrichment when core stays underutilized and fan-out +queues grow. Reduce core admission when DocReader CPU, memory, or latency is the +bottleneck. diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index 1c7b5762..5b4d5871 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -507,6 +507,10 @@ export interface RuntimeWorkerPool { name: string concurrency: number queue_count: number + instances: number + cluster_capacity: number + active: number + utilization: number } export interface ModelRuntimeStat { @@ -520,9 +524,9 @@ export interface ModelRuntimeStat { /** * Runtime queue dashboard payload. `available` is false in Lite mode * (no Redis/asynq) — render an "unavailable in this deployment" state - * rather than an empty table. Pool concurrency values are CONFIGURED - * per-process capacity (effective on next restart), not live busy-worker - * count or cluster-wide capacity. + * rather than an empty table. Each pool includes both configured per-process + * concurrency and live cluster capacity/active workers aggregated from asynq + * server heartbeats. */ export interface RuntimeQueuesResponse { available: boolean diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index fc5700b2..bae3de32 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -3496,11 +3496,14 @@ export default { detailsTitle: "Queue details", detailsDescription: "Live load and wait time for every processing lane.", poolsTitle: "Worker pools", - poolsDescription: "Core parsing, enrichment, maintenance, and Wiki use isolated concurrency budgets.", - perInstance: "Concurrency is per process", + poolsDescription: "Each stage has guaranteed capacity; core and enrichment may also borrow the shared elastic pool.", + perInstance: "Card value is cluster active/capacity", + poolConfigured: "Per-process {value}", + poolInstances: "{value} instances", + poolUtilization: "{value}% utilized", queueCount: "{value} queues", weight: "Pool weight {value}", - footnote: "Concurrency is a per-process cap (restart required); weight only controls scheduling within one pool; Active is cluster-wide.", + footnote: "Card values show cluster active/live capacity; per-process configuration requires restart. The shared elastic pool only consumes core and enrichment queues.", updatedAt: "Updated at {value}", errors: { generic: "Failed to load queue stats", @@ -3541,18 +3544,23 @@ export default { }, pools: { core: "Core parsing", + postprocess: "Post-process orchestration", enrichment: "Enrichment", maintenance: "Maintenance & sync", + shared: "Shared elastic", wiki: "Wiki pool", }, poolDescriptions: { - core: "Document parsing and post-process orchestration", + core: "Guaranteed document and manual parsing capacity", + postprocess: "Parse finalization and enrichment fan-out", enrichment: "Summaries, images, graph, and question generation", maintenance: "Source sync, batch work, and deletion cleanup", + shared: "Borrowed by core or enrichment according to backlog", wiki: "Wiki content generation and global finalization", }, queueNames: { default: "Default (document parse)", + postprocess: "Post-process orchestration", summary: "Summary generation", sync: "Source sync", low: "Maintenance & batch jobs", @@ -3562,7 +3570,8 @@ export default { wiki: "Wiki ingest", }, queueDescriptions: { - default: "Document parsing, manual updates, and post-process orchestration", + default: "Document parsing and manual updates", + postprocess: "Parse finalization and enrichment task fan-out", summary: "Document and data-table summaries", sync: "Manual and scheduled data-source sync", low: "FAQ import, clone/move, batch reparse, and deletion cleanup", @@ -3584,7 +3593,11 @@ export default { default_storage_quota_gb: 'Default storage quota for new tenants (GB)', }, asynq: { - concurrency: 'Total upstream worker budget', + core_concurrency: 'Guaranteed core parse concurrency', + postprocess_concurrency: 'Post-process concurrency', + enrichment_concurrency: 'Guaranteed enrichment concurrency', + maintenance_concurrency: 'Maintenance concurrency', + shared_concurrency: 'Shared elastic concurrency', wiki_concurrency: 'Wiki worker concurrency', }, model: { @@ -3607,8 +3620,11 @@ export default { 'Default storage quota (GB) assigned when a new tenant is created, covering vectors, originals, text, indexes, and related data. Read only at creation time — changes apply to newly created tenants only and do not retroactively update existing tenants. 0 or a negative value uses the built-in default of 10 GB.', }, asynq: { - concurrency: - 'Total per-process concurrency budget for upstream background work (excluding Wiki). It is split into isolated pools: 1/2 core parsing, 3/8 enrichment, and the remainder maintenance and sync. Minimum 3; requires a service restart.', + core_concurrency: 'Guaranteed per-process concurrency for document and manual parsing. Core may also borrow the shared elastic pool. Minimum 1; requires a service restart.', + postprocess_concurrency: 'Dedicated per-process concurrency for lightweight parse finalization and enrichment fan-out. Minimum 1; requires a service restart.', + enrichment_concurrency: 'Guaranteed per-process concurrency for summaries, images, graph, and question generation. Enrichment may also borrow the shared elastic pool. Minimum 1; requires a service restart.', + maintenance_concurrency: 'Per-process concurrency for source sync, batch work, and cleanup, hard-isolated from the user-facing pipeline. Minimum 1; requires a service restart.', + shared_concurrency: 'Per-process elastic concurrency shared by core parsing and enrichment; whichever side has backlog borrows it automatically. Minimum 1; requires a service restart.', wiki_concurrency: 'Per-process concurrency for the dedicated Wiki worker pool, isolated from upstream tasks. Minimum 1; requires a service restart.', }, diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 33b0aa7d..09fe8140 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2481,11 +2481,14 @@ export default { detailsTitle: "큐 세부 정보", detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.", poolsTitle: "워커 풀", - poolsDescription: "핵심 파싱, 보강, 유지 관리 및 Wiki는 독립된 동시성 예산을 사용합니다.", - perInstance: "동시성은 프로세스별 설정", + poolsDescription: "각 단계에 보장 용량이 있으며 핵심 파싱과 보강은 공유 탄력 풀도 사용할 수 있습니다.", + perInstance: "카드 값은 클러스터 실행 중/용량", + poolConfigured: "프로세스별 {value}", + poolInstances: "인스턴스 {value}개", + poolUtilization: "사용률 {value}%", queueCount: "큐 {value}개", weight: "풀 내 가중치 {value}", - footnote: "동시성은 프로세스별 상한이며 재시작 후 적용됩니다. 가중치는 같은 풀 안의 스케줄링 비율이며 실행 중 수치는 클러스터 전체입니다.", + footnote: "카드는 클러스터 실행 중/실시간 용량을 표시합니다. 프로세스별 설정은 재시작 후 적용되며 공유 풀은 핵심 파싱과 보강 큐만 처리합니다.", updatedAt: "{value} 업데이트됨", errors: { generic: "큐 상태를 불러오지 못했습니다", @@ -2526,18 +2529,23 @@ export default { }, pools: { core: "핵심 파싱", + postprocess: "후처리 오케스트레이션", enrichment: "콘텐츠 보강", maintenance: "유지 관리 및 동기화", + shared: "공유 탄력 풀", wiki: "Wiki 풀", }, poolDescriptions: { - core: "문서 파싱 및 후처리 오케스트레이션", + core: "문서 및 수동 파싱 보장 용량", + postprocess: "파싱 마무리 및 보강 작업 분배", enrichment: "요약, 이미지, 그래프 및 질문 생성", maintenance: "데이터 소스 동기화, 일괄 작업 및 삭제 정리", + shared: "적체에 따라 핵심 파싱 또는 보강이 공유", wiki: "Wiki 콘텐츠 생성 및 전체 마무리", }, queueNames: { default: "기본(문서 파싱)", + postprocess: "후처리 오케스트레이션", summary: "요약 생성", sync: "데이터 소스 동기화", low: "백그라운드 유지 관리 및 일괄 작업", @@ -2547,7 +2555,8 @@ export default { wiki: "Wiki 동기화", }, queueDescriptions: { - default: "문서 파싱, 수동 업데이트 및 후처리 오케스트레이션", + default: "문서 파싱 및 수동 업데이트", + postprocess: "파싱 마무리 및 보강 작업 분배", summary: "문서 및 데이터 테이블 요약", sync: "수동 및 예약 데이터 소스 동기화", low: "FAQ 가져오기, 복제/이동, 일괄 재파싱 및 삭제 정리", @@ -2569,7 +2578,11 @@ export default { default_storage_quota_gb: "신규 테넌트 기본 저장 용량 (GB)", }, asynq: { - concurrency: "상위 워커 총 동시성 예산", + core_concurrency: "핵심 파싱 보장 동시성", + postprocess_concurrency: "후처리 동시성", + enrichment_concurrency: "콘텐츠 보강 보장 동시성", + maintenance_concurrency: "유지 관리 동시성", + shared_concurrency: "공유 탄력 동시성", wiki_concurrency: "Wiki 워커 동시성", }, model: { @@ -2592,8 +2605,11 @@ export default { "신규 테넌트 생성 시 기본으로 할당되는 저장 용량(GB)으로, 벡터·원본·텍스트·인덱스 등을 포함합니다. 생성 시에만 읽으며, 변경은 이후 생성되는 테넌트에만 적용되고 기존 테넌트에는 소급되지 않습니다. 0 또는 음수는 내장 기본값 10GB를 사용합니다.", }, asynq: { - concurrency: - "Wiki를 제외한 상위 백그라운드 작업의 프로세스별 총 동시성 예산입니다. 핵심 파싱 1/2, 콘텐츠 보강 3/8, 나머지는 유지 관리 및 동기화로 분리됩니다. 최소값은 3이며 적용하려면 서비스를 재시작해야 합니다.", + core_concurrency: "문서 및 수동 파싱의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.", + postprocess_concurrency: "파싱 마무리 및 보강 작업 분배 전용 프로세스별 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.", + enrichment_concurrency: "요약, 이미지, 그래프 및 질문 생성의 프로세스별 보장 동시성입니다. 공유 탄력 풀도 사용할 수 있습니다. 최소 1, 서비스 재시작이 필요합니다.", + maintenance_concurrency: "동기화, 일괄 작업 및 정리의 프로세스별 동시성으로 사용자 파이프라인과 격리됩니다. 최소 1, 서비스 재시작이 필요합니다.", + shared_concurrency: "핵심 파싱과 콘텐츠 보강이 적체에 따라 공유하는 프로세스별 탄력 동시성입니다. 최소 1, 서비스 재시작이 필요합니다.", wiki_concurrency: "상위 작업과 분리된 전용 Wiki 워커 풀의 프로세스별 동시성입니다. 최소값은 1이며 적용하려면 서비스를 재시작해야 합니다.", }, diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e54543cc..096459f1 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2179,11 +2179,14 @@ export default { detailsTitle: "Сведения об очередях", detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.", poolsTitle: "Пулы воркеров", - poolsDescription: "Основной разбор, обогащение, обслуживание и Wiki используют независимые бюджеты параллелизма.", - perInstance: "Параллелизм указан на процесс", + poolsDescription: "Каждый этап имеет гарантированную ёмкость; разбор и обогащение также используют общий эластичный пул.", + perInstance: "На карточке активно/ёмкость кластера", + poolConfigured: "На процесс: {value}", + poolInstances: "Экземпляров: {value}", + poolUtilization: "Загрузка {value}%", queueCount: "Очередей: {value}", weight: "Вес в пуле {value}", - footnote: "Параллелизм указан на процесс и требует перезапуска; вес действует только внутри пула; активные задачи показаны по всему кластеру.", + footnote: "Карточки показывают активные задачи/живую ёмкость кластера. Настройки на процесс требуют перезапуска; общий пул обслуживает только разбор и обогащение.", updatedAt: "Обновлено в {value}", errors: { generic: "Не удалось загрузить состояние очередей", @@ -2224,18 +2227,23 @@ export default { }, pools: { core: "Основной разбор", + postprocess: "Оркестрация постобработки", enrichment: "Обогащение", maintenance: "Обслуживание и синхронизация", + shared: "Общий эластичный пул", wiki: "Пул Wiki", }, poolDescriptions: { - core: "Разбор документов и оркестрация постобработки", + core: "Гарантированная ёмкость разбора документов", + postprocess: "Завершение разбора и запуск обогащения", enrichment: "Сводки, изображения, граф и генерация вопросов", maintenance: "Синхронизация источников, пакетные задачи и удаление", + shared: "Используется разбором или обогащением по мере очереди", wiki: "Создание содержимого Wiki и финальная обработка", }, queueNames: { default: "По умолчанию (парсинг документов)", + postprocess: "Оркестрация постобработки", summary: "Создание сводок", sync: "Синхронизация источников", low: "Фоновое обслуживание и пакетные задачи", @@ -2245,7 +2253,8 @@ export default { wiki: "Синхронизация Wiki", }, queueDescriptions: { - default: "Разбор документов, ручные обновления и оркестрация постобработки", + default: "Разбор документов и ручные обновления", + postprocess: "Завершение разбора и запуск задач обогащения", summary: "Сводки документов и таблиц данных", sync: "Ручная и плановая синхронизация источников", low: "Импорт FAQ, клонирование/перемещение, пакетный повторный разбор и удаление", @@ -2267,7 +2276,11 @@ export default { default_storage_quota_gb: 'Квота хранилища для новых тенантов по умолчанию (ГБ)', }, asynq: { - concurrency: 'Общий бюджет фоновых воркеров', + core_concurrency: 'Гарантированный параллелизм разбора', + postprocess_concurrency: 'Параллелизм постобработки', + enrichment_concurrency: 'Гарантированный параллелизм обогащения', + maintenance_concurrency: 'Параллелизм обслуживания', + shared_concurrency: 'Общий эластичный параллелизм', wiki_concurrency: 'Параллелизм воркеров Wiki', }, model: { @@ -2290,8 +2303,11 @@ export default { 'Квота хранилища по умолчанию (ГБ) для нового тенанта: векторы, оригиналы, текст, индексы и т.д. Читается только при создании — изменения применяются только к новым тенантам и не перезаписывают существующие. 0 или отрицательное значение — встроенное значение по умолчанию 10 ГБ.', }, asynq: { - concurrency: - 'Общий бюджет параллелизма фоновых задач на процесс (без Wiki). Он делится на изолированные пулы: 1/2 основной разбор, 3/8 обогащение, остаток — обслуживание и синхронизация. Минимум 3; требуется перезапуск сервиса.', + core_concurrency: 'Гарантированный параллелизм разбора документов на процесс; также может использовать общий эластичный пул. Минимум 1; требуется перезапуск.', + postprocess_concurrency: 'Выделенный параллелизм завершения разбора и запуска обогащения на процесс. Минимум 1; требуется перезапуск.', + enrichment_concurrency: 'Гарантированный параллелизм сводок, изображений, графа и вопросов на процесс; также может использовать общий пул. Минимум 1; требуется перезапуск.', + maintenance_concurrency: 'Параллелизм синхронизации, пакетных операций и очистки на процесс, изолированный от пользовательского конвейера. Минимум 1; требуется перезапуск.', + shared_concurrency: 'Эластичный параллелизм на процесс, общий для разбора и обогащения и используемый стороной с очередью. Минимум 1; требуется перезапуск.', wiki_concurrency: 'Параллелизм выделенного пула Wiki на процесс, изолированного от остальных задач. Минимум 1; требуется перезапуск сервиса.', }, diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index c51fd234..65c3779f 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2484,11 +2484,14 @@ export default { detailsTitle: "队列明细", detailsDescription: "各处理通道的实时负载与等待情况。", poolsTitle: "Worker 池", - poolsDescription: "核心解析、内容富化、维护同步和 Wiki 使用相互隔离的并发预算。", - perInstance: "并发数为单实例配置", + poolsDescription: "各阶段拥有保底容量,核心解析与内容富化还可借用共享弹性池。", + perInstance: "卡片主值为集群运行中/容量", + poolConfigured: "单实例配置 {value}", + poolInstances: "{value} 个实例", + poolUtilization: "利用率 {value}%", queueCount: "{value} 个队列", weight: "池内权重 {value}", - footnote: "Worker 并发数为单实例上限(修改后需重启);权重只决定同一池内的调度份额;「运行中」为集群当前任务数。", + footnote: "卡片主值为集群运行中/实时容量;单实例配置修改后需重启。共享弹性池只消费核心解析和内容富化队列。", updatedAt: "更新于 {value}", errors: { generic: "获取队列状态失败", @@ -2529,18 +2532,23 @@ export default { }, pools: { core: "核心解析", + postprocess: "后处理编排", enrichment: "内容富化", maintenance: "维护与同步", + shared: "共享弹性", wiki: "Wiki 池", }, poolDescriptions: { - core: "文档解析与后处理调度", + core: "文档解析与手工重解析的保底容量", + postprocess: "解析完成后的收尾与富化扇出", enrichment: "摘要、图片、图谱与问题生成", maintenance: "数据源同步、批处理与删除清理", + shared: "由核心解析与内容富化按积压借用", wiki: "Wiki 内容生成与全局收尾", }, queueNames: { default: "默认(文档解析)", + postprocess: "后处理编排", summary: "摘要生成", sync: "数据源同步", low: "后台维护与批处理", @@ -2550,7 +2558,8 @@ export default { wiki: "Wiki 同步", }, queueDescriptions: { - default: "文档解析、手工更新与后处理调度", + default: "文档解析与手工更新", + postprocess: "解析完成后的状态收尾与富化任务扇出", summary: "文档摘要与表格摘要", sync: "手动与定时数据源同步", low: "FAQ 导入、复制移动、批量重解析与删除清理", @@ -2572,7 +2581,11 @@ export default { default_storage_quota_gb: "新租户默认存储配额 (GB)", }, asynq: { - concurrency: "上游 Worker 总并发预算", + core_concurrency: "核心解析保底并发数", + postprocess_concurrency: "后处理编排并发数", + enrichment_concurrency: "内容富化保底并发数", + maintenance_concurrency: "维护与同步并发数", + shared_concurrency: "共享弹性并发数", wiki_concurrency: "Wiki Worker 并发数", }, model: { @@ -2600,10 +2613,11 @@ export default { "0 或负数表示使用内置默认值 10GB。", }, asynq: { - concurrency: - "每个服务实例用于上游后台任务的总并发预算(不含 Wiki)。系统按核心解析 1/2、" + - "内容富化 3/8、维护与同步为剩余容量,拆分成相互隔离的 Worker 池。" + - "最小值为 3;修改后需重启服务进程方可生效。", + core_concurrency: "文档解析与手工重解析的每实例保底并发,可额外借用共享弹性池;修改后需重启。", + postprocess_concurrency: "解析完成后的轻量收尾和富化扇出专用并发;修改后需重启。", + enrichment_concurrency: "摘要、图片、图谱与问题生成的每实例保底并发,可额外借用共享弹性池;修改后需重启。", + maintenance_concurrency: "数据源同步、批处理和清理任务的每实例并发,与用户面流水线硬隔离;修改后需重启。", + shared_concurrency: "核心解析与内容富化共同使用的每实例弹性并发,由有积压的一侧自动借用;修改后需重启。", wiki_concurrency: "每个服务实例的 Wiki 专用 Worker 并发数,与上游任务池相互隔离。" + "最小值为 1;修改后需重启服务进程方可生效。", diff --git a/frontend/src/views/system/RuntimeQueues.vue b/frontend/src/views/system/RuntimeQueues.vue index 9dfbf7e7..f11bb155 100644 --- a/frontend/src/views/system/RuntimeQueues.vue +++ b/frontend/src/views/system/RuntimeQueues.vue @@ -106,12 +106,19 @@
{{ poolLabel(pool.name) }} - {{ pool.concurrency }} + + {{ pool.instances > 0 ? `${pool.active}/${pool.cluster_capacity}` : pool.concurrency }} +

{{ poolDescription(pool.name) }} - {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }} + {{ t('system.globalSettings.runtime.poolConfigured', { value: pool.concurrency }) }} + + · {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}

@@ -310,6 +317,10 @@ function poolQueueCount(pool: string): number { return pools.value.find((item) => item.name === pool)?.queue_count ?? 0 } +function poolUtilization(pool: RuntimeWorkerPool): number { + return Math.round(Math.max(0, Math.min(1, pool.utilization || 0)) * 100) +} + function formatLatency(ms: number): string { if (!ms || ms <= 0) return '—' if (ms < 1000) return `${ms} ms` diff --git a/frontend/src/views/system/SystemSettings.vue b/frontend/src/views/system/SystemSettings.vue index ea110765..adc7b8e5 100644 --- a/frontend/src/views/system/SystemSettings.vue +++ b/frontend/src/views/system/SystemSettings.vue @@ -761,8 +761,7 @@ function placeholderFor(item: SystemSettingItem): string { } function minimumFor(item: SystemSettingItem): number { - if (item.key === 'asynq.concurrency') return 3 - if (item.key === 'asynq.wiki_concurrency') return 1 + if (item.key.startsWith('asynq.') && item.key.endsWith('_concurrency')) return 1 return 0 } diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 47adc444..fa187e71 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -626,7 +626,7 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con } task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, - asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute)) + knowledgePostProcessTaskOptions()...) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err) } else { diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index 386efb64..02995b63 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -186,8 +186,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { if queueSkipped > 0 { // Visibility into "stale span heartbeat but tasks still queued" // — i.e. backpressure, not a stuck row. Persistent counts here - // mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY - // or document_process_timeout), not that housekeeping misfires. + // mean the queue is the bottleneck (raise the matching per-pool or + // shared asynq concurrency, or document_process_timeout), not that + // housekeeping misfires. logger.Infof(ctx, "[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)", queueSkipped) diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index 368b565a..f2c483c7 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -126,6 +126,12 @@ func (f fakeTaskInspector) QueueStats( return nil, false, nil } +func (f fakeTaskInspector) WorkerServerStats( + _ context.Context, +) ([]types.WorkerServerStat, bool, error) { + return nil, false, nil +} + func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService { return newHousekeepingSvcWithInspector(db, fakeTaskInspector{}) } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index ebe3feda..ccbc192a 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -675,7 +675,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, payloadBytes, err := json.Marshal(postProcessPayload) if err == nil { task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, - asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute)) + knowledgePostProcessTaskOptions()...) if _, err := s.task.Enqueue(task); err != nil { logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err) } else { diff --git a/internal/application/service/knowledge_task_options.go b/internal/application/service/knowledge_task_options.go index f65aca3c..9d7a1ae3 100644 --- a/internal/application/service/knowledge_task_options.go +++ b/internal/application/service/knowledge_task_options.go @@ -1,6 +1,8 @@ package service import ( + "time" + "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/types" "github.com/hibiken/asynq" @@ -15,3 +17,11 @@ func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asy opts = append(opts, extra...) return opts } + +func knowledgePostProcessTaskOptions() []asynq.Option { + return []asynq.Option{ + asynq.Queue(types.QueuePostProcess), + asynq.MaxRetry(3), + asynq.Timeout(30 * time.Minute), + } +} diff --git a/internal/application/service/knowledge_task_options_test.go b/internal/application/service/knowledge_task_options_test.go index 1b049c55..8a60543c 100644 --- a/internal/application/service/knowledge_task_options_test.go +++ b/internal/application/service/knowledge_task_options_test.go @@ -79,3 +79,12 @@ func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) { require.NotNil(t, maxRetry) assert.Equal(t, 3, *maxRetry) } + +func TestKnowledgePostProcessTaskOptionsUseDedicatedQueue(t *testing.T) { + t.Parallel() + queue, timeout, maxRetry := parseDocumentProcessOpts(t, knowledgePostProcessTaskOptions()) + assert.Equal(t, types.QueuePostProcess, queue) + assert.Equal(t, 30*time.Minute, timeout) + require.NotNil(t, maxRetry) + assert.Equal(t, 3, *maxRetry) +} diff --git a/internal/application/service/system_setting.go b/internal/application/service/system_setting.go index 5b2779ee..d596d0d6 100644 --- a/internal/application/service/system_setting.go +++ b/internal/application/service/system_setting.go @@ -183,24 +183,50 @@ var registry = map[string]settingSpec{ "用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" + "每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。", }, - // asynq.concurrency is the total per-process upstream worker budget. - // It is divided deterministically across independent core, enrichment, - // and maintenance servers. Read once at startup; changing it requires a - // process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32). - "asynq.concurrency": { + "asynq.core_concurrency": { Type: "int", - EnvName: "WEKNORA_ASYNQ_CONCURRENCY", - Default: int64(types.DefaultUpstreamWorkerConcurrency), + EnvName: "WEKNORA_ASYNQ_CORE_CONCURRENCY", + Default: int64(types.DefaultCoreWorkerConcurrency), Category: "worker", RequiresRestart: true, - Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" + - "内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" + - "最小值为 3;修改后需重启服务进程方可生效。", + Description: "文档解析、手工重解析等核心任务的每实例保底并发。可额外使用共享弹性池;修改后需重启。", + }, + "asynq.postprocess_concurrency": { + Type: "int", + EnvName: "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY", + Default: int64(types.DefaultPostProcessWorkerConcurrency), + Category: "worker", + RequiresRestart: true, + Description: "解析完成后的轻量编排与富化扇出专用并发,避免被长时间文档解析阻塞;修改后需重启。", + }, + "asynq.enrichment_concurrency": { + Type: "int", + EnvName: "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY", + Default: int64(types.DefaultEnrichmentWorkerConcurrency), + Category: "worker", + RequiresRestart: true, + Description: "摘要、图片、图谱和问题生成的每实例保底并发。可额外使用共享弹性池;修改后需重启。", + }, + "asynq.maintenance_concurrency": { + Type: "int", + EnvName: "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY", + Default: int64(types.DefaultMaintenanceWorkerConcurrency), + Category: "worker", + RequiresRestart: true, + Description: "数据源同步、批处理、移动和删除清理的每实例保底并发,与用户面流水线硬隔离;修改后需重启。", + }, + "asynq.shared_concurrency": { + Type: "int", + EnvName: "WEKNORA_ASYNQ_SHARED_CONCURRENCY", + Default: int64(types.DefaultSharedWorkerConcurrency), + Category: "worker", + RequiresRestart: true, + Description: "核心解析与内容富化共用的每实例弹性并发。空闲容量由有积压的一侧借用;修改后需重启。", }, // asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool, - // separate from asynq.concurrency. Read once when the wiki asynq server + // separate from the upstream pools. Read once when the wiki asynq server // starts — changing it in the UI requires a process restart. Mirrors - // WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16). + // WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 8). "asynq.wiki_concurrency": { Type: "int", EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY", @@ -669,6 +695,10 @@ func (s *systemSettingService) List(ctx context.Context) ([]*types.SystemSetting for _, row := range rows { byKey[row.Key] = row } + // asynq.concurrency was the old fixed-ratio aggregate. Keeping an old DB + // row visible would suggest it still controls runtime capacity, so retire it + // explicitly while preserving all genuinely unknown rows for diagnostics. + delete(byKey, "asynq.concurrency") keys := make([]string, 0, len(registry)) for key := range registry { @@ -1228,17 +1258,15 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) { // 400 body verbatim). func validateRegistryEntry(key string, rawValue any) error { switch key { - case "asynq.concurrency", "asynq.wiki_concurrency": + case "asynq.core_concurrency", "asynq.postprocess_concurrency", + "asynq.enrichment_concurrency", "asynq.maintenance_concurrency", + "asynq.shared_concurrency", "asynq.wiki_concurrency": n, err := coerceToPositiveInt64(rawValue) if err != nil { return err } - minimum := int64(1) - if key == "asynq.concurrency" { - minimum = 3 - } - if n < minimum { - return fmt.Errorf("concurrency must be at least %d", minimum) + if n < 1 { + return errors.New("concurrency must be at least 1") } case "ssrf.whitelist": // Coerce into the same shape encodeForType produced. We don't diff --git a/internal/application/service/system_setting_validation_test.go b/internal/application/service/system_setting_validation_test.go index 17ea5232..0523e6b2 100644 --- a/internal/application/service/system_setting_validation_test.go +++ b/internal/application/service/system_setting_validation_test.go @@ -9,9 +9,12 @@ func TestValidateWorkerConcurrencyMinimums(t *testing.T) { value any wantErr bool }{ - {name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true}, - {name: "upstream minimum", key: "asynq.concurrency", value: 3}, - {name: "upstream normal", key: "asynq.concurrency", value: 32}, + {name: "core zero", key: "asynq.core_concurrency", value: 0, wantErr: true}, + {name: "core minimum", key: "asynq.core_concurrency", value: 1}, + {name: "postprocess minimum", key: "asynq.postprocess_concurrency", value: 1}, + {name: "enrichment minimum", key: "asynq.enrichment_concurrency", value: 1}, + {name: "maintenance minimum", key: "asynq.maintenance_concurrency", value: 1}, + {name: "shared minimum", key: "asynq.shared_concurrency", value: 1}, {name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true}, {name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1}, } diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index b30ae0a3..e69346b2 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -79,7 +79,7 @@ const ( // wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}. wikiInflightPrefix = "wiki:inflight:" // wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight - // is unset. 4 leaves headroom in a default 16-worker wiki pool for other + // is unset. 4 leaves half of the default 8-worker wiki pool for other // KBs while still giving one KB solid parallelism. wikiInflightDefault = 4 // wikiInflightTTL is how long a reserved slot survives without renewal. diff --git a/internal/container/container.go b/internal/container/container.go index 0d5c9052..558b8677 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -264,13 +264,14 @@ func BuildContainer(container *dig.Container) *dig.Container { redisAvailable := os.Getenv("REDIS_ADDR") != "" if redisAvailable { must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer)))) - // Independent worker pools keep user-facing parsing, high-fanout - // enrichment, maintenance/sync, and Wiki generation from consuming one - // another's capacity. All pools share one handler mux but subscribe to - // disjoint queues from the central topology registry. + // Dedicated pools guarantee capacity for each stage. The shared pool + // additionally subscribes to core/enrichment queues to provide elastic + // borrowing while post-process and maintenance remain hard-isolated. must(container.Provide(router.NewCoreAsynqServer, dig.Name("coreAsynqServer"))) + must(container.Provide(router.NewPostProcessAsynqServer, dig.Name("postProcessAsynqServer"))) must(container.Provide(router.NewEnrichmentAsynqServer, dig.Name("enrichmentAsynqServer"))) must(container.Provide(router.NewMaintenanceAsynqServer, dig.Name("maintenanceAsynqServer"))) + must(container.Provide(router.NewSharedAsynqServer, dig.Name("sharedAsynqServer"))) must(container.Provide(router.NewWikiAsynqServer, dig.Name("wikiAsynqServer"))) // Asynq inspector for cancel-by-knowledge-id (best-effort // dequeue of pending/scheduled/retry tasks + active-task cancel). diff --git a/internal/handler/system.go b/internal/handler/system.go index d57fac3f..32de90cd 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -1385,13 +1385,16 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) { // RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue // dashboard. `available` is false in Lite mode (no Redis/asynq) so the // UI can render an "unavailable in this deployment" state instead of an -// empty table. Pool concurrency is the configured per-process capacity -// (resolved DB > ENV > default), not live busy-worker count or cluster-wide -// capacity (see QueueStat.Active for currently running tasks). +// empty table. Each pool reports configured per-process concurrency plus live +// cluster capacity/active workers aggregated from asynq server heartbeats. type RuntimeWorkerPool struct { - Name string `json:"name"` - Concurrency int `json:"concurrency"` - QueueCount int `json:"queue_count"` + Name string `json:"name"` + Concurrency int `json:"concurrency"` // configured per instance + QueueCount int `json:"queue_count"` + Instances int `json:"instances"` // live asynq server heartbeats + ClusterCapacity int `json:"cluster_capacity"` // sum of live server concurrency + Active int `json:"active"` // live workers assigned to this pool + Utilization float64 `json:"utilization"` // Active / ClusterCapacity } type RuntimeQueuesResponse struct { @@ -1406,6 +1409,56 @@ type RuntimeQueuesResponse struct { Timestamp int64 `json:"timestamp"` } +func aggregateRuntimeWorkerPools(pools []RuntimeWorkerPool, servers []types.WorkerServerStat) { + queueConfigs := map[string]map[string]int{ + types.WorkerPoolCore: types.QueueWeightsForPool(types.WorkerPoolCore), + types.WorkerPoolPostProcess: types.QueueWeightsForPool(types.WorkerPoolPostProcess), + types.WorkerPoolEnrichment: types.QueueWeightsForPool(types.WorkerPoolEnrichment), + types.WorkerPoolMaintenance: types.QueueWeightsForPool(types.WorkerPoolMaintenance), + types.WorkerPoolShared: types.QueueWeightsForSharedPool(), + types.WorkerPoolWiki: types.QueueWeightsForPool(types.WorkerPoolWiki), + } + indexes := make(map[string]int, len(pools)) + for i := range pools { + indexes[pools[i].Name] = i + } + for _, server := range servers { + if server.Status != "active" { + continue + } + for poolName, expectedQueues := range queueConfigs { + if !sameQueueWeights(server.Queues, expectedQueues) { + continue + } + idx, ok := indexes[poolName] + if !ok { + break + } + pools[idx].Instances++ + pools[idx].ClusterCapacity += server.Concurrency + pools[idx].Active += server.Active + break + } + } + for i := range pools { + if pools[i].ClusterCapacity > 0 { + pools[i].Utilization = float64(pools[i].Active) / float64(pools[i].ClusterCapacity) + } + } +} + +func sameQueueWeights(left, right map[string]int) bool { + if len(left) != len(right) { + return false + } + for name, weight := range left { + if right[name] != weight { + return false + } + } + return true +} + // GetRuntimeQueues godoc // @Summary 获取解析任务队列运行时状态 // @Description 返回各 asynq 队列的实时深度(pending/active/scheduled/retry 等)与 worker 并发配置,仅系统管理员可见 @@ -1415,34 +1468,30 @@ type RuntimeQueuesResponse struct { // @Router /system/admin/runtime/queues [get] func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) { ctx := logger.CloneContext(c.Request.Context()) - upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency - wikiConcurrency := types.DefaultWikiWorkerConcurrency + var allocation types.WorkerPoolConcurrency if h.systemSettingSvc != nil { - if configured := h.systemSettingSvc.GetInt( - ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency, - ); configured > 0 { - upstreamConcurrency = int(configured) - } - if configured := h.systemSettingSvc.GetInt( - ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency, - ); configured > 0 { - wikiConcurrency = int(configured) - } + allocation = types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int { + configured := h.systemSettingSvc.GetInt(ctx, key, env, int64(fallback)) + return int(configured) + }) + } else { + allocation = types.DefaultWorkerPoolConcurrency() } - allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency) queueCounts := make(map[string]int) for _, definition := range types.QueueDefinitions() { queueCounts[definition.Pool]++ } resp := RuntimeQueuesResponse{ - UpstreamConcurrency: allocation.Total, - ParseConcurrency: allocation.Total, - WikiConcurrency: wikiConcurrency, + UpstreamConcurrency: allocation.UpstreamTotal(), + ParseConcurrency: allocation.UpstreamTotal(), + WikiConcurrency: allocation.Wiki, Pools: []RuntimeWorkerPool{ {Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]}, + {Name: types.WorkerPoolPostProcess, Concurrency: allocation.PostProcess, QueueCount: queueCounts[types.WorkerPoolPostProcess]}, {Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]}, {Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]}, - {Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]}, + {Name: types.WorkerPoolShared, Concurrency: allocation.Shared, QueueCount: len(types.QueueWeightsForSharedPool())}, + {Name: types.WorkerPoolWiki, Concurrency: allocation.Wiki, QueueCount: queueCounts[types.WorkerPoolWiki]}, }, Timestamp: time.Now().Unix(), } @@ -1456,6 +1505,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) { if stats != nil { resp.Queues = stats } + serverStats, _, serverErr := h.taskInspector.WorkerServerStats(ctx) + if serverErr != nil { + logger.Errorf(ctx, "get worker server stats failed: %v", serverErr) + } else { + aggregateRuntimeWorkerPools(resp.Pools, serverStats) + } } // Always emit a non-nil array so the JSON serialises to `[]` rather // than `null`, keeping the frontend iteration simple. diff --git a/internal/handler/system_runtime_test.go b/internal/handler/system_runtime_test.go index 8b145f34..2ddd91c5 100644 --- a/internal/handler/system_runtime_test.go +++ b/internal/handler/system_runtime_test.go @@ -15,10 +15,18 @@ type runtimeTestSettings struct{} func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 { switch key { - case "asynq.concurrency": - return 32 + case "asynq.core_concurrency": + return 8 + case "asynq.postprocess_concurrency": + return 2 + case "asynq.enrichment_concurrency": + return 12 + case "asynq.maintenance_concurrency": + return 4 + case "asynq.shared_concurrency": + return 6 case "asynq.wiki_concurrency": - return 16 + return 8 default: return def } @@ -55,6 +63,17 @@ func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) { return []types.QueueStat{}, true, nil } +func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) { + return []types.WorkerServerStat{ + {Concurrency: 8, Active: 4, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)}, + {Concurrency: 2, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolPostProcess)}, + {Concurrency: 12, Active: 6, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolEnrichment)}, + {Concurrency: 4, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolMaintenance)}, + {Concurrency: 6, Active: 3, Status: "active", Queues: types.QueueWeightsForSharedPool()}, + {Concurrency: 8, Active: 2, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolWiki)}, + {Concurrency: 99, Active: 0, Status: "stopped", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)}, + }, true, nil +} func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) { gin.SetMode(gin.TestMode) @@ -85,10 +104,12 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) { concurrency int queueCount int }{ - types.WorkerPoolCore: {16, 1}, + types.WorkerPoolCore: {8, 1}, + types.WorkerPoolPostProcess: {2, 1}, types.WorkerPoolEnrichment: {12, 4}, types.WorkerPoolMaintenance: {4, 2}, - types.WorkerPoolWiki: {16, 1}, + types.WorkerPoolShared: {6, 5}, + types.WorkerPoolWiki: {8, 1}, } if len(response.Pools) != len(want) { t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want)) @@ -102,6 +123,9 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) { t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d", pool.Name, pool, expected.concurrency, expected.queueCount) } + if pool.Instances != 1 || pool.ClusterCapacity != expected.concurrency { + t.Fatalf("pool %q live capacity = %+v", pool.Name, pool) + } } } diff --git a/internal/router/task.go b/internal/router/task.go index 5ec1cd73..333807d4 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -24,13 +24,15 @@ import ( type AsynqTaskParams struct { dig.In - // Four independent servers provide hard capacity isolation between the - // user-facing parse path, high-fanout enrichment, maintenance/sync work, - // and Wiki generation. They share one handler mux but subscribe to disjoint - // queue sets from types.QueueDefinitions. + // Dedicated servers provide minimum capacity for parse, post-process, + // enrichment, maintenance, and Wiki work. SharedServer is the elastic tier: + // it also subscribes to core/enrichment queues so either stage can borrow + // otherwise-idle capacity without consuming the other stage's guarantee. CoreServer *asynq.Server `name:"coreAsynqServer"` + PostProcessServer *asynq.Server `name:"postProcessAsynqServer"` EnrichmentServer *asynq.Server `name:"enrichmentAsynqServer"` MaintenanceServer *asynq.Server `name:"maintenanceAsynqServer"` + SharedServer *asynq.Server `name:"sharedAsynqServer"` WikiServer *asynq.Server `name:"wikiAsynqServer"` KnowledgeService interfaces.KnowledgeService KnowledgeBaseService interfaces.KnowledgeBaseService @@ -151,32 +153,41 @@ func backgroundTaskMiddleware() asynq.MiddlewareFunc { } } -func resolveUpstreamPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency { - total := types.DefaultUpstreamWorkerConcurrency - if svc != nil { - n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency) - if n > 0 { - total = int(n) - } +func resolveWorkerPoolConcurrency(svc interfaces.SystemSettingService) types.WorkerPoolConcurrency { + if svc == nil { + return types.DefaultWorkerPoolConcurrency() } - return types.AllocateWorkerPoolConcurrency(total) + ctx := context.Background() + return types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int { + value := svc.GetInt(ctx, key, env, int64(fallback)) + return int(value) + }) } -// NewCoreAsynqServer runs the latency-sensitive document parse and -// post-process orchestration path. +// NewCoreAsynqServer runs document and manual parsing with guaranteed capacity. func NewCoreAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { - allocation := resolveUpstreamPoolConcurrency(svc) + allocation := resolveWorkerPoolConcurrency(svc) log.Printf("asynq core-pool server starting with concurrency=%d total_upstream=%d redis_op_timeout=%dms", - allocation.Core, allocation.Total, readRedisOpTimeoutMs()) + allocation.Core, allocation.UpstreamTotal(), readRedisOpTimeoutMs()) return newAsynqServer(allocation.Core, types.QueueWeightsForPool(types.WorkerPoolCore)) } +// NewPostProcessAsynqServer reserves capacity for the lightweight but +// latency-sensitive fan-out stage. It cannot be trapped behind a burst of +// long-running document parses in QueueDefault. +func NewPostProcessAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { + allocation := resolveWorkerPoolConcurrency(svc) + log.Printf("asynq postprocess-pool server starting with concurrency=%d total_upstream=%d", + allocation.PostProcess, allocation.UpstreamTotal()) + return newAsynqServer(allocation.PostProcess, types.QueueWeightsForPool(types.WorkerPoolPostProcess)) +} + // NewEnrichmentAsynqServer runs high-fanout summary, multimodal, graph, and // question generation without consuming core parsing capacity. func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { - allocation := resolveUpstreamPoolConcurrency(svc) + allocation := resolveWorkerPoolConcurrency(svc) log.Printf("asynq enrichment-pool server starting with concurrency=%d total_upstream=%d", - allocation.Enrichment, allocation.Total) + allocation.Enrichment, allocation.UpstreamTotal()) return newAsynqServer(allocation.Enrichment, types.QueueWeightsForPool(types.WorkerPoolEnrichment)) } @@ -184,25 +195,29 @@ func NewEnrichmentAsynqServer(svc interfaces.SystemSettingService) *asynq.Server // dispatch work. QueueMaintenance keeps the legacy Redis name "low", so old // tasks are drained safely during rolling upgrades. func NewMaintenanceAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { - allocation := resolveUpstreamPoolConcurrency(svc) + allocation := resolveWorkerPoolConcurrency(svc) log.Printf("asynq maintenance-pool server starting with concurrency=%d total_upstream=%d", - allocation.Maintenance, allocation.Total) + allocation.Maintenance, allocation.UpstreamTotal()) return newAsynqServer(allocation.Maintenance, types.QueueWeightsForPool(types.WorkerPoolMaintenance)) } +// NewSharedAsynqServer is the elastic tier. Asynq dequeue is atomic, so it is +// safe for this server and the dedicated servers to subscribe to the same +// queues: every task is still executed by exactly one worker. +func NewSharedAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { + allocation := resolveWorkerPoolConcurrency(svc) + log.Printf("asynq shared-pool server starting with concurrency=%d total_upstream=%d", + allocation.Shared, allocation.UpstreamTotal()) + return newAsynqServer(allocation.Shared, types.QueueWeightsForSharedPool()) +} + // NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. It runs // the shared handler mux but only ever pulls wiki tasks, so its concurrency -// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on +// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 8) is spent exclusively on // wiki generation. This is the hard capacity isolation that prevents the parse // pipeline from starving wiki (and vice-versa) during concurrent uploads. func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { - concurrency := types.DefaultWikiWorkerConcurrency - if svc != nil { - n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency) - if n > 0 { - concurrency = int(n) - } - } + concurrency := resolveWorkerPoolConcurrency(svc).Wiki log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency) return newAsynqServer(concurrency, types.QueueWeightsForPool(types.WorkerPoolWiki)) } @@ -293,8 +308,8 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle) mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle) - // Run the same mux on every pool. Queue definitions are disjoint, so a given - // task is dequeued by exactly one pool while handlers stay registered once. + // Run the same mux on every pool. Shared and dedicated servers intentionally + // overlap, but Redis dequeue is atomic, so each task still executes once. runPool := func(name string, srv *asynq.Server) { go func() { if err := srv.Run(mux); err != nil { @@ -303,8 +318,10 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { }() } runPool("core-pool", params.CoreServer) + runPool("postprocess-pool", params.PostProcessServer) runPool("enrichment-pool", params.EnrichmentServer) runPool("maintenance-pool", params.MaintenanceServer) + runPool("shared-pool", params.SharedServer) runPool("wiki-pool", params.WikiServer) return mux } diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index a2e4dd4d..3ace346a 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -183,6 +183,35 @@ func (a *asynqTaskInspector) QueueStats( return stats, true, nil } +func (a *asynqTaskInspector) WorkerServerStats( + ctx context.Context, +) ([]types.WorkerServerStat, bool, error) { + if a == nil || a.inspector == nil { + return nil, false, nil + } + servers, err := a.inspector.Servers() + if err != nil { + return nil, true, err + } + stats := make([]types.WorkerServerStat, 0, len(servers)) + for _, server := range servers { + if server == nil { + continue + } + queues := make(map[string]int, len(server.Queues)) + for name, weight := range server.Queues { + queues[name] = weight + } + stats = append(stats, types.WorkerServerStat{ + Concurrency: server.Concurrency, + Active: len(server.ActiveWorkers), + Status: server.Status, + Queues: queues, + }) + } + return stats, true, nil +} + // queueStateHasMatch pages through one (queue, state) list looking for a // task that references knowledgeID. Mirrors the delete* scanners but is // strictly read-only and returns early on the first hit. A backend error @@ -390,3 +419,9 @@ func (noopTaskInspector) QueueStats( ) ([]types.QueueStat, bool, error) { return nil, false, nil } + +func (noopTaskInspector) WorkerServerStats( + ctx context.Context, +) ([]types.WorkerServerStat, bool, error) { + return nil, false, nil +} diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go index 7ca1a901..87da0f5c 100644 --- a/internal/types/interfaces/task_inspector.go +++ b/internal/types/interfaces/task_inspector.go @@ -57,4 +57,9 @@ type TaskInspector interface { // surfaced as a zeroed row (so the full lane set stays visible) rather // than failing the whole call. QueueStats(ctx context.Context) (stats []types.QueueStat, supported bool, err error) + + // WorkerServerStats returns live asynq server heartbeats across all + // replicas. The runtime dashboard uses them to aggregate actual cluster + // capacity and busy workers for each configured pool. + WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error) } diff --git a/internal/types/task.go b/internal/types/task.go index b9385ac2..3d7426d4 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -5,15 +5,24 @@ package types // between pools instead of being only a weighted dequeue preference. const ( WorkerPoolCore = "core" + WorkerPoolPostProcess = "postprocess" WorkerPoolEnrichment = "enrichment" WorkerPoolMaintenance = "maintenance" + WorkerPoolShared = "shared" WorkerPoolWiki = "wiki" - // DefaultUpstreamWorkerConcurrency is split across core, enrichment, - // and maintenance. Wiki stays separately configurable because its model - // workload has a different capacity profile. - DefaultUpstreamWorkerConcurrency = 32 - DefaultWikiWorkerConcurrency = 16 + // Upstream defaults are explicit guarantees plus an elastic pool. The + // shared pool may consume core and enrichment queues, so idle capacity in + // either stage can be borrowed without sacrificing the dedicated minimums. + DefaultCoreWorkerConcurrency = 8 + DefaultPostProcessWorkerConcurrency = 2 + DefaultEnrichmentWorkerConcurrency = 12 + DefaultMaintenanceWorkerConcurrency = 4 + DefaultSharedWorkerConcurrency = 6 + DefaultWikiWorkerConcurrency = 8 + DefaultUpstreamWorkerConcurrency = DefaultCoreWorkerConcurrency + + DefaultPostProcessWorkerConcurrency + DefaultEnrichmentWorkerConcurrency + + DefaultMaintenanceWorkerConcurrency + DefaultSharedWorkerConcurrency ) // Asynq queue names. QueueMaintenance intentionally keeps the physical Redis @@ -21,6 +30,7 @@ const ( // rolling deployment. New code uses the business-semantic constant. const ( QueueDefault = "default" + QueuePostProcess = "postprocess" QueueSummary = "summary" QueueMultimodal = "multimodal" QueueGraph = "graph" @@ -35,22 +45,26 @@ const ( // scheduling weights shown to operators from drifting from the actual server // configuration. type QueueDefinition struct { - Name string - Pool string - Weight int - TaskTypes []string + Name string + Pool string + Weight int + SharedWeight int + TaskTypes []string } var queueDefinitions = []QueueDefinition{ - {Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, TaskTypes: []string{ - TypeDocumentProcess, TypeManualProcess, TypeKnowledgePostProcess, + {Name: QueueDefault, Pool: WorkerPoolCore, Weight: 1, SharedWeight: 3, TaskTypes: []string{ + TypeDocumentProcess, TypeManualProcess, }}, - {Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, TaskTypes: []string{ + {Name: QueuePostProcess, Pool: WorkerPoolPostProcess, Weight: 1, TaskTypes: []string{ + TypeKnowledgePostProcess, + }}, + {Name: QueueSummary, Pool: WorkerPoolEnrichment, Weight: 2, SharedWeight: 2, TaskTypes: []string{ TypeSummaryGeneration, TypeDataTableSummary, }}, - {Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeImageMultimodal}}, - {Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeChunkExtract}}, - {Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, TaskTypes: []string{TypeQuestionGeneration}}, + {Name: QueueMultimodal, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeImageMultimodal}}, + {Name: QueueGraph, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeChunkExtract}}, + {Name: QueueQuestion, Pool: WorkerPoolEnrichment, Weight: 1, SharedWeight: 1, TaskTypes: []string{TypeQuestionGeneration}}, {Name: QueueSync, Pool: WorkerPoolMaintenance, Weight: 2, TaskTypes: []string{TypeDataSourceSync}}, {Name: QueueMaintenance, Pool: WorkerPoolMaintenance, Weight: 1, TaskTypes: []string{ TypeFAQImport, TypeKBClone, TypeIndexDelete, TypeKBDelete, @@ -95,49 +109,71 @@ func QueueWeightsForPool(pool string) map[string]int { return weights } -// WorkerPoolConcurrency is the deterministic split of the total upstream -// worker budget. Keeping this pure allocation function in types lets the -// server constructors and the System Admin API report identical values. -type WorkerPoolConcurrency struct { - Total int - Core int - Enrichment int - Maintenance int -} - -// AllocateWorkerPoolConcurrency reserves 1/2 of the upstream budget for the -// latency-sensitive core pipeline, 3/8 for LLM/VLM enrichment, and the -// remainder for sync/cleanup/batch work. A minimum total of three guarantees -// every independent server has at least one worker. -func AllocateWorkerPoolConcurrency(total int) WorkerPoolConcurrency { - if total < 3 { - total = 3 - } - core := total / 2 - enrichment := (total * 3) / 8 - if core < 1 { - core = 1 - } - if enrichment < 1 { - enrichment = 1 - } - maintenance := total - core - enrichment - if maintenance < 1 { - maintenance = 1 - if core >= enrichment && core > 1 { - core-- - } else if enrichment > 1 { - enrichment-- +// QueueWeightsForSharedPool returns the core/enrichment queues eligible for +// elastic capacity. Post-process and maintenance are deliberately excluded: +// post-process needs a small latency guarantee, while long maintenance tasks +// must not pin burst capacity intended for the user-facing pipeline. +func QueueWeightsForSharedPool() map[string]int { + weights := make(map[string]int) + for _, definition := range queueDefinitions { + if definition.SharedWeight > 0 { + weights[definition.Name] = definition.SharedWeight } } + return weights +} + +// WorkerPoolConcurrency contains the explicit per-instance pool capacities. +// Unlike the old ratio allocator, each field is independently configurable. +type WorkerPoolConcurrency struct { + Core int + PostProcess int + Enrichment int + Maintenance int + Shared int + Wiki int +} + +func DefaultWorkerPoolConcurrency() WorkerPoolConcurrency { return WorkerPoolConcurrency{ - Total: total, - Core: core, - Enrichment: enrichment, - Maintenance: maintenance, + Core: DefaultCoreWorkerConcurrency, + PostProcess: DefaultPostProcessWorkerConcurrency, + Enrichment: DefaultEnrichmentWorkerConcurrency, + Maintenance: DefaultMaintenanceWorkerConcurrency, + Shared: DefaultSharedWorkerConcurrency, + Wiki: DefaultWikiWorkerConcurrency, } } +// ResolveWorkerPoolConcurrency centralizes setting keys, environment names, +// defaults, and invalid-value fallback for both server construction and the +// runtime API. The callback lets this package stay independent of the system +// setting service interface. +func ResolveWorkerPoolConcurrency(read func(key, env string, fallback int) int) WorkerPoolConcurrency { + allocation := DefaultWorkerPoolConcurrency() + if read == nil { + return allocation + } + positive := func(key, env string, fallback int) int { + value := read(key, env, fallback) + if value < 1 { + return fallback + } + return value + } + allocation.Core = positive("asynq.core_concurrency", "WEKNORA_ASYNQ_CORE_CONCURRENCY", allocation.Core) + allocation.PostProcess = positive("asynq.postprocess_concurrency", "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY", allocation.PostProcess) + allocation.Enrichment = positive("asynq.enrichment_concurrency", "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY", allocation.Enrichment) + allocation.Maintenance = positive("asynq.maintenance_concurrency", "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY", allocation.Maintenance) + allocation.Shared = positive("asynq.shared_concurrency", "WEKNORA_ASYNQ_SHARED_CONCURRENCY", allocation.Shared) + allocation.Wiki = positive("asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", allocation.Wiki) + return allocation +} + +func (c WorkerPoolConcurrency) UpstreamTotal() int { + return c.Core + c.PostProcess + c.Enrichment + c.Maintenance + c.Shared +} + // QueueStat is a read-only depth snapshot of a single asynq queue, used // by the System Admin runtime dashboard. Field names mirror the counts // exposed by asynq.QueueInfo. Pool / Weight are static metadata (which @@ -170,6 +206,17 @@ type QueueStat struct { MemoryUsageBytes int64 `json:"memory_usage_bytes"` } +// WorkerServerStat is one live asynq server heartbeat. Queue weights identify +// which logical pool the server belongs to; the runtime handler aggregates +// these records across replicas so configured per-instance capacity is not +// confused with actual cluster capacity. +type WorkerServerStat struct { + Concurrency int + Active int + Status string + Queues map[string]int +} + const ( TypeChunkExtract = "chunk:extract" TypeDocumentProcess = "document:process" // 文档处理任务 diff --git a/internal/types/task_queue_test.go b/internal/types/task_queue_test.go index 9922f164..c13472b8 100644 --- a/internal/types/task_queue_test.go +++ b/internal/types/task_queue_test.go @@ -10,6 +10,7 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) { validPools := map[string]bool{ WorkerPoolCore: true, + WorkerPoolPostProcess: true, WorkerPoolEnrichment: true, WorkerPoolMaintenance: true, WorkerPoolWiki: true, @@ -47,6 +48,13 @@ func TestQueueDefinitionsAreUniqueAndConsumable(t *testing.T) { t.Fatalf("worker pool %q has no queues", pool) } } + shared := QueueWeightsForSharedPool() + if len(shared) == 0 || shared[QueueDefault] <= 0 || shared[QueueSummary] <= 0 { + t.Fatalf("shared pool must cover core and enrichment queues: %+v", shared) + } + if shared[QueuePostProcess] != 0 || shared[QueueMaintenance] != 0 { + t.Fatalf("shared pool must not consume post-process or maintenance: %+v", shared) + } } func TestQueueMaintenanceKeepsLegacyPhysicalName(t *testing.T) { @@ -71,22 +79,31 @@ func TestEveryAsynqTaskTypeHasADeclaredQueue(t *testing.T) { } } -func TestAllocateWorkerPoolConcurrencyPreservesBudget(t *testing.T) { - for _, total := range []int{3, 4, 8, 32, 64} { - allocation := AllocateWorkerPoolConcurrency(total) - if allocation.Total != total { - t.Fatalf("total=%d: allocation reported total %d", total, allocation.Total) - } - if allocation.Core < 1 || allocation.Enrichment < 1 || allocation.Maintenance < 1 { - t.Fatalf("total=%d: every pool must receive capacity: %+v", total, allocation) - } - if allocation.Core+allocation.Enrichment+allocation.Maintenance != total { - t.Fatalf("total=%d: split does not preserve budget: %+v", total, allocation) - } +func TestDefaultWorkerPoolConcurrencyIsExplicitBudget(t *testing.T) { + allocation := DefaultWorkerPoolConcurrency() + if allocation.UpstreamTotal() != DefaultUpstreamWorkerConcurrency { + t.Fatalf("upstream total = %d, want %d", allocation.UpstreamTotal(), DefaultUpstreamWorkerConcurrency) } - - minimum := AllocateWorkerPoolConcurrency(1) - if minimum.Total != 3 || minimum.Core+minimum.Enrichment+minimum.Maintenance != 3 { - t.Fatalf("undersized budget should clamp to three workers: %+v", minimum) + if allocation.Core < 1 || allocation.PostProcess < 1 || allocation.Enrichment < 1 || + allocation.Maintenance < 1 || allocation.Shared < 1 || allocation.Wiki < 1 { + t.Fatalf("every pool must have positive explicit capacity: %+v", allocation) + } +} + +func TestResolveWorkerPoolConcurrencyFallsBackPerPool(t *testing.T) { + allocation := ResolveWorkerPoolConcurrency(func(key, _ string, fallback int) int { + if key == "asynq.core_concurrency" { + return 15 + } + if key == "asynq.shared_concurrency" { + return 0 + } + return fallback + }) + if allocation.Core != 15 { + t.Fatalf("core = %d, want override 15", allocation.Core) + } + if allocation.Shared != DefaultSharedWorkerConcurrency { + t.Fatalf("shared = %d, want fallback %d", allocation.Shared, DefaultSharedWorkerConcurrency) } }