From d496d0fc73187917bd26f2381e7d07163fbc9f8f Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 10 Jul 2026 23:19:41 +0800 Subject: [PATCH] feat(task-queue): split upstream workers into isolated core/enrichment/maintenance pools Route summary/sync/maintenance tasks to dedicated queues and servers so parsing, enrichment, and batch work no longer share one concurrency budget, while extending the runtime dashboard and settings validation to reflect the new topology. --- frontend/src/api/system/index.ts | 18 +- frontend/src/i18n/locales/en-US.ts | 37 +++- frontend/src/i18n/locales/ko-KR.ts | 37 +++- frontend/src/i18n/locales/ru-RU.ts | 37 +++- frontend/src/i18n/locales/zh-CN.ts | 42 +++-- frontend/src/views/system/RuntimeQueues.vue | 170 +++++++++++++----- frontend/src/views/system/SystemSettings.vue | 8 +- .../application/service/datasource_service.go | 5 +- internal/application/service/extract.go | 7 +- .../application/service/image_multimodal.go | 3 +- .../application/service/knowledge_create.go | 3 +- .../service/knowledge_faq_import.go | 3 +- .../service/knowledge_post_process.go | 6 +- .../application/service/knowledge_process.go | 6 +- .../service/knowledge_task_options.go | 4 +- .../service/knowledge_task_options_test.go | 13 +- internal/application/service/knowledgebase.go | 3 +- .../application/service/system_setting.go | 28 +-- .../service/system_setting_validation_test.go | 30 ++++ internal/application/service/tag.go | 6 +- internal/container/container.go | 13 +- internal/datasource/scheduler.go | 4 +- internal/datasource/scheduler_test.go | 13 +- internal/handler/knowledge.go | 7 +- internal/handler/knowledgebase.go | 3 +- internal/handler/system.go | 78 ++++---- internal/handler/system_runtime_test.go | 133 ++++++++++++++ internal/router/task.go | 115 ++++++------ internal/router/task_inspector.go | 44 ++--- internal/types/task.go | 169 +++++++++++++---- internal/types/task_queue_test.go | 92 ++++++++++ 31 files changed, 847 insertions(+), 290 deletions(-) create mode 100644 internal/application/service/system_setting_validation_test.go create mode 100644 internal/handler/system_runtime_test.go create mode 100644 internal/types/task_queue_test.go diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index d8b2a85a..bd05e581 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -479,8 +479,8 @@ export async function listSystemAuditLog( /** * QueueStat mirrors types.QueueStat on the backend: a read-only depth - * snapshot of one asynq queue. `pool` groups the queue by worker pool - * ("parse" | "wiki"); `weight` is its scheduling weight within that pool. + * snapshot of one asynq queue. `pool` identifies the independent worker pool; + * `weight` is the queue's scheduling weight within that pool. * Counts follow asynq.QueueInfo semantics — `active` is the number of * tasks currently being processed (the closest thing to "workers busy"), * `pending` is the backlog waiting to be picked up. @@ -503,17 +503,25 @@ export interface QueueStat { memory_usage_bytes: number } +export interface RuntimeWorkerPool { + name: string + concurrency: number + queue_count: number +} + /** * Runtime queue dashboard payload. `available` is false in Lite mode * (no Redis/asynq) — render an "unavailable in this deployment" state - * rather than an empty table. `parse_concurrency` / `wiki_concurrency` - * are the CONFIGURED worker-pool sizes (effective on next restart), not - * the live busy-worker count (see QueueStat.active for that). + * rather than an empty table. Pool concurrency values are CONFIGURED + * per-process capacity (effective on next restart), not live busy-worker + * count or cluster-wide capacity. */ export interface RuntimeQueuesResponse { available: boolean + upstream_concurrency: number parse_concurrency: number wiki_concurrency: number + pools: RuntimeWorkerPool[] queues: QueueStat[] timestamp: number } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 6113512b..91d3b798 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -3461,7 +3461,7 @@ export default { }, runtime: { title: "Task Queue Runtime", - description: "Live depth of the background task queues (parsing / enrichment) and the worker concurrency config. Read-only; auto-refreshes every 5s.", + description: "Live background-queue load and per-process capacity for each isolated worker pool. Read-only; auto-refreshes every 5s.", refresh: "Refresh", autoRefresh: "Auto-refresh (every 5s)", loading: "Loading...", @@ -3472,8 +3472,12 @@ export default { empty: "No queue data available", detailsTitle: "Queue details", detailsDescription: "Live load and wait time for every processing lane.", - weight: "Weight {value}", - footnote: "Concurrency is the configured worker-pool cap (takes effect after restart); \"Active\" is the number of tasks currently being processed.", + poolsTitle: "Worker pools", + poolsDescription: "Core parsing, enrichment, maintenance, and Wiki use isolated concurrency budgets.", + perInstance: "Concurrency is per process", + queueCount: "{value} queues", + weight: "Pool weight {value}", + footnote: "Concurrency is a per-process cap (restart required); weight only controls scheduling within one pool; Active is cluster-wide.", updatedAt: "Updated at {value}", errors: { generic: "Failed to load queue stats", @@ -3483,7 +3487,7 @@ export default { active: "Active", pending: "Pending", retry: "Retrying", - concurrency: "Parse / Wiki concurrency", + archived: "Dead letters", }, columns: { queue: "Queue", @@ -3503,11 +3507,21 @@ export default { paused: "Paused", }, pools: { - parse: "Parse pool", + core: "Core parsing", + enrichment: "Enrichment", + maintenance: "Maintenance & sync", wiki: "Wiki pool", }, + poolDescriptions: { + core: "Document parsing and post-process orchestration", + enrichment: "Summaries, images, graph, and question generation", + maintenance: "Source sync, batch work, and deletion cleanup", + wiki: "Wiki content generation and global finalization", + }, queueNames: { default: "Default (document parse)", + summary: "Summary generation", + sync: "Source sync", low: "Maintenance & batch jobs", multimodal: "Multimodal (images)", graph: "Graph extraction", @@ -3515,8 +3529,10 @@ export default { wiki: "Wiki ingest", }, queueDescriptions: { - default: "Document parsing and general post-processing", - low: "Summaries, source sync, batch reparse, and deletion cleanup", + default: "Document parsing, manual updates, and post-process orchestration", + summary: "Document and data-table summaries", + sync: "Manual and scheduled data-source sync", + low: "FAQ import, clone/move, batch reparse, and deletion cleanup", multimodal: "Image OCR and multimodal descriptions", graph: "Knowledge graph extraction from document chunks", question: "Question generation from document chunks", @@ -3535,7 +3551,8 @@ export default { default_storage_quota_gb: 'Default storage quota for new tenants (GB)', }, asynq: { - concurrency: 'Async task worker concurrency', + concurrency: 'Total upstream worker budget', + wiki_concurrency: 'Wiki worker concurrency', }, model: { max_concurrency: 'Default per-model concurrency limit', @@ -3558,7 +3575,9 @@ export default { }, asynq: { concurrency: - 'Async task worker concurrency (asynq thread-pool size). Document parsing, embedding, and similar tasks are mostly I/O-bound, so raising this value can shorten queue time for bulk uploads. Requires a service process restart to take effect.', + 'Total per-process concurrency budget for upstream background work (excluding Wiki). It is split into isolated pools: 1/2 core parsing, 3/8 enrichment, and the remainder maintenance and sync. Minimum 3; requires a service restart.', + wiki_concurrency: + 'Per-process concurrency for the dedicated Wiki worker pool, isolated from upstream tasks. Minimum 1; requires a service restart.', }, model: { max_concurrency: diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 69440d0c..2ad91da9 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2466,7 +2466,7 @@ export default { }, runtime: { title: "작업 큐 런타임", - description: "백그라운드 작업 큐(파싱/보강)의 실시간 깊이와 워커 동시성 설정입니다. 읽기 전용이며 5초마다 자동 새로고침됩니다.", + description: "백그라운드 작업 큐의 실시간 부하와 각 독립 워커 풀의 프로세스별 동시성 설정입니다. 5초마다 자동 새로고침됩니다.", refresh: "새로고침", autoRefresh: "자동 새로고침 (5초마다)", loading: "불러오는 중...", @@ -2477,8 +2477,12 @@ export default { empty: "표시할 큐 데이터가 없습니다", detailsTitle: "큐 세부 정보", detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.", - weight: "가중치 {value}", - footnote: "동시성은 설정된 워커 풀 상한이며(재시작 후 적용), \"실행 중\"은 현재 처리 중인 작업 수입니다.", + poolsTitle: "워커 풀", + poolsDescription: "핵심 파싱, 보강, 유지 관리 및 Wiki는 독립된 동시성 예산을 사용합니다.", + perInstance: "동시성은 프로세스별 설정", + queueCount: "큐 {value}개", + weight: "풀 내 가중치 {value}", + footnote: "동시성은 프로세스별 상한이며 재시작 후 적용됩니다. 가중치는 같은 풀 안의 스케줄링 비율이며 실행 중 수치는 클러스터 전체입니다.", updatedAt: "{value} 업데이트됨", errors: { generic: "큐 상태를 불러오지 못했습니다", @@ -2488,7 +2492,7 @@ export default { active: "실행 중", pending: "대기 중", retry: "재시도 중", - concurrency: "파싱 / Wiki 동시성", + archived: "데드 레터", }, columns: { queue: "큐", @@ -2508,11 +2512,21 @@ export default { paused: "일시중지됨", }, pools: { - parse: "파싱 풀", + core: "핵심 파싱", + enrichment: "콘텐츠 보강", + maintenance: "유지 관리 및 동기화", wiki: "Wiki 풀", }, + poolDescriptions: { + core: "문서 파싱 및 후처리 오케스트레이션", + enrichment: "요약, 이미지, 그래프 및 질문 생성", + maintenance: "데이터 소스 동기화, 일괄 작업 및 삭제 정리", + wiki: "Wiki 콘텐츠 생성 및 전체 마무리", + }, queueNames: { default: "기본(문서 파싱)", + summary: "요약 생성", + sync: "데이터 소스 동기화", low: "백그라운드 유지 관리 및 일괄 작업", multimodal: "멀티모달(이미지)", graph: "그래프 추출", @@ -2520,8 +2534,10 @@ export default { wiki: "Wiki 동기화", }, queueDescriptions: { - default: "문서 파싱 및 일반 후처리", - low: "요약, 데이터 소스 동기화, 일괄 재파싱 및 삭제 정리", + default: "문서 파싱, 수동 업데이트 및 후처리 오케스트레이션", + summary: "문서 및 데이터 테이블 요약", + sync: "수동 및 예약 데이터 소스 동기화", + low: "FAQ 가져오기, 복제/이동, 일괄 재파싱 및 삭제 정리", multimodal: "이미지 OCR 및 멀티모달 설명", graph: "문서 청크에서 지식 그래프 추출", question: "문서 청크를 기반으로 질문 생성", @@ -2540,7 +2556,8 @@ export default { default_storage_quota_gb: "신규 테넌트 기본 저장 용량 (GB)", }, asynq: { - concurrency: "비동기 작업 워커 동시 처리 수", + concurrency: "상위 워커 총 동시성 예산", + wiki_concurrency: "Wiki 워커 동시성", }, model: { max_concurrency: "모델 기본 동시 처리 상한", @@ -2563,7 +2580,9 @@ export default { }, asynq: { concurrency: - "비동기 작업 worker 동시 처리 수(asynq 스레드 풀 크기)입니다. 문서 파싱·임베딩 등은 대부분 I/O 대기이므로 값을 올리면 대량 업로드 대기 시간을 줄일 수 있습니다. 적용하려면 서비스 프로세스를 재시작해야 합니다.", + "Wiki를 제외한 상위 백그라운드 작업의 프로세스별 총 동시성 예산입니다. 핵심 파싱 1/2, 콘텐츠 보강 3/8, 나머지는 유지 관리 및 동기화로 분리됩니다. 최소값은 3이며 적용하려면 서비스를 재시작해야 합니다.", + wiki_concurrency: + "상위 작업과 분리된 전용 Wiki 워커 풀의 프로세스별 동시성입니다. 최소값은 1이며 적용하려면 서비스를 재시작해야 합니다.", }, model: { max_concurrency: diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e472100e..bd6858fe 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2164,7 +2164,7 @@ export default { }, runtime: { title: "Среда выполнения очередей", - description: "Актуальная глубина фоновых очередей задач (парсинг/обогащение) и настройки параллелизма воркеров. Только для просмотра; автообновление каждые 5 сек.", + description: "Текущая нагрузка фоновых очередей и ёмкость каждого изолированного пула на процесс. Автообновление каждые 5 сек.", refresh: "Обновить", autoRefresh: "Автообновление (каждые 5 сек)", loading: "Загрузка...", @@ -2175,8 +2175,12 @@ export default { empty: "Нет данных об очередях", detailsTitle: "Сведения об очередях", detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.", - weight: "Вес {value}", - footnote: "Параллелизм — это настроенный предел пула воркеров (вступает в силу после перезапуска); «Активные» — число задач, обрабатываемых сейчас.", + poolsTitle: "Пулы воркеров", + poolsDescription: "Основной разбор, обогащение, обслуживание и Wiki используют независимые бюджеты параллелизма.", + perInstance: "Параллелизм указан на процесс", + queueCount: "Очередей: {value}", + weight: "Вес в пуле {value}", + footnote: "Параллелизм указан на процесс и требует перезапуска; вес действует только внутри пула; активные задачи показаны по всему кластеру.", updatedAt: "Обновлено в {value}", errors: { generic: "Не удалось загрузить состояние очередей", @@ -2186,7 +2190,7 @@ export default { active: "Активные", pending: "В очереди", retry: "Повтор", - concurrency: "Параллелизм Parse / Wiki", + archived: "Мёртвые задачи", }, columns: { queue: "Очередь", @@ -2206,11 +2210,21 @@ export default { paused: "Приостановлено", }, pools: { - parse: "Пул парсинга", + core: "Основной разбор", + enrichment: "Обогащение", + maintenance: "Обслуживание и синхронизация", wiki: "Пул Wiki", }, + poolDescriptions: { + core: "Разбор документов и оркестрация постобработки", + enrichment: "Сводки, изображения, граф и генерация вопросов", + maintenance: "Синхронизация источников, пакетные задачи и удаление", + wiki: "Создание содержимого Wiki и финальная обработка", + }, queueNames: { default: "По умолчанию (парсинг документов)", + summary: "Создание сводок", + sync: "Синхронизация источников", low: "Фоновое обслуживание и пакетные задачи", multimodal: "Мультимодальные (изображения)", graph: "Извлечение графа", @@ -2218,8 +2232,10 @@ export default { wiki: "Синхронизация Wiki", }, queueDescriptions: { - default: "Парсинг документов и общая постобработка", - low: "Сводки, синхронизация источников, пакетный повторный разбор и удаление", + default: "Разбор документов, ручные обновления и оркестрация постобработки", + summary: "Сводки документов и таблиц данных", + sync: "Ручная и плановая синхронизация источников", + low: "Импорт FAQ, клонирование/перемещение, пакетный повторный разбор и удаление", multimodal: "OCR изображений и мультимодальные описания", graph: "Извлечение графа знаний из фрагментов документов", question: "Генерация вопросов по фрагментам документов", @@ -2238,7 +2254,8 @@ export default { default_storage_quota_gb: 'Квота хранилища для новых тенантов по умолчанию (ГБ)', }, asynq: { - concurrency: 'Параллелизм воркеров асинхронных задач', + concurrency: 'Общий бюджет фоновых воркеров', + wiki_concurrency: 'Параллелизм воркеров Wiki', }, model: { max_concurrency: 'Лимит параллелизма модели по умолчанию', @@ -2261,7 +2278,9 @@ export default { }, asynq: { concurrency: - 'Параллелизм воркеров асинхронных задач (размер пула потоков asynq). Парсинг документов, эмбеддинги и подобные задачи в основном ждут I/O, поэтому увеличение значения сокращает очередь при массовой загрузке. Для применения требуется перезапуск процесса сервиса.', + 'Общий бюджет параллелизма фоновых задач на процесс (без Wiki). Он делится на изолированные пулы: 1/2 основной разбор, 3/8 обогащение, остаток — обслуживание и синхронизация. Минимум 3; требуется перезапуск сервиса.', + wiki_concurrency: + 'Параллелизм выделенного пула Wiki на процесс, изолированного от остальных задач. Минимум 1; требуется перезапуск сервиса.', }, model: { max_concurrency: diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index f2ff13db..4eb19214 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2469,7 +2469,7 @@ export default { }, runtime: { title: "任务队列运行时", - description: "解析/富化等后台任务队列的实时深度,以及 worker 并发配置。仅供观察,每 5 秒自动刷新。", + description: "后台任务队列的实时负载,以及各独立 worker 池的每实例并发配置。仅供观察,每 5 秒自动刷新。", refresh: "刷新", autoRefresh: "自动刷新(每 5 秒)", loading: "加载中...", @@ -2480,8 +2480,12 @@ export default { empty: "暂无队列数据", detailsTitle: "队列明细", detailsDescription: "各处理通道的实时负载与等待情况。", - weight: "权重 {value}", - footnote: "并发数为配置的 worker 池上限(修改后需重启生效);「运行中」为当前实际处理任务数。", + poolsTitle: "Worker 池", + poolsDescription: "核心解析、内容富化、维护同步和 Wiki 使用相互隔离的并发预算。", + perInstance: "并发数为单实例配置", + queueCount: "{value} 个队列", + weight: "池内权重 {value}", + footnote: "Worker 并发数为单实例上限(修改后需重启);权重只决定同一池内的调度份额;「运行中」为集群当前任务数。", updatedAt: "更新于 {value}", errors: { generic: "获取队列状态失败", @@ -2491,7 +2495,7 @@ export default { active: "运行中", pending: "排队中", retry: "重试中", - concurrency: "解析 / Wiki 并发", + archived: "死信", }, columns: { queue: "队列", @@ -2511,11 +2515,21 @@ export default { paused: "已暂停", }, pools: { - parse: "解析池", + core: "核心解析", + enrichment: "内容富化", + maintenance: "维护与同步", wiki: "Wiki 池", }, + poolDescriptions: { + core: "文档解析与后处理调度", + enrichment: "摘要、图片、图谱与问题生成", + maintenance: "数据源同步、批处理与删除清理", + wiki: "Wiki 内容生成与全局收尾", + }, queueNames: { default: "默认(文档解析)", + summary: "摘要生成", + sync: "数据源同步", low: "后台维护与批处理", multimodal: "多模态(图片)", graph: "图谱抽取", @@ -2523,8 +2537,10 @@ export default { wiki: "Wiki 同步", }, queueDescriptions: { - default: "文档解析与通用后处理", - low: "摘要、数据源同步、批量重解析与删除清理", + default: "文档解析、手工更新与后处理调度", + summary: "文档摘要与表格摘要", + sync: "手动与定时数据源同步", + low: "FAQ 导入、复制移动、批量重解析与删除清理", multimodal: "图片 OCR 与多模态描述", graph: "从文档分块提取知识图谱", question: "基于文档分块生成问题", @@ -2543,7 +2559,8 @@ export default { default_storage_quota_gb: "新租户默认存储配额 (GB)", }, asynq: { - concurrency: "异步任务并发数", + concurrency: "上游 Worker 总并发预算", + wiki_concurrency: "Wiki Worker 并发数", }, model: { max_concurrency: "模型默认并发上限", @@ -2571,9 +2588,12 @@ export default { }, asynq: { concurrency: - "异步任务 worker 并发数(asynq 线程池大小)。" + - "文档解析、嵌入等任务多为 I/O 等待,适当提高可缩短批量上传排队时间。" + - "修改后需重启服务进程方可生效。", + "每个服务实例用于上游后台任务的总并发预算(不含 Wiki)。系统按核心解析 1/2、" + + "内容富化 3/8、维护与同步为剩余容量,拆分成相互隔离的 Worker 池。" + + "最小值为 3;修改后需重启服务进程方可生效。", + wiki_concurrency: + "每个服务实例的 Wiki 专用 Worker 并发数,与上游任务池相互隔离。" + + "最小值为 1;修改后需重启服务进程方可生效。", }, model: { max_concurrency: diff --git a/frontend/src/views/system/RuntimeQueues.vue b/frontend/src/views/system/RuntimeQueues.vue index 29a3ec4e..1058aaff 100644 --- a/frontend/src/views/system/RuntimeQueues.vue +++ b/frontend/src/views/system/RuntimeQueues.vue @@ -86,13 +86,29 @@ {{ t('system.globalSettings.runtime.summary.retry') }} {{ totalRetry }} -
- {{ t('system.globalSettings.runtime.summary.concurrency') }} - - {{ parseConcurrency }}{{ poolLabel('parse') }} - / - {{ wikiConcurrency }}{{ poolLabel('wiki') }} - +
+ {{ t('system.globalSettings.runtime.summary.archived') }} + {{ totalArchived }} +
+
+ + +
+
+
+

{{ t('system.globalSettings.runtime.poolsTitle') }}

+

{{ t('system.globalSettings.runtime.poolsDescription') }}

+
+ {{ t('system.globalSettings.runtime.perInstance') }} +
+
+
+
+ {{ poolLabel(pool.name) }} + {{ pool.concurrency }} +
+

{{ poolDescription(pool.name) }}

+ {{ t('system.globalSettings.runtime.queueCount', { value: pool.queue_count }) }}
@@ -126,8 +142,10 @@
{{ queueLabel(row.name) }} - {{ queueDescription(row.name) }} · {{ poolLabel(row.pool) }} · - {{ t('system.globalSettings.runtime.weight', { value: row.weight }) }} + {{ queueDescription(row.name) }} · {{ poolLabel(row.pool) }} +
@@ -168,16 +186,15 @@