From fe47ac1f96564a353c58d5dc3e43a0b5b609e018 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 10 Jul 2026 22:36:58 +0800 Subject: [PATCH] feat(system): add runtime task queue observability dashboard for system admins Expose live asynq queue depths and worker-pool concurrency via a new SystemAdmin API and settings UI, so operators can monitor parsing/wiki backlog without Redis CLI access. --- frontend/src/api/system/index.ts | 54 ++ frontend/src/i18n/locales/en-US.ts | 66 ++ frontend/src/i18n/locales/ko-KR.ts | 66 ++ frontend/src/i18n/locales/ru-RU.ts | 66 ++ frontend/src/i18n/locales/zh-CN.ts | 66 ++ frontend/src/router/index.ts | 15 +- frontend/src/views/settings/Settings.vue | 31 +- frontend/src/views/system/RuntimeQueues.vue | 766 ++++++++++++++++++ .../service/knowledge_housekeeping_test.go | 6 + .../application/service/knowledge_process.go | 2 +- internal/handler/system.go | 77 ++ internal/router/router.go | 6 + internal/router/task.go | 1 - internal/router/task_inspector.go | 77 +- internal/types/interfaces/task_inspector.go | 20 +- internal/types/task.go | 42 +- 16 files changed, 1339 insertions(+), 22 deletions(-) create mode 100644 frontend/src/views/system/RuntimeQueues.vue diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index 4c0474cd..d8b2a85a 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -474,3 +474,57 @@ export async function listSystemAuditLog( const url = `/api/v1/system/admin/audit-log${tail ? '?' + tail : ''}` return (await get(url)) as unknown as ListAuditLogResponse } + +// ---- Runtime queue observability (system-scope) ---- + +/** + * QueueStat mirrors types.QueueStat on the backend: a read-only depth + * snapshot of one asynq queue. `pool` groups the queue by worker pool + * ("parse" | "wiki"); `weight` is its scheduling weight within that pool. + * Counts follow asynq.QueueInfo semantics — `active` is the number of + * tasks currently being processed (the closest thing to "workers busy"), + * `pending` is the backlog waiting to be picked up. + */ +export interface QueueStat { + name: string + pool: string + weight: number + size: number + pending: number + active: number + scheduled: number + retry: number + archived: number + completed: number + processed: number + failed: number + paused: boolean + latency_ms: number + memory_usage_bytes: number +} + +/** + * Runtime queue dashboard payload. `available` is false in Lite mode + * (no Redis/asynq) — render an "unavailable in this deployment" state + * rather than an empty table. `parse_concurrency` / `wiki_concurrency` + * are the CONFIGURED worker-pool sizes (effective on next restart), not + * the live busy-worker count (see QueueStat.active for that). + */ +export interface RuntimeQueuesResponse { + available: boolean + parse_concurrency: number + wiki_concurrency: number + queues: QueueStat[] + timestamp: number +} + +/** + * Fetch the live asynq queue depths + worker-pool concurrency. + * Backend: GET /api/v1/system/admin/runtime/queues (SystemAdmin only). + * Returns the object directly — no {data: ...} wrapping (see + * utils/request.ts interceptor). + */ +export async function getRuntimeQueues(): Promise { + const response = await get('/api/v1/system/admin/runtime/queues') + return response as unknown as RuntimeQueuesResponse +} diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 228b630a..6113512b 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1213,6 +1213,7 @@ export default { storageEngine: 'Storage Engine', mcpService: 'MCP Service', versionInfo: 'Version Info', + taskQueue: 'Task Queues', tenantInfo: 'Tenant Info', apiInfo: 'API Info', navGroups: { @@ -1220,6 +1221,7 @@ export default { workspace: 'Workspace', modelsRuntime: 'Models', dataExtensions: 'Data & Extensions', + systemAdministration: 'System Administration', platform: 'Platform', }, roleDenied: { @@ -3457,6 +3459,70 @@ export default { tier2: 'Items not saved here fall back to the environment variable, or to the built-in default if no env var is set.', tier3: 'To put an item back under environment-variable control, click the "Reset" button on its row.', }, + runtime: { + title: "Task Queue Runtime", + description: "Live depth of the background task queues (parsing / enrichment) and the worker concurrency config. Read-only; auto-refreshes every 5s.", + refresh: "Refresh", + autoRefresh: "Auto-refresh (every 5s)", + loading: "Loading...", + retry: "Retry", + unavailableTitle: "Task queues unavailable", + unavailable: "This deployment has no Redis / asynq queue (Lite mode) — nothing to display.", + paused: "Paused", + empty: "No queue data available", + detailsTitle: "Queue details", + detailsDescription: "Live load and wait time for every processing lane.", + weight: "Weight {value}", + footnote: "Concurrency is the configured worker-pool cap (takes effect after restart); \"Active\" is the number of tasks currently being processed.", + updatedAt: "Updated at {value}", + errors: { + generic: "Failed to load queue stats", + }, + summary: { + title: "Runtime overview", + active: "Active", + pending: "Pending", + retry: "Retrying", + concurrency: "Parse / Wiki concurrency", + }, + columns: { + queue: "Queue", + active: "Active", + pending: "Pending", + scheduled: "Scheduled", + retry: "Retry", + archived: "Dead", + latency: "Oldest wait", + status: "Status", + }, + status: { + working: "Working", + waiting: "Waiting", + idle: "Idle", + attention: "Attention", + paused: "Paused", + }, + pools: { + parse: "Parse pool", + wiki: "Wiki pool", + }, + queueNames: { + default: "Default (document parse)", + low: "Maintenance & batch jobs", + multimodal: "Multimodal (images)", + graph: "Graph extraction", + question: "Question generation", + wiki: "Wiki ingest", + }, + queueDescriptions: { + default: "Document parsing and general post-processing", + low: "Summaries, source sync, batch reparse, and deletion cleanup", + multimodal: "Image OCR and multimodal descriptions", + graph: "Knowledge graph extraction from document chunks", + question: "Question generation from document chunks", + wiki: "Wiki content generation and index sync", + }, + }, keyLabels: { auth: { registration_mode: 'Self-service registration mode', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 52d2c1a6..69440d0c 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1070,6 +1070,7 @@ export default { storageEngine: "스토리지 엔진", mcpService: "MCP 서비스", versionInfo: "버전 정보", + taskQueue: "작업 큐", tenantInfo: "테넌트 정보", apiInfo: "API 정보", navGroups: { @@ -1077,6 +1078,7 @@ export default { workspace: "공간", modelsRuntime: "모델", dataExtensions: "데이터 및 확장", + systemAdministration: "시스템 관리", platform: "플랫폼", }, roleDenied: { @@ -2462,6 +2464,70 @@ export default { tier2: "여기서 저장하지 않은 항목은 환경 변수를 따르며, 환경 변수도 없으면 내장 기본값을 사용합니다.", tier3: "특정 항목을 다시 환경 변수 기반으로 되돌리려면 해당 행의 \"초기화\" 버튼을 누르세요.", }, + runtime: { + title: "작업 큐 런타임", + description: "백그라운드 작업 큐(파싱/보강)의 실시간 깊이와 워커 동시성 설정입니다. 읽기 전용이며 5초마다 자동 새로고침됩니다.", + refresh: "새로고침", + autoRefresh: "자동 새로고침 (5초마다)", + loading: "불러오는 중...", + retry: "다시 시도", + unavailableTitle: "작업 큐를 사용할 수 없음", + unavailable: "현재 배포는 Redis / asynq 큐가 없습니다(Lite 모드) — 표시할 내용이 없습니다.", + paused: "일시중지됨", + empty: "표시할 큐 데이터가 없습니다", + detailsTitle: "큐 세부 정보", + detailsDescription: "각 처리 경로의 실시간 부하와 대기 상태입니다.", + weight: "가중치 {value}", + footnote: "동시성은 설정된 워커 풀 상한이며(재시작 후 적용), \"실행 중\"은 현재 처리 중인 작업 수입니다.", + updatedAt: "{value} 업데이트됨", + errors: { + generic: "큐 상태를 불러오지 못했습니다", + }, + summary: { + title: "실행 개요", + active: "실행 중", + pending: "대기 중", + retry: "재시도 중", + concurrency: "파싱 / Wiki 동시성", + }, + columns: { + queue: "큐", + active: "실행 중", + pending: "대기", + scheduled: "예약", + retry: "재시도", + archived: "실패", + latency: "최장 대기", + status: "상태", + }, + status: { + working: "처리 중", + waiting: "대기 중", + idle: "유휴", + attention: "확인 필요", + paused: "일시중지됨", + }, + pools: { + parse: "파싱 풀", + wiki: "Wiki 풀", + }, + queueNames: { + default: "기본(문서 파싱)", + low: "백그라운드 유지 관리 및 일괄 작업", + multimodal: "멀티모달(이미지)", + graph: "그래프 추출", + question: "질문 생성", + wiki: "Wiki 동기화", + }, + queueDescriptions: { + default: "문서 파싱 및 일반 후처리", + low: "요약, 데이터 소스 동기화, 일괄 재파싱 및 삭제 정리", + multimodal: "이미지 OCR 및 멀티모달 설명", + graph: "문서 청크에서 지식 그래프 추출", + question: "문서 청크를 기반으로 질문 생성", + wiki: "Wiki 콘텐츠 생성 및 인덱스 동기화", + }, + }, keyLabels: { auth: { registration_mode: "셀프 가입 모드", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e44f814f..e472100e 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1126,6 +1126,7 @@ export default { conversationConfig: 'Настройки диалога', conversationStrategy: 'Стратегия диалога', versionInfo: 'Информация о версии', + taskQueue: 'Очереди задач', tenantInfo: 'Информация о арендаторе', apiInfo: 'Информация API', navGroups: { @@ -1133,6 +1134,7 @@ export default { workspace: 'Пространство', modelsRuntime: 'Модели', dataExtensions: 'Данные и расширения', + systemAdministration: 'Системное администрирование', platform: 'Платформа', }, roleDenied: { @@ -2160,6 +2162,70 @@ export default { tier2: 'Параметры, не сохранённые здесь, берутся из переменной окружения, а если её нет — из встроенного значения по умолчанию.', tier3: 'Чтобы вернуть параметр под управление переменной окружения, нажмите кнопку «Сбросить» в его строке.', }, + runtime: { + title: "Среда выполнения очередей", + description: "Актуальная глубина фоновых очередей задач (парсинг/обогащение) и настройки параллелизма воркеров. Только для просмотра; автообновление каждые 5 сек.", + refresh: "Обновить", + autoRefresh: "Автообновление (каждые 5 сек)", + loading: "Загрузка...", + retry: "Повторить", + unavailableTitle: "Очереди задач недоступны", + unavailable: "В этой конфигурации нет очереди Redis / asynq (режим Lite) — отображать нечего.", + paused: "Приостановлено", + empty: "Нет данных об очередях", + detailsTitle: "Сведения об очередях", + detailsDescription: "Текущая нагрузка и ожидание для каждого канала обработки.", + weight: "Вес {value}", + footnote: "Параллелизм — это настроенный предел пула воркеров (вступает в силу после перезапуска); «Активные» — число задач, обрабатываемых сейчас.", + updatedAt: "Обновлено в {value}", + errors: { + generic: "Не удалось загрузить состояние очередей", + }, + summary: { + title: "Обзор выполнения", + active: "Активные", + pending: "В очереди", + retry: "Повтор", + concurrency: "Параллелизм Parse / Wiki", + }, + columns: { + queue: "Очередь", + active: "Активные", + pending: "Ожидают", + scheduled: "Запланир.", + retry: "Повтор", + archived: "Мёртвые", + latency: "Макс. ожидание", + status: "Статус", + }, + status: { + working: "В работе", + waiting: "Ожидает", + idle: "Простой", + attention: "Требует внимания", + paused: "Приостановлено", + }, + pools: { + parse: "Пул парсинга", + wiki: "Пул Wiki", + }, + queueNames: { + default: "По умолчанию (парсинг документов)", + low: "Фоновое обслуживание и пакетные задачи", + multimodal: "Мультимодальные (изображения)", + graph: "Извлечение графа", + question: "Генерация вопросов", + wiki: "Синхронизация Wiki", + }, + queueDescriptions: { + default: "Парсинг документов и общая постобработка", + low: "Сводки, синхронизация источников, пакетный повторный разбор и удаление", + multimodal: "OCR изображений и мультимодальные описания", + graph: "Извлечение графа знаний из фрагментов документов", + question: "Генерация вопросов по фрагментам документов", + wiki: "Создание содержимого Wiki и синхронизация индекса", + }, + }, keyLabels: { auth: { registration_mode: 'Режим самостоятельной регистрации', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index c45b2234..f2ff13db 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1078,6 +1078,7 @@ export default { storageEngine: "存储引擎", mcpService: "MCP服务", versionInfo: "版本信息", + taskQueue: "任务队列", tenantInfo: "空间信息", apiInfo: "API信息", navGroups: { @@ -1085,6 +1086,7 @@ export default { workspace: "空间", modelsRuntime: "模型", dataExtensions: "数据与扩展", + systemAdministration: "系统管理", platform: "平台", }, roleDenied: { @@ -2465,6 +2467,70 @@ export default { tier2: "未在此处保存过的项 — 如果环境变量里有就用环境变量,否则用程序内置默认值。", tier3: "若想让某项重新由环境变量控制,点击该行的「重置」按钮即可清除当前 UI 设置。", }, + runtime: { + title: "任务队列运行时", + description: "解析/富化等后台任务队列的实时深度,以及 worker 并发配置。仅供观察,每 5 秒自动刷新。", + refresh: "刷新", + autoRefresh: "自动刷新(每 5 秒)", + loading: "加载中...", + retry: "重试", + unavailableTitle: "任务队列不可用", + unavailable: "当前部署未启用 Redis / asynq 队列(Lite 模式),无队列可展示。", + paused: "已暂停", + empty: "暂无队列数据", + detailsTitle: "队列明细", + detailsDescription: "各处理通道的实时负载与等待情况。", + weight: "权重 {value}", + footnote: "并发数为配置的 worker 池上限(修改后需重启生效);「运行中」为当前实际处理任务数。", + updatedAt: "更新于 {value}", + errors: { + generic: "获取队列状态失败", + }, + summary: { + title: "运行概览", + active: "运行中", + pending: "排队中", + retry: "重试中", + concurrency: "解析 / Wiki 并发", + }, + columns: { + queue: "队列", + active: "运行中", + pending: "排队", + scheduled: "定时", + retry: "重试", + archived: "死信", + latency: "最早等待", + status: "状态", + }, + status: { + working: "处理中", + waiting: "等待中", + idle: "空闲", + attention: "需关注", + paused: "已暂停", + }, + pools: { + parse: "解析池", + wiki: "Wiki 池", + }, + queueNames: { + default: "默认(文档解析)", + low: "后台维护与批处理", + multimodal: "多模态(图片)", + graph: "图谱抽取", + question: "问题生成", + wiki: "Wiki 同步", + }, + queueDescriptions: { + default: "文档解析与通用后处理", + low: "摘要、数据源同步、批量重解析与删除清理", + multimodal: "图片 OCR 与多模态描述", + graph: "从文档分块提取知识图谱", + question: "基于文档分块生成问题", + wiki: "Wiki 内容生成与索引同步", + }, + }, keyLabels: { auth: { registration_mode: "自助注册模式", diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index 4eaca0fc..d85a223d 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -163,11 +163,10 @@ const router = createRouter({ component: () => import("../views/organization/OrganizationList.vue"), meta: { requiresInit: true, requiresAuth: true } }, - // Compatibility redirects for legacy /platform/system/* URLs. - // The whole system administration surface — global settings - // and the system-admin roster — now lives as a single section - // inside the standard Settings modal. We keep the routes - // around so old bookmarks / external links don't 404. + // Compatibility redirects for /platform/system/* URLs. System + // administration surfaces live as dedicated sections inside the + // standard Settings modal; keep stable URLs for bookmarks and + // external links. { path: "system", redirect: { path: "/platform/settings", query: { section: "system-global" } }, @@ -185,6 +184,12 @@ const router = createRouter({ redirect: { path: "/platform/settings", query: { section: "system-global" } }, meta: { requiresInit: true, requiresAuth: true, requiresSystemAdmin: true }, }, + { + path: "system/queues", + name: "systemQueues", + redirect: { path: "/platform/settings", query: { section: "runtime-queues" } }, + meta: { requiresInit: true, requiresAuth: true, requiresSystemAdmin: true }, + }, ], }, // Dev-only markdown rendering test page diff --git a/frontend/src/views/settings/Settings.vue b/frontend/src/views/settings/Settings.vue index b3aee774..26ad11ce 100644 --- a/frontend/src/views/settings/Settings.vue +++ b/frontend/src/views/settings/Settings.vue @@ -71,7 +71,7 @@
+
+ +
+
@@ -195,6 +200,7 @@ import StorageEngineSettings from './StorageEngineSettings.vue' import WeKnoraCloudSettings from './WeKnoraCloudSettings.vue' import TenantMembers from './TenantMembers.vue' import SystemSettings from '@/views/system/SystemSettings.vue' +import RuntimeQueues from '@/views/system/RuntimeQueues.vue' import IntegrationSettingsSection from '@/views/integrations/IntegrationSettingsSection.vue' import { INTEGRATION_PREVIEW_ITEMS, @@ -259,7 +265,7 @@ const SECTION_MIN_ROLE: Record = { members: 'viewer', } -const SYSTEM_ADMIN_SECTIONS = new Set(['system-global']) +const SYSTEM_ADMIN_SECTIONS = new Set(['system-global', 'runtime-queues']) const INTEGRATION_SECTION_PREFIX = 'integration-' const integrationSectionKey = (tab: IntegrationTab) => `${INTEGRATION_SECTION_PREFIX}${tab}` @@ -329,6 +335,7 @@ const navItems = computed(() => { { key: 'mcp', icon: 'tools', label: t('settings.mcpService') }, { key: 'system', icon: 'info-circle', label: t('settings.versionInfo') }, { key: 'system-global', icon: 'server', label: t('settings.system') }, + { key: 'runtime-queues', icon: 'queue', label: t('settings.taskQueue') }, { key: 'userprofile', icon: 'user', label: t('userProfile.title') }, { key: 'tenant', icon: 'user-circle', label: t('settings.tenantInfo') }, { key: 'members', icon: 'usergroup', label: t('tenantMember.title') }, @@ -346,7 +353,7 @@ const navItems = computed(() => { const navGroups = computed(() => { const itemMap = new Map(navItems.value.map((item) => [item.key, item])) const pickItems = (keys: string[]) => keys.map((key) => itemMap.get(key)).filter(Boolean) as NavItem[] - // 分组:账户 → 空间 → 模型 → 发布集成 → 数据与扩展 → 平台(文案见 i18n settings.navGroups) + // 分组:账户 → 空间 → 模型 → 发布集成 → 数据与扩展 → 系统管理 → 平台 // 关键调整:把个人偏好(general)和用户信息收进「账户」; // 把空间内功能开关(chathistory)从「平台」挪到「空间」; // 把检索引擎和外部集成合并为「数据与扩展」,避免两个 2~3 项的窄分组。 @@ -388,10 +395,15 @@ const navGroups = computed(() => { 'mcp', ]), }, + { + key: 'system_administration', + label: t('settings.navGroups.systemAdministration'), + items: pickItems(['system-global', 'runtime-queues']), + }, { key: 'platform', label: t('settings.navGroups.platform'), - items: pickItems(['system-global', 'system']), + items: pickItems(['system']), }, ].filter((group) => group.items.length > 0) }) @@ -413,7 +425,7 @@ const handleNavClick = (item: any) => { // 切换到对应页面 currentSection.value = item.key - if (isIntegrationSection(item.key) && route.path === '/platform/settings') { + if (route.path === '/platform/settings' && isIntegrationSection(item.key)) { router.replace({ path: '/platform/settings', query: { @@ -422,6 +434,13 @@ const handleNavClick = (item: any) => { tab: integrationTabFromSection(item.key), }, }) + } else if (route.path === '/platform/settings' && SYSTEM_ADMIN_SECTIONS.has(item.key)) { + const query = { ...route.query } + delete query.tab + router.replace({ + path: '/platform/settings', + query: { ...query, section: item.key }, + }) } } @@ -450,7 +469,7 @@ const handleClose = () => { // 如果当前路由是设置页,返回上一页 if (route.path === '/platform/settings') { const sec = route.query.section - if (sec === 'system-global') { + if (sec === 'system-global' || sec === 'runtime-queues') { router.push('/platform/knowledge-bases') } else { router.back() diff --git a/frontend/src/views/system/RuntimeQueues.vue b/frontend/src/views/system/RuntimeQueues.vue new file mode 100644 index 00000000..29a3ec4e --- /dev/null +++ b/frontend/src/views/system/RuntimeQueues.vue @@ -0,0 +1,766 @@ + + + + + diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index 2edf26f3..beb3ae4a 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -120,6 +120,12 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge( return f.queued[knowledgeID], nil } +func (f fakeTaskInspector) QueueStats( + _ context.Context, +) ([]types.QueueStat, bool, error) { + return nil, false, nil +} + func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService { return newHousekeepingSvcWithInspector(db, fakeTaskInspector{}) } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index b91bd6f3..d975e160 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -2246,7 +2246,7 @@ func (s *knowledgeService) ReparseKnowledge( // - Any in-flight worker reads the new status at its next checkpoint and // bails (see processChunks / ProcessDocument / downstream handlers). // - The asynq inspector (if available) dequeues pending / scheduled / retry -// tasks for this knowledge_id across the default / critical / low queues +// tasks for this knowledge_id across all queues used by the pipeline // and signals active workers to stop. Lite mode (no Redis) skips the // dequeue step — the checkpoint-based abort is the only stop signal there. // - Idempotent: re-calling on an already-cancelled row is a no-op. diff --git a/internal/handler/system.go b/internal/handler/system.go index b360b811..3630bcbf 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -43,6 +43,11 @@ type SystemHandler struct { // tests that wire a partial container still compile. In production // the dig graph always provides one. auditSvc interfaces.AuditLogService + // taskInspector backs the SystemAdmin runtime queue dashboard. Always + // provided by the container (asynq-backed in Redis mode, a no-op in + // Lite mode), so GetRuntimeQueues can distinguish "no queues in this + // deployment" from "queues are empty". + taskInspector interfaces.TaskInspector } // NewSystemHandler creates a new system handler @@ -53,6 +58,7 @@ func NewSystemHandler(cfg *config.Config, userSvc interfaces.UserService, systemSettingSvc interfaces.SystemSettingService, auditSvc interfaces.AuditLogService, + taskInspector interfaces.TaskInspector, ) *SystemHandler { return &SystemHandler{ cfg: cfg, @@ -62,6 +68,7 @@ func NewSystemHandler(cfg *config.Config, userSvc: userSvc, systemSettingSvc: systemSettingSvc, auditSvc: auditSvc, + taskInspector: taskInspector, } } @@ -1374,6 +1381,76 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) { // @Success 200 {array} types.SystemSetting "list of settings" // @Failure 403 {object} map[string]interface{} "Forbidden: not a system admin" // @Router /system/admin/settings [get] +// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue +// dashboard. `available` is false in Lite mode (no Redis/asynq) so the +// UI can render an "unavailable in this deployment" state instead of an +// empty table. ParseConcurrency / WikiConcurrency are the *configured* +// worker-pool sizes (resolved DB > ENV > default); they are static +// capacity, not the live active-worker count (see QueueStat.Active for +// what's actually running). +type RuntimeQueuesResponse struct { + Available bool `json:"available"` + ParseConcurrency int `json:"parse_concurrency"` + WikiConcurrency int `json:"wiki_concurrency"` + Queues []types.QueueStat `json:"queues"` + Timestamp int64 `json:"timestamp"` +} + +// defaultParseConcurrency / defaultWikiConcurrency mirror the unexported +// router constants (defaultAsynqConcurrency / defaultWikiAsynqConcurrency). +// Duplicated here rather than exported from router to avoid an import +// cycle (router already imports handler); kept in sync by convention. +const ( + defaultParseConcurrency = 32 + defaultWikiConcurrency = 16 +) + +// GetRuntimeQueues godoc +// @Summary 获取解析任务队列运行时状态 +// @Description 返回各 asynq 队列的实时深度(pending/active/scheduled/retry 等)与 worker 并发配置,仅系统管理员可见 +// @Tags 系统管理 +// @Produce json +// @Success 200 {object} RuntimeQueuesResponse +// @Router /system/admin/runtime/queues [get] +func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) { + ctx := logger.CloneContext(c.Request.Context()) + resp := RuntimeQueuesResponse{ + ParseConcurrency: defaultParseConcurrency, + WikiConcurrency: defaultWikiConcurrency, + Timestamp: time.Now().Unix(), + } + + if h.taskInspector != nil { + stats, supported, err := h.taskInspector.QueueStats(ctx) + if err != nil { + logger.Errorf(ctx, "get queue stats failed: %v", err) + } + resp.Available = supported + if stats != nil { + resp.Queues = stats + } + } + // Always emit a non-nil array so the JSON serialises to `[]` rather + // than `null`, keeping the frontend iteration simple. + if resp.Queues == nil { + resp.Queues = []types.QueueStat{} + } + + // Resolve configured worker-pool sizes (DB > ENV > default). These + // are the same keys NewParseAsynqServer / NewWikiAsynqServer read at + // startup, so operators see the value that will apply on next restart. + if h.systemSettingSvc != nil { + resp.ParseConcurrency = int(h.systemSettingSvc.GetInt( + ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultParseConcurrency, + )) + resp.WikiConcurrency = int(h.systemSettingSvc.GetInt( + ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiConcurrency, + )) + } + + c.JSON(http.StatusOK, resp) +} + func (h *SystemHandler) ListSystemSettings(c *gin.Context) { ctx := logger.CloneContext(c.Request.Context()) rows, err := h.systemSettingSvc.List(ctx) diff --git a/internal/router/router.go b/internal/router/router.go index 39df783a..22b0883f 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -875,6 +875,12 @@ func RegisterSystemAdminRoutes( adminRoutes.PUT("/settings/:key", handler.UpdateSystemSetting) adminRoutes.DELETE("/settings/:key", handler.ResetSystemSetting) + // Runtime observability: live asynq queue depths + worker pool + // concurrency for the parse/wiki pools. Read-only snapshot for the + // SystemAdmin runtime dashboard. Returns available=false in Lite + // mode (no Redis) so the UI degrades gracefully. + adminRoutes.GET("/runtime/queues", handler.GetRuntimeQueues) + // Bulk action — write the current default-quota setting onto // every existing tenant. Lives under /tenants instead of // /settings because it changes tenants, not the setting row. diff --git a/internal/router/task.go b/internal/router/task.go index c779ebf4..87149487 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -182,7 +182,6 @@ func NewParseAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { log.Printf("asynq parse-pool server starting with concurrency=%d redis_op_timeout=%dms", concurrency, readRedisOpTimeoutMs()) return newAsynqServer(concurrency, map[string]int{ - types.QueueCritical: 6, // Highest priority queue types.QueueDefault: 3, // Default priority queue types.QueueLow: 1, // Lowest priority queue types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index c46f8088..42295121 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -18,8 +18,8 @@ func NewAsynqInspector() *asynq.Inspector { } // asynqTaskInspector implements interfaces.TaskInspector backed by an -// *asynq.Inspector. Scans the queues we actually use ("default", -// "critical", "low") and matches tasks whose payload carries the given +// *asynq.Inspector. Scans the queues we actually use and matches tasks +// whose payload carries the given // knowledge_id. Best-effort: any scan/delete error is logged and // swallowed so the cancel API still returns success even when Redis is // flaky. @@ -52,7 +52,6 @@ type knowledgeIDProbe struct { // its (potentially hundreds of) pending image:multimodal tasks. var queuesScanned = []string{ types.QueueDefault, - types.QueueCritical, types.QueueLow, types.QueueMultimodal, types.QueueGraph, @@ -60,6 +59,22 @@ var queuesScanned = []string{ types.QueueWiki, } +// queueMeta records each queue's worker pool and scheduling weight for +// the runtime dashboard. Weights MUST stay in sync with the maps in +// NewParseAsynqServer / NewWikiAsynqServer (task.go) — this is display +// metadata only, the servers remain the source of truth for scheduling. +var queueMeta = map[string]struct { + pool string + weight int +}{ + types.QueueDefault: {pool: "parse", weight: 3}, + types.QueueLow: {pool: "parse", weight: 1}, + types.QueueMultimodal: {pool: "parse", weight: 1}, + types.QueueGraph: {pool: "parse", weight: 1}, + types.QueueQuestion: {pool: "parse", weight: 1}, + types.QueueWiki: {pool: "wiki", weight: 1}, +} + // taskTypesForKnowledgeCancel lists every asynq task type that carries // a knowledge_id in its payload and should be cancelable. The set is // deliberately narrow: we don't touch FAQ import / KB-level tasks @@ -137,6 +152,53 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( return false, nil } +// QueueStats returns a depth snapshot for every queue this app enqueues +// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps +// the result onto types.QueueStat, attaching static pool/weight metadata +// from queueMeta. A queue that has never received a task yields +// ErrQueueNotFound from asynq; we still surface it as a zeroed row so the +// dashboard shows the complete lane set even before a queue receives its +// first task. +func (a *asynqTaskInspector) QueueStats( + ctx context.Context, +) ([]types.QueueStat, bool, error) { + if a == nil || a.inspector == nil { + return nil, false, nil + } + stats := make([]types.QueueStat, 0, len(queuesScanned)) + for _, queue := range queuesScanned { + meta := queueMeta[queue] + stat := types.QueueStat{ + Name: queue, + Pool: meta.pool, + Weight: meta.weight, + } + info, err := a.inspector.GetQueueInfo(queue) + if err != nil { + if !errors.Is(err, asynq.ErrQueueNotFound) { + logger.Warnf(ctx, "[TaskInspector] queue info queue=%s: %v", queue, err) + } + // Zeroed row: queue not created yet (or transient error). + stats = append(stats, stat) + continue + } + stat.Size = info.Size + stat.Pending = info.Pending + stat.Active = info.Active + stat.Scheduled = info.Scheduled + stat.Retry = info.Retry + stat.Archived = info.Archived + stat.Completed = info.Completed + stat.Processed = info.Processed + stat.Failed = info.Failed + stat.Paused = info.Paused + stat.LatencyMs = info.Latency.Milliseconds() + stat.MemoryUsageBytes = info.MemoryUsage + stats = append(stats, stat) + } + return stats, true, nil +} + // queueStateHasMatch pages through one (queue, state) list looking for a // task that references knowledgeID. Mirrors the delete* scanners but is // strictly read-only and returns early on the first hit. A backend error @@ -335,3 +397,12 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge( ) (bool, error) { return false, nil } + +// QueueStats reports "not supported" in Lite mode: there is no Redis / +// asynq backend to inspect, so the runtime dashboard renders an +// "unavailable in this deployment" state instead of an empty table. +func (noopTaskInspector) QueueStats( + ctx context.Context, +) ([]types.QueueStat, bool, error) { + return nil, false, nil +} diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go index ca597dfb..7ca1a901 100644 --- a/internal/types/interfaces/task_inspector.go +++ b/internal/types/interfaces/task_inspector.go @@ -1,6 +1,10 @@ package interfaces -import "context" +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) // TaskInspector abstracts queue inspection / cancellation against the // task backend. It is best-effort: implementations may scan a finite @@ -39,4 +43,18 @@ type TaskInspector interface { // always returns false — inline executors never queue, so the // span/updated_at checks remain authoritative there. HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error) + + // QueueStats returns a read-only depth snapshot for every queue this + // application enqueues into, for the System Admin runtime dashboard. + // + // The `supported` bool reports whether queue inspection is available + // on the current backend: true in asynq/Redis mode, false in Lite + // mode (no Redis) where tasks run inline and no queue exists. Callers + // use it to render an "unavailable in this deployment" state rather + // than an empty table. + // + // Best-effort: a per-queue backend error is logged and that queue is + // surfaced as a zeroed row (so the full lane set stays visible) rather + // than failing the whole call. + QueueStats(ctx context.Context) (stats []types.QueueStat, supported bool, err error) } diff --git a/internal/types/task.go b/internal/types/task.go index efe3632c..b529547f 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -4,9 +4,8 @@ package types // router.NewParseAsynqServer / router.NewWikiAsynqServer — a task enqueued to a // queue that no server subscribes to will never be consumed. const ( - QueueCritical = "critical" - QueueDefault = "default" - QueueLow = "low" + QueueDefault = "default" + QueueLow = "low" // QueueMultimodal isolates high-volume, slow VLM image tasks (OCR + caption) // so a single large scanned PDF (hundreds–thousands of page images) cannot // saturate the shared worker pool and block user-facing document parsing in @@ -33,6 +32,39 @@ const ( QueueWiki = "wiki" ) +// QueueStat is a read-only depth snapshot of a single asynq queue, used +// by the System Admin runtime dashboard. Field names mirror the counts +// exposed by asynq.QueueInfo. Pool / Weight are static metadata (which +// worker pool drains the queue and its scheduling weight within that +// pool) so the UI can group and explain the lanes without the frontend +// hard-coding the topology. +type QueueStat struct { + Name string `json:"name"` + // Pool is the worker pool that drains this queue: "parse" or "wiki". + Pool string `json:"pool"` + // Weight is the queue's scheduling weight inside its pool. Must match + // the maps in router.NewParseAsynqServer / NewWikiAsynqServer. + Weight int `json:"weight"` + // Size is the total number of tasks in the queue (pending + active + + // scheduled + retry + aggregating + archived). + Size int `json:"size"` + Pending int `json:"pending"` + Active int `json:"active"` + Scheduled int `json:"scheduled"` + Retry int `json:"retry"` + Archived int `json:"archived"` + Completed int `json:"completed"` + // Processed / Failed are today's counters (reset daily). + Processed int `json:"processed"` + Failed int `json:"failed"` + // Paused reports whether the queue is paused (tasks not consumed). + Paused bool `json:"paused"` + // LatencyMs is the age of the oldest pending task, in milliseconds. + LatencyMs int64 `json:"latency_ms"` + // MemoryUsageBytes is the approximate Redis memory the queue occupies. + MemoryUsageBytes int64 `json:"memory_usage_bytes"` +} + const ( TypeChunkExtract = "chunk:extract" TypeDocumentProcess = "document:process" // 文档处理任务 @@ -216,8 +248,8 @@ type KnowledgeListDeletePayload struct { // KnowledgeListReparsePayload represents the batch knowledge reparse task payload type KnowledgeListReparsePayload struct { TracingContext - TenantID uint64 `json:"tenant_id"` - KnowledgeIDs []string `json:"knowledge_ids"` + TenantID uint64 `json:"tenant_id"` + KnowledgeIDs []string `json:"knowledge_ids"` ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"` }