From 624e24ba49dece93f598c10a392cb6c88c7ab10c Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 2 Apr 2026 16:43:37 +0800 Subject: [PATCH] feat(asr): integrate Automatic Speech Recognition (ASR) support - Added ASR configuration options to the knowledge base and model settings. - Implemented ASR model selection and transcription capabilities in the knowledge processing pipeline. - Enhanced file upload support to include audio formats for ASR processing. - Updated localization files to include ASR-related labels and descriptions. - Refactored model handling to accommodate ASR as a new model type. --- frontend/src/api/initialization/index.ts | 5 + frontend/src/api/knowledge-base/index.ts | 5 + frontend/src/api/model/index.ts | 2 +- frontend/src/components/ModelEditorDialog.vue | 53 +++++--- frontend/src/components/ModelSelector.vue | 2 +- frontend/src/i18n/locales/en-US.ts | 20 ++- frontend/src/i18n/locales/ko-KR.ts | 20 ++- frontend/src/i18n/locales/ru-RU.ts | 22 +++- frontend/src/i18n/locales/zh-CN.ts | 18 +++ .../src/views/knowledge/KnowledgeBase.vue | 2 +- .../knowledge/KnowledgeBaseEditorModal.vue | 64 ++++++++++ frontend/src/views/settings/ModelSettings.vue | 73 +++++++++-- internal/application/service/file/s3.go | 8 ++ internal/application/service/knowledge.go | 86 ++++++++++++- internal/application/service/model.go | 48 +++++++ .../docparser/builtin_converter.go | 31 +++++ .../docparser/engine_registry.go | 2 +- internal/models/asr/asr.go | 44 +++++++ internal/models/asr/openai.go | 119 ++++++++++++++++++ internal/models/provider/aliyun.go | 2 + internal/models/provider/generic.go | 1 + internal/models/provider/gpustack.go | 2 + internal/models/provider/openai.go | 2 + internal/models/provider/siliconflow.go | 2 + internal/types/docparser.go | 2 + internal/types/interfaces/model.go | 3 + internal/types/knowledgebase.go | 31 +++++ internal/types/model.go | 1 + .../versioned/000025_add_asr_config.down.sql | 7 ++ .../versioned/000025_add_asr_config.up.sql | 10 ++ 30 files changed, 652 insertions(+), 35 deletions(-) create mode 100644 internal/models/asr/asr.go create mode 100644 internal/models/asr/openai.go create mode 100644 migrations/versioned/000025_add_asr_config.down.sql create mode 100644 migrations/versioned/000025_add_asr_config.up.sql diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index 928cec6e..7b8bdc6f 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -81,6 +81,11 @@ export interface KBModelConfigRequest { enabled: boolean model_id?: string } + asr_config?: { + enabled: boolean + model_id?: string + language?: string + } documentSplitting: { chunkSize: number chunkOverlap: number diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index e4946c4d..f5d23b56 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -21,6 +21,11 @@ export function createKnowledgeBase(data: { }; storage_provider_config?: { provider: string }; storage_config?: any; // legacy, kept for backward compat (dual-write) + asr_config?: { + enabled: boolean; + model_id?: string; + language?: string; + }; extract_config?: any; faq_config?: { index_mode: string; question_index_mode?: string }; }) { diff --git a/frontend/src/api/model/index.ts b/frontend/src/api/model/index.ts index 79448c4e..d886a344 100644 --- a/frontend/src/api/model/index.ts +++ b/frontend/src/api/model/index.ts @@ -8,7 +8,7 @@ export interface ModelConfig { id?: string; tenant_id?: number; name: string; - type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM'; + type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR'; source: 'local' | 'remote'; description?: string; parameters: { diff --git a/frontend/src/components/ModelEditorDialog.vue b/frontend/src/components/ModelEditorDialog.vue index 1534d4f3..8688e424 100644 --- a/frontend/src/components/ModelEditorDialog.vue +++ b/frontend/src/components/ModelEditorDialog.vue @@ -280,7 +280,7 @@ interface ModelFormData { interface Props { visible: boolean - modelType: 'chat' | 'embedding' | 'rerank' | 'vllm' + modelType: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr' modelData?: ModelFormData | null } @@ -303,29 +303,31 @@ const loadingProviders = ref(false) // 硬编码的后备 Provider 配置 (当 API 不可用时使用) const fallbackProviderOptions = computed(() => [ - { - value: 'openai', - label: t('model.editor.providers.openai.label'), + { + value: 'openai', + label: t('model.editor.providers.openai.label'), defaultUrls: { chat: 'https://api.openai.com/v1', embedding: 'https://api.openai.com/v1', rerank: 'https://api.openai.com/v1', - vllm: 'https://api.openai.com/v1' + vllm: 'https://api.openai.com/v1', + asr: 'https://api.openai.com/v1' }, description: t('model.editor.providers.openai.description'), - modelTypes: ['chat', 'embedding', 'vllm'] + modelTypes: ['chat', 'embedding', 'vllm', 'asr'] }, - { - value: 'aliyun', - label: t('model.editor.providers.aliyun.label'), + { + value: 'aliyun', + label: t('model.editor.providers.aliyun.label'), defaultUrls: { chat: 'https://dashscope.aliyuncs.com/compatible-mode/v1', embedding: 'https://dashscope.aliyuncs.com/compatible-mode/v1', rerank: 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank', - vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1' + vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1', + asr: 'https://dashscope.aliyuncs.com/compatible-mode/v1' }, description: t('model.editor.providers.aliyun.description'), - modelTypes: ['chat', 'embedding', 'rerank', 'vllm'] + modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr'] }, { value: 'zhipu', @@ -394,10 +396,10 @@ const fallbackProviderOptions = computed(() => [ }, { value: 'generic', - label: t('model.editor.providers.generic.label'), + label: t('model.editor.providers.generic.label'), defaultUrls: {}, description: t('model.editor.providers.generic.description'), - modelTypes: ['chat', 'embedding', 'rerank', 'vllm'] + modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr'] }, ]) @@ -536,15 +538,22 @@ const getModelNamePlaceholder = () => { ? t('model.editor.modelNamePlaceholder.localVllm') : t('model.editor.modelNamePlaceholder.remoteVllm') } + if (props.modelType === 'asr') { + return t('model.editor.modelNamePlaceholder.remoteAsr') + } return formData.value.source === 'local' ? t('model.editor.modelNamePlaceholder.local') : t('model.editor.modelNamePlaceholder.remote') } const getBaseUrlPlaceholder = () => { - return props.modelType === 'vllm' - ? t('model.editor.baseUrlPlaceholderVllm') - : t('model.editor.baseUrlPlaceholder') + if (props.modelType === 'vllm') { + return t('model.editor.baseUrlPlaceholderVllm') + } + if (props.modelType === 'asr') { + return t('model.editor.baseUrlPlaceholderAsr') + } + return t('model.editor.baseUrlPlaceholder') } // 检查Ollama服务状态 @@ -840,7 +849,17 @@ const checkRemoteAPI = async () => { apiKey: formData.value.apiKey || '' }) break - + + case 'asr': + // ASR 模型(语音识别) + // ASR 使用 checkRemoteModel 进行基础连接测试 + result = await checkRemoteModel({ + modelName: formData.value.modelName, + baseUrl: formData.value.baseUrl, + apiKey: formData.value.apiKey || '' + }) + break + default: MessagePlugin.error(t('model.editor.unsupportedModelType')) return diff --git a/frontend/src/components/ModelSelector.vue b/frontend/src/components/ModelSelector.vue index eab371f9..cff6f1bc 100644 --- a/frontend/src/components/ModelSelector.vue +++ b/frontend/src/components/ModelSelector.vue @@ -46,7 +46,7 @@ import { MessagePlugin } from 'tdesign-vue-next' import { useI18n } from 'vue-i18n' interface Props { - modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' + modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR' selectedModelId?: string disabled?: boolean placeholder?: string diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index feda27fc..5691ae51 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1471,6 +1471,7 @@ export default { graph: 'Knowledge Graph', multimodal: 'Multimodal', datasource: 'Data Sources', + asr: 'Speech Recognition', share: 'Sharing' }, basic: { @@ -1676,6 +1677,15 @@ export default { title: 'Multimodal Configuration', description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images', }, + asr: { + title: 'Speech Recognition Configuration', + description: 'Configure speech recognition to upload audio files (mp3, wav, m4a, flac, ogg) and automatically transcribe to text', + label: 'Enable Speech Recognition', + desc: 'When enabled, audio files can be uploaded to the knowledge base and automatically transcribed to text', + modelLabel: 'ASR Model', + modelDescription: 'Speech-to-text model for audio transcription (e.g. OpenAI Whisper)', + modelPlaceholder: 'Select an ASR model', + }, advanced: { title: 'Advanced Settings', description: 'Configure question generation and other advanced features', @@ -2076,17 +2086,20 @@ export default { embedding: 'Configure embedding models for text vectorization', rerank: 'Configure models for result re-ranking', vllm: 'Configure vision-language models for multimodal understanding', + asr: 'Configure speech-to-text models for audio transcription', default: 'Configure model information' }, modelNamePlaceholder: { local: 'e.g. llama2:latest', remote: 'e.g. gpt-4, claude-3-opus', localVllm: 'e.g. llava:latest', - remoteVllm: 'e.g. gpt-4-vision-preview' + remoteVllm: 'e.g. gpt-4-vision-preview', + remoteAsr: 'e.g. whisper-1' }, baseUrlLabel: 'Base URL', baseUrlPlaceholder: 'e.g. https://api.openai.com/v1', baseUrlPlaceholderVllm: 'e.g. http://localhost:11434/v1', + baseUrlPlaceholderAsr: 'e.g. https://api.openai.com/v1', apiKeyOptional: 'API Key (optional)', apiKeyPlaceholder: 'Enter API Key', connectionTest: 'Connection Test', @@ -2582,6 +2595,11 @@ export default { desc: 'Configure vision-language models for multimodal understanding', empty: 'No VLLM models' }, + asr: { + title: 'ASR Speech Models', + desc: 'Configure speech-to-text models for audio transcription (e.g. OpenAI Whisper)', + empty: 'No ASR models' + }, toasts: { nameRequired: 'Model name cannot be empty', nameTooLong: 'Model name cannot exceed 100 characters', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 2745b084..8d2751f6 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1534,6 +1534,7 @@ export default { embedding: "텍스트 벡터화용 임베딩 모델 설정", rerank: "결과 재정렬용 모델 설정", vllm: "시각 이해 및 멀티모달용 비전 언어 모델 설정", + asr: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정", default: "모델 정보 설정", }, modelNamePlaceholder: { @@ -1541,10 +1542,12 @@ export default { remote: "예: gpt-4, claude-3-opus", localVllm: "예: llava:latest", remoteVllm: "예: gpt-4-vision-preview", + remoteAsr: "예: whisper-1", }, baseUrlLabel: "Base URL", baseUrlPlaceholder: "예: https://api.openai.com/v1", baseUrlPlaceholderVllm: "예: http://localhost:11434/v1", + baseUrlPlaceholderAsr: "예: https://api.openai.com/v1", apiKeyOptional: "API 키 (선택)", apiKeyPlaceholder: "API 키 입력", connectionTest: "연결 테스트", @@ -1955,6 +1958,7 @@ export default { faq: "FAQ 설정", graph: "지식 그래프", multimodal: "멀티모달", + asr: "음성 인식", share: "공유관리", storage: "스토리지 엔진", datasource: "데이터 소스", @@ -2170,7 +2174,16 @@ export default { }, multimodal: { title: "멀티모달 설정", - description: "멀티모달 콘텐츠 이해 기능을 설정합니다. 활성화하면 이미지 등 텍스트가 아닌 콘텐츠의 파싱 및 검색을 지원합니다.", + description: "멀티모달 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다", + }, + asr: { + title: "음성 인식 설정", + description: "음성 인식 기능을 설정하여 오디오 파일(mp3, wav, m4a, flac, ogg)을 업로드하고 자동으로 텍스트로 전사합니다", + label: "음성 인식 활성화", + desc: "활성화하면 오디오 파일을 지식베이스에 업로드하고 자동으로 텍스트로 전사할 수 있습니다", + modelLabel: "ASR 모델", + modelDescription: "오디오 전사를 위한 음성 인식 모델 (예: OpenAI Whisper)", + modelPlaceholder: "ASR 모델을 선택해주세요", }, advanced: { title: "고급 설정", @@ -2615,6 +2628,11 @@ export default { desc: "시각 이해 및 멀티모달용 비전 언어 모델 설정", empty: "VLLM 비전 모델 없음", }, + asr: { + title: "ASR 음성 모델", + desc: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정 (예: OpenAI Whisper)", + empty: "ASR 음성 모델 없음", + }, toasts: { nameRequired: "모델 이름은 비워둘 수 없습니다", nameTooLong: "모델 이름은 100자를 초과할 수 없습니다", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index d147e468..ab9be06c 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1381,17 +1381,20 @@ export default { embedding: 'Настройте модель встраивания для текстовой векторизации', rerank: 'Настройте модель для повторного ранжирования результатов', vllm: 'Настройте визуально-языковую модель для мультимодального понимания', + asr: 'Настройте модель распознавания речи для транскрибации аудио', default: 'Настройте информацию о модели' }, modelNamePlaceholder: { local: 'например: llama2:latest', remote: 'например: gpt-4, claude-3-opus', localVllm: 'например: llava:latest', - remoteVllm: 'например: gpt-4-vision-preview' + remoteVllm: 'например: gpt-4-vision-preview', + remoteAsr: 'например: whisper-1' }, baseUrlLabel: 'Base URL', baseUrlPlaceholder: 'например: https://api.openai.com/v1', baseUrlPlaceholderVllm: 'например: http://localhost:11434/v1', + baseUrlPlaceholderAsr: 'например: https://api.openai.com/v1', apiKeyOptional: 'API Key (опционально)', apiKeyPlaceholder: 'Введите API Key', connectionTest: 'Проверка соединения', @@ -1641,6 +1644,7 @@ export default { faq: 'FAQ настройки', graph: 'Граф знаний', multimodal: 'Мультимодальность', + asr: 'Распознавание речи', storage: 'Storage Engine', datasource: 'Источники данных', share: 'Sharing' @@ -1836,7 +1840,16 @@ export default { }, multimodal: { title: 'Мультимодальная конфигурация', - description: 'Настройка понимания мультимодального контента для парсинга и поиска нетекстового содержимого, такого как изображения', + description: 'Настройте мультимодальное понимание контента для парсинга и поиска нетекстового контента, например изображений', + }, + asr: { + title: 'Конфигурация распознавания речи', + description: 'Настройте распознавание речи для загрузки аудиофайлов (mp3, wav, m4a, flac, ogg) и автоматической транскрибации в текст', + label: 'Включить распознавание речи', + desc: 'При включении аудиофайлы могут быть загружены в базу знаний и автоматически транскрибированы в текст', + modelLabel: 'Модель ASR', + modelDescription: 'Модель распознавания речи для транскрибации аудио (например, OpenAI Whisper)', + modelPlaceholder: 'Выберите модель ASR', }, advanced: { title: 'Расширенные настройки', @@ -2380,6 +2393,11 @@ export default { desc: 'Визуально-языковые модели для мультимодального понимания', empty: 'Нет VLLM моделей' }, + asr: { + title: 'ASR модели речи', + desc: 'Модели распознавания речи для транскрибации аудио (например, OpenAI Whisper)', + empty: 'Нет ASR моделей' + }, toasts: { nameRequired: 'Название модели не может быть пустым', nameTooLong: 'Название модели не может превышать 100 символов', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 7235ba49..ae983723 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1528,6 +1528,7 @@ export default { embedding: "配置用于文本向量化的嵌入模型", rerank: "配置用于结果重排序的模型", vllm: "配置用于视觉理解和多模态的视觉语言模型", + asr: "配置用于语音识别和音频转录的语音转文本模型", default: "配置模型信息", }, modelNamePlaceholder: { @@ -1535,10 +1536,12 @@ export default { remote: "例如:gpt-4, claude-3-opus", localVllm: "例如:llava:latest", remoteVllm: "例如:gpt-4-vision-preview", + remoteAsr: "例如:whisper-1", }, baseUrlLabel: "Base URL", baseUrlPlaceholder: "例如:https://api.openai.com/v1", baseUrlPlaceholderVllm: "例如:http://localhost:11434/v1", + baseUrlPlaceholderAsr: "例如:https://api.openai.com/v1", apiKeyOptional: "API Key(可选)", apiKeyPlaceholder: "输入 API Key", connectionTest: "连接测试", @@ -1943,6 +1946,7 @@ export default { graph: "知识图谱", multimodal: "多模态", datasource: "数据源", + asr: "语音识别", share: "共享管理", }, basic: { @@ -2145,6 +2149,15 @@ export default { title: "多模态配置", description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索", }, + asr: { + title: "语音识别配置", + description: "配置语音识别功能,启用后支持上传音频文件(mp3、wav、m4a、flac、ogg)并自动转录为文本", + label: "启用语音识别", + desc: "启用后可以上传音频文件到知识库,系统将自动转录为文本进行处理", + modelLabel: "ASR 模型", + modelDescription: "用于语音转文本的语音识别模型(如 OpenAI Whisper)", + modelPlaceholder: "请选择 ASR 模型", + }, advanced: { title: "高级设置", description: "配置问题生成等高级功能", @@ -2580,6 +2593,11 @@ export default { desc: "配置用于视觉理解和多模态的视觉语言模型", empty: "暂无 VLLM 视觉模型", }, + asr: { + title: "ASR 语音模型", + desc: "配置用于语音识别和音频转录的语音转文本模型(如 OpenAI Whisper)", + empty: "暂无 ASR 语音模型", + }, toasts: { nameRequired: "模型名称不能为空", nameTooLong: "模型名称不能超过100个字符", diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index cddc8d7e..a9d324c3 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -1514,7 +1514,7 @@ async function createNewSession(value: string): Promise { ref="uploadInputRef" type="file" class="document-upload-input" - :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'" + :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'" multiple @change="handleDocumentUpload" /> diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 68397428..e220c2bb 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -202,6 +202,49 @@ + +
+
+
+

{{ $t('knowledgeEditor.asr.title') }}

+

{{ $t('knowledgeEditor.asr.description') }}

+
+ +
+ +
+
+ +

{{ $t('knowledgeEditor.asr.desc') }}

+
+
+ +
+
+ + +
+
+ +

{{ $t('knowledgeEditor.asr.modelDescription') }}

+
+
+ +
+
+
+
+
+
{ { key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') }, { key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') }, { key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') }, + { key: 'asr', icon: 'sound', label: t('knowledgeEditor.sidebar.asr') }, { key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') } ) if (props.mode === 'edit' && props.kbId) { @@ -363,6 +407,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => { enabled: false, vllmModelId: '' }, + asrConfig: { + enabled: false, + modelId: '', + language: '' + }, nodeExtractConfig: { enabled: false, text: '', @@ -443,6 +492,11 @@ const loadKBData = async () => { enabled: !!kb.vlm_config?.enabled, vllmModelId: kb.vlm_config?.model_id || '' }, + asrConfig: { + enabled: !!kb.asr_config?.enabled, + modelId: kb.asr_config?.model_id || '', + language: kb.asr_config?.language || '' + }, nodeExtractConfig: { enabled: kb.extract_config?.enabled || false, text: kb.extract_config?.text || '', @@ -593,6 +647,15 @@ const buildSubmitData = () => { : '' } + // 添加ASR语音识别配置 + data.asr_config = { + enabled: formData.value.asrConfig?.enabled || false, + model_id: formData.value.asrConfig?.enabled + ? (formData.value.asrConfig?.modelId || '') + : '', + language: formData.value.asrConfig?.language || '' + } + // 存储引擎:仅传 provider,参数从全局设置读取 // Write to storage_provider_config (authoritative) + storage_config (legacy dual-write) data.storage_provider_config = { @@ -705,6 +768,7 @@ const doSubmit = async () => { llmModelId: data.summary_model_id, embeddingModelId: data.embedding_model_id, vlm_config: data.vlm_config, + asr_config: data.asr_config, documentSplitting: { chunkSize: data.chunking_config.chunk_size, chunkOverlap: data.chunking_config.chunk_overlap, diff --git a/frontend/src/views/settings/ModelSettings.vue b/frontend/src/views/settings/ModelSettings.vue index b8d2b398..8647f976 100644 --- a/frontend/src/views/settings/ModelSettings.vue +++ b/frontend/src/views/settings/ModelSettings.vue @@ -221,6 +221,54 @@
+ +
+
+
+

{{ $t('modelSettings.asr.title') }}

+

{{ $t('modelSettings.asr.desc') }}

+
+ + + {{ $t('modelSettings.actions.addModel') }} + +
+ +
+
+
+
+ {{ model.name }} + {{ $t('modelSettings.builtinTag') }} +
+
+ {{ model.source === 'local' ? 'Ollama' : $t('modelSettings.source.openaiCompatible') }} +
+
+
+ + + + + +
+
+
+
+

{{ $t('modelSettings.asr.empty') }}

+ + {{ $t('modelSettings.actions.addModel') }} + +
+
+ ('chat') +const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'>('chat') const editingModel = ref(null) const loading = ref(true) @@ -268,12 +316,18 @@ const rerankModels = computed(() => .map(convertToLegacyFormat) ) -const vllmModels = computed(() => +const vllmModels = computed(() => allModels.value .filter(m => m.type === 'VLLM') .map(convertToLegacyFormat) ) +const asrModels = computed(() => + allModels.value + .filter(m => m.type === 'ASR') + .map(convertToLegacyFormat) +) + // 将后端模型格式转换为旧的前端格式 function convertToLegacyFormat(model: ModelConfig) { return { @@ -306,14 +360,14 @@ const loadModels = async () => { } // 打开添加对话框 -const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm') => { +const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr') => { currentModelType.value = type editingModel.value = null showDialog.value = true } // 编辑模型 -const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => { +const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => { // 内置模型不能编辑 if (model.isBuiltin) { MessagePlugin.warning(t('modelSettings.toasts.builtinCannotEdit')) @@ -405,7 +459,7 @@ const handleModelSave = async (modelData: any) => { } // 删除模型 -const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', modelId: string) => { +const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', modelId: string) => { // 检查是否是内置模型 const model = allModels.value.find(m => m.id === modelId) if (model?.is_builtin) { @@ -425,7 +479,7 @@ const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model } // 获取模型操作菜单选项 -const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => { +const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => { const options: any[] = [] // 内置模型不能编辑和删除 @@ -450,7 +504,7 @@ const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: } // 处理菜单操作 -const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => { +const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => { const value = data.value if (value.indexOf('edit-') === 0) { @@ -464,12 +518,13 @@ const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | } // 获取后端模型类型 -function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' { +function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR' { const typeMap = { chat: 'KnowledgeQA' as const, embedding: 'Embedding' as const, rerank: 'Rerank' as const, - vllm: 'VLLM' as const + vllm: 'VLLM' as const, + asr: 'ASR' as const } return typeMap[type] } diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go index 101374a6..172cc4bf 100644 --- a/internal/application/service/file/s3.go +++ b/internal/application/service/file/s3.go @@ -209,6 +209,14 @@ func getContentTypeByExt(ext string) string { return "image/svg+xml" case ".mp3": return "audio/mpeg" + case ".wav": + return "audio/wav" + case ".m4a": + return "audio/mp4" + case ".flac": + return "audio/flac" + case ".ogg": + return "audio/ogg" case ".mp4": return "video/mp4" default: diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index dd709cfa..18f91c52 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -259,6 +259,15 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, logger.Info(ctx, "Image multimodal configuration validation passed") } + // 检查音频ASR配置完整性 - 只在音频文件时校验 + if IsAudioType(getFileType(fileName)) { + if !kb.ASRConfig.IsASREnabled() { + logger.Error(ctx, "ASR model is not configured") + return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型") + } + logger.Info(ctx, "Audio ASR configuration validation passed") + } + // Validate file type logger.Infof(ctx, "Checking file type: %s", fileName) if !isValidFileType(fileName) { @@ -596,6 +605,11 @@ var allowedFileURLExtensions = map[string]bool{ "pdf": true, "docx": true, "doc": true, + "mp3": true, + "wav": true, + "m4a": true, + "flac": true, + "ogg": true, } // maxFileURLSize is the maximum allowed file size for file URL import (10MB) @@ -2987,7 +3001,8 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { - case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json": + case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", + "mp3", "wav", "m4a", "flac", "ogg": return true default: return false @@ -7399,6 +7414,16 @@ func IsImageType(fileType string) bool { } } +// IsAudioType checks if a file type is an audio format +func IsAudioType(fileType string) bool { + switch strings.ToLower(fileType) { + case "mp3", "wav", "m4a", "flac", "ogg": + return true + default: + return false + } +} + // downloadFileFromURL downloads a remote file to a temp file and returns its binary content. // payloadFileName and payloadFileType are in/out pointers: if they point to an empty string, // the function resolves the value from Content-Disposition / URL path and writes it back. @@ -7641,6 +7666,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } + // 检查音频ASR配置(仅对文件导入) + if payload.FilePath != "" && IsAudioType(payload.FileType) && !kb.ASRConfig.IsASREnabled() { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + Errorf("processDocument audio without ASR model configured") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "上传音频文件需要设置ASR语音识别模型" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + return nil + } + // New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks var convertResult *types.ReadResult var chunks []types.ParsedChunk @@ -7764,6 +7800,54 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e } } + // Step 1.5: ASR transcription for audio files + if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 { + if !kb.ASRConfig.IsASREnabled() { + logger.Error(ctx, "Audio file detected but ASR is not configured") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "ASR model is not configured for audio transcription" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + return nil + } + + logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes", + knowledge.ID, len(convertResult.AudioData)) + + asrModel, err := s.modelService.GetASRModel(ctx, kb.ASRConfig.ModelID) + if err != nil { + logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err) + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = fmt.Sprintf("failed to get ASR model: %v", err) + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + return nil + } + + transcribedText, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName) + if err != nil { + logger.Errorf(ctx, "[ASR] Transcription failed: %v", err) + if isLastRetry { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = fmt.Sprintf("audio transcription failed: %v", err) + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + } + return fmt.Errorf("audio transcription failed: %w", err) + } + + if transcribedText == "" { + logger.Warn(ctx, "[ASR] Transcription returned empty text") + transcribedText = "[No speech detected in audio file]" + } + + logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(transcribedText)) + // Replace the audio placeholder with the transcribed text + convertResult.MarkdownContent = transcribedText + convertResult.IsAudio = false + convertResult.AudioData = nil + } + // Step 2: Store images and update markdown references var storedImages []docparser.StoredImage diff --git a/internal/application/service/model.go b/internal/application/service/model.go index 38941e18..79f92825 100644 --- a/internal/application/service/model.go +++ b/internal/application/service/model.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" @@ -458,3 +459,50 @@ func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM // Note: default model selection logic has been removed; models no longer // maintain a per-type default flag at the service layer. + +// GetASRModel retrieves and initializes an automatic speech recognition model instance. +func (s *modelService) GetASRModel(ctx context.Context, modelId string) (asr.ASR, error) { + if modelId == "" { + return nil, errors.New("model ID cannot be empty") + } + + tenantID := types.MustTenantIDFromContext(ctx) + + model, err := s.repo.GetByID(ctx, tenantID, modelId) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": modelId, + "tenant_id": tenantID, + }) + return nil, err + } + + if model == nil { + return nil, ErrModelNotFound + } + + logger.Infof(ctx, "Getting ASR model: %s, source: %s", model.Name, model.Source) + + ifType := model.Parameters.InterfaceType + if ifType == "" { + ifType = "openai" + } + + sttModel, err := asr.NewASR(&asr.Config{ + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + InterfaceType: ifType, + }) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "model_name": model.Name, + }) + return nil, err + } + + return sttModel, nil +} diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go index 385116ff..3779cfa7 100644 --- a/internal/infrastructure/docparser/builtin_converter.go +++ b/internal/infrastructure/docparser/builtin_converter.go @@ -24,10 +24,17 @@ var imageFormats = map[string]bool{ "bmp": true, "tiff": true, "webp": true, } +var audioFormats = map[string]bool{ + "mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true, +} + func init() { for k := range imageFormats { simpleFormats[k] = true } + for k := range audioFormats { + simpleFormats[k] = true + } } // IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader. @@ -65,6 +72,8 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t return &types.ReadResult{MarkdownContent: md}, nil case imageFormats[ft]: return imageToResult(req.FileName, req.FileContent), nil + case audioFormats[ft]: + return audioToResult(req.FileName, req.FileContent), nil default: return nil, fmt.Errorf("unsupported simple format: %s", ft) } @@ -99,6 +108,28 @@ func IsImageFormat(fileType string) bool { return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))] } +// IsAudioFormat returns true if the file type is a recognized audio format. +func IsAudioFormat(fileType string) bool { + return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))] +} + +// audioToResult wraps a standalone audio file. The actual transcription is +// handled by the ASR model in the knowledge service pipeline. Here we just +// return a placeholder markdown with the raw bytes preserved for upstream +// processing. +func audioToResult(fileName string, data []byte) *types.ReadResult { + if fileName == "" { + fileName = "audio.mp3" + } + // Return a placeholder; the knowledge service will replace this with + // the ASR transcription result. + return &types.ReadResult{ + MarkdownContent: fmt.Sprintf("[Audio file: %s]", fileName), + IsAudio: true, + AudioData: data, + } +} + // ensureOriginalImageRef checks whether the input file is an image and, if the // returned markdown does not already contain a markdown image reference for it, // prepends one and appends the raw bytes to imageRefs. This guarantees that diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 4a5940ee..8469aff1 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string { return "Simple format & image parsing (no external service required)" } func (e *simpleEngine) FileTypes(_ bool) []string { - return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"} + return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} } func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { return true, "" diff --git a/internal/models/asr/asr.go b/internal/models/asr/asr.go new file mode 100644 index 00000000..f7e3951c --- /dev/null +++ b/internal/models/asr/asr.go @@ -0,0 +1,44 @@ +package asr + +import ( + "context" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +// ASR defines the interface for Automatic Speech Recognition model operations. +type ASR interface { + // Transcribe sends audio bytes to the ASR model and returns the transcribed text. + Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error) + + GetModelName() string + GetModelID() string +} + +// Config holds the configuration needed to create an ASR instance. +type Config struct { + Source types.ModelSource + BaseURL string + ModelName string + APIKey string + ModelID string + InterfaceType string // "openai" (default) + Language string // optional: specify language for transcription +} + +// NewASR creates an ASR instance based on the provided configuration. +func NewASR(config *Config) (ASR, error) { + ifType := strings.ToLower(config.InterfaceType) + if ifType == "" { + ifType = "openai" + } + + switch ifType { + case "openai": + return NewOpenAIASR(config) + default: + return nil, fmt.Errorf("unsupported ASR interface type: %s", ifType) + } +} diff --git a/internal/models/asr/openai.go b/internal/models/asr/openai.go new file mode 100644 index 00000000..ee5e85bc --- /dev/null +++ b/internal/models/asr/openai.go @@ -0,0 +1,119 @@ +package asr + +import ( + "bytes" + "context" + "fmt" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + openai "github.com/sashabaranov/go-openai" +) + +const ( + asrDefaultTimeout = 300 * time.Second // audio transcription can be slow +) + +// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API. +type OpenAIASR struct { + modelName string + modelID string + client *openai.Client + baseURL string + language string +} + +// NewOpenAIASR creates an OpenAI-compatible ASR instance. +func NewOpenAIASR(config *Config) (*OpenAIASR, error) { + apiCfg := openai.DefaultConfig(config.APIKey) + if config.BaseURL != "" { + apiCfg.BaseURL = config.BaseURL + } + apiCfg.HTTPClient = &http.Client{Timeout: asrDefaultTimeout} + + return &OpenAIASR{ + modelName: config.ModelName, + modelID: config.ModelID, + client: openai.NewClientWithConfig(apiCfg), + baseURL: config.BaseURL, + language: config.Language, + }, nil +} + +// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API. +func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error) { + if len(audioBytes) == 0 { + return "", fmt.Errorf("audio bytes are empty") + } + + // Ensure fileName has an extension for proper MIME type detection + if fileName == "" { + fileName = "audio.mp3" + } + + logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s", + s.modelName, s.baseURL, len(audioBytes), fileName) + + req := openai.AudioRequest{ + Model: s.modelName, + FilePath: fileName, + Reader: bytes.NewReader(audioBytes), + Format: openai.AudioResponseFormatJSON, + } + + if s.language != "" { + req.Language = s.language + } + + resp, err := s.client.CreateTranscription(ctx, req) + if err != nil { + return "", fmt.Errorf("ASR transcription request failed: %w", err) + } + + text := strings.TrimSpace(resp.Text) + logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text)) + return text, nil +} + +func (s *OpenAIASR) GetModelName() string { return s.modelName } +func (s *OpenAIASR) GetModelID() string { return s.modelID } + +// DetectAudioFormat returns a file extension hint for the given audio bytes. +func DetectAudioFormat(data []byte, fileName string) string { + if fileName != "" { + ext := strings.ToLower(filepath.Ext(fileName)) + if ext != "" { + return ext + } + } + // Try to detect from magic bytes + if len(data) >= 4 { + // MP3: starts with ID3 or 0xFF 0xFB + if data[0] == 'I' && data[1] == 'D' && data[2] == '3' { + return ".mp3" + } + if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 { + return ".mp3" + } + // FLAC: starts with "fLaC" + if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' { + return ".flac" + } + // OGG: starts with "OggS" + if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' { + return ".ogg" + } + // WAV: starts with "RIFF" + if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' { + return ".wav" + } + } + // M4A: check for ftyp box + if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' { + return ".m4a" + } + return ".mp3" // default +} diff --git a/internal/models/provider/aliyun.go b/internal/models/provider/aliyun.go index a6477412..3ddcf62f 100644 --- a/internal/models/provider/aliyun.go +++ b/internal/models/provider/aliyun.go @@ -32,12 +32,14 @@ func (p *AliyunProvider) Info() ProviderInfo { types.ModelTypeEmbedding: AliyunChatBaseURL, types.ModelTypeRerank: AliyunRerankBaseURL, types.ModelTypeVLLM: AliyunChatBaseURL, + types.ModelTypeASR: AliyunChatBaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RequiresAuth: true, } diff --git a/internal/models/provider/generic.go b/internal/models/provider/generic.go index fe8aa862..570e6e93 100644 --- a/internal/models/provider/generic.go +++ b/internal/models/provider/generic.go @@ -25,6 +25,7 @@ func (p *GenericProvider) Info() ProviderInfo { types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RequiresAuth: false, // 可能需要也可能不需要 } diff --git a/internal/models/provider/gpustack.go b/internal/models/provider/gpustack.go index 686b3516..20ccf531 100644 --- a/internal/models/provider/gpustack.go +++ b/internal/models/provider/gpustack.go @@ -31,12 +31,14 @@ func (p *GPUStackProvider) Info() ProviderInfo { types.ModelTypeEmbedding: GPUStackBaseURL, types.ModelTypeRerank: GPUStackRerankBaseURL, types.ModelTypeVLLM: GPUStackBaseURL, + types.ModelTypeASR: GPUStackBaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RequiresAuth: true, // GPUStack 需要 API Key } diff --git a/internal/models/provider/openai.go b/internal/models/provider/openai.go index f0ef0dcd..208e0cb9 100644 --- a/internal/models/provider/openai.go +++ b/internal/models/provider/openai.go @@ -28,12 +28,14 @@ func (p *OpenAIProvider) Info() ProviderInfo { types.ModelTypeEmbedding: OpenAIBaseURL, types.ModelTypeRerank: OpenAIBaseURL, types.ModelTypeVLLM: OpenAIBaseURL, + types.ModelTypeASR: OpenAIBaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RequiresAuth: true, } diff --git a/internal/models/provider/siliconflow.go b/internal/models/provider/siliconflow.go index fa310226..f1b3f328 100644 --- a/internal/models/provider/siliconflow.go +++ b/internal/models/provider/siliconflow.go @@ -28,12 +28,14 @@ func (p *SiliconFlowProvider) Info() ProviderInfo { types.ModelTypeEmbedding: SiliconFlowBaseURL, types.ModelTypeRerank: SiliconFlowBaseURL, types.ModelTypeVLLM: SiliconFlowBaseURL, + types.ModelTypeASR: SiliconFlowBaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RequiresAuth: true, } diff --git a/internal/types/docparser.go b/internal/types/docparser.go index 1fb46eef..861a8af4 100644 --- a/internal/types/docparser.go +++ b/internal/types/docparser.go @@ -20,6 +20,8 @@ type ReadResult struct { ImageDirPath string Metadata map[string]string Error string + IsAudio bool // true when the result contains raw audio data needing ASR transcription + AudioData []byte // raw audio bytes for ASR processing } // ImageRef represents an image reference extracted from the document. diff --git a/internal/types/interfaces/model.go b/internal/types/interfaces/model.go index 88ada182..60acd698 100644 --- a/internal/types/interfaces/model.go +++ b/internal/types/interfaces/model.go @@ -6,6 +6,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" ) @@ -32,6 +33,8 @@ type ModelService interface { GetChatModel(ctx context.Context, modelId string) (chat.Chat, error) // GetVLMModel gets a vision language model GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error) + // GetASRModel gets an automatic speech recognition model + GetASRModel(ctx context.Context, modelId string) (asr.ASR, error) } // ModelRepository defines the model repository interface diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 27af88bb..e280f7c8 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -60,6 +60,8 @@ type KnowledgeBase struct { SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"` // VLM config VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"` + // ASR config (Automatic Speech Recognition) + ASRConfig ASRConfig `yaml:"asr_config" json:"asr_config" gorm:"type:json"` // Storage provider config (new): only stores provider selection; credentials from tenant StorageEngineConfig StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"` // Deprecated: legacy COS config column. Kept for backward compatibility with old data. @@ -386,6 +388,35 @@ func (c *VLMConfig) Scan(value interface{}) error { return json.Unmarshal(b, c) } +// ASRConfig represents the ASR (Automatic Speech Recognition) configuration +type ASRConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + ModelID string `yaml:"model_id" json:"model_id"` + Language string `yaml:"language" json:"language"` // optional: language hint for transcription +} + +// IsASREnabled checks if ASR is enabled with a valid model +func (c ASRConfig) IsASREnabled() bool { + return c.Enabled && c.ModelID != "" +} + +// Value implements the driver.Valuer interface, used to convert ASRConfig to database value +func (c ASRConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to ASRConfig +func (c *ASRConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + // ExtractConfig represents the extract configuration for a knowledge base type ExtractConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` diff --git a/internal/types/model.go b/internal/types/model.go index 78938570..b5a52d5f 100644 --- a/internal/types/model.go +++ b/internal/types/model.go @@ -18,6 +18,7 @@ const ( ModelTypeRerank ModelType = "Rerank" // Rerank model ModelTypeKnowledgeQA ModelType = "KnowledgeQA" // KnowledgeQA model ModelTypeVLLM ModelType = "VLLM" // VLLM model + ModelTypeASR ModelType = "ASR" // ASR (Automatic Speech Recognition) model ) // ModelStatus represents the status of the model diff --git a/migrations/versioned/000025_add_asr_config.down.sql b/migrations/versioned/000025_add_asr_config.down.sql new file mode 100644 index 00000000..c4975456 --- /dev/null +++ b/migrations/versioned/000025_add_asr_config.down.sql @@ -0,0 +1,7 @@ +-- Migration: 000025_add_asr_config (rollback) +-- Description: Remove asr_config column from knowledge_bases. +DO $$ BEGIN RAISE NOTICE '[Migration 000025] Removing asr_config column from knowledge_bases'; END $$; + +ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS asr_config; + +DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column removed successfully'; END $$; diff --git a/migrations/versioned/000025_add_asr_config.up.sql b/migrations/versioned/000025_add_asr_config.up.sql new file mode 100644 index 00000000..2b43731e --- /dev/null +++ b/migrations/versioned/000025_add_asr_config.up.sql @@ -0,0 +1,10 @@ +-- Migration: 000025_add_asr_config +-- Description: Add asr_config column to knowledge_bases for ASR (Automatic Speech Recognition) configuration. +-- ASR config stores the model ID and language settings for audio transcription. +DO $$ BEGIN RAISE NOTICE '[Migration 000025] Adding asr_config column to knowledge_bases'; END $$; + +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS asr_config JSONB; + +COMMENT ON COLUMN knowledge_bases.asr_config IS 'ASR (Automatic Speech Recognition) configuration: {"enabled": bool, "model_id": string, "language": string}'; + +DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column added successfully'; END $$;