feat(asr): integrate Automatic Speech Recognition (ASR) support

- Added ASR configuration options to the knowledge base and model settings.
- Implemented ASR model selection and transcription capabilities in the knowledge processing pipeline.
- Enhanced file upload support to include audio formats for ASR processing.
- Updated localization files to include ASR-related labels and descriptions.
- Refactored model handling to accommodate ASR as a new model type.
This commit is contained in:
wizardchen
2026-04-02 16:43:37 +08:00
committed by lyingbug
parent d68c94a11a
commit 624e24ba49
30 changed files with 652 additions and 35 deletions
+5
View File
@@ -81,6 +81,11 @@ export interface KBModelConfigRequest {
enabled: boolean
model_id?: string
}
asr_config?: {
enabled: boolean
model_id?: string
language?: string
}
documentSplitting: {
chunkSize: number
chunkOverlap: number
+5
View File
@@ -21,6 +21,11 @@ export function createKnowledgeBase(data: {
};
storage_provider_config?: { provider: string };
storage_config?: any; // legacy, kept for backward compat (dual-write)
asr_config?: {
enabled: boolean;
model_id?: string;
language?: string;
};
extract_config?: any;
faq_config?: { index_mode: string; question_index_mode?: string };
}) {
+1 -1
View File
@@ -8,7 +8,7 @@ export interface ModelConfig {
id?: string;
tenant_id?: number;
name: string;
type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM';
type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR';
source: 'local' | 'remote';
description?: string;
parameters: {
+36 -17
View File
@@ -280,7 +280,7 @@ interface ModelFormData {
interface Props {
visible: boolean
modelType: 'chat' | 'embedding' | 'rerank' | 'vllm'
modelType: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'
modelData?: ModelFormData | null
}
@@ -303,29 +303,31 @@ const loadingProviders = ref(false)
// 硬编码的后备 Provider 配置 (当 API 不可用时使用)
const fallbackProviderOptions = computed(() => [
{
value: 'openai',
label: t('model.editor.providers.openai.label'),
{
value: 'openai',
label: t('model.editor.providers.openai.label'),
defaultUrls: {
chat: 'https://api.openai.com/v1',
embedding: 'https://api.openai.com/v1',
rerank: 'https://api.openai.com/v1',
vllm: 'https://api.openai.com/v1'
vllm: 'https://api.openai.com/v1',
asr: 'https://api.openai.com/v1'
},
description: t('model.editor.providers.openai.description'),
modelTypes: ['chat', 'embedding', 'vllm']
modelTypes: ['chat', 'embedding', 'vllm', 'asr']
},
{
value: 'aliyun',
label: t('model.editor.providers.aliyun.label'),
{
value: 'aliyun',
label: t('model.editor.providers.aliyun.label'),
defaultUrls: {
chat: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
embedding: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
rerank: 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank',
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
asr: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
},
description: t('model.editor.providers.aliyun.description'),
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr']
},
{
value: 'zhipu',
@@ -394,10 +396,10 @@ const fallbackProviderOptions = computed(() => [
},
{
value: 'generic',
label: t('model.editor.providers.generic.label'),
label: t('model.editor.providers.generic.label'),
defaultUrls: {},
description: t('model.editor.providers.generic.description'),
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr']
},
])
@@ -536,15 +538,22 @@ const getModelNamePlaceholder = () => {
? t('model.editor.modelNamePlaceholder.localVllm')
: t('model.editor.modelNamePlaceholder.remoteVllm')
}
if (props.modelType === 'asr') {
return t('model.editor.modelNamePlaceholder.remoteAsr')
}
return formData.value.source === 'local'
? t('model.editor.modelNamePlaceholder.local')
: t('model.editor.modelNamePlaceholder.remote')
}
const getBaseUrlPlaceholder = () => {
return props.modelType === 'vllm'
? t('model.editor.baseUrlPlaceholderVllm')
: t('model.editor.baseUrlPlaceholder')
if (props.modelType === 'vllm') {
return t('model.editor.baseUrlPlaceholderVllm')
}
if (props.modelType === 'asr') {
return t('model.editor.baseUrlPlaceholderAsr')
}
return t('model.editor.baseUrlPlaceholder')
}
// 检查Ollama服务状态
@@ -840,7 +849,17 @@ const checkRemoteAPI = async () => {
apiKey: formData.value.apiKey || ''
})
break
case 'asr':
// ASR 模型(语音识别)
// ASR 使用 checkRemoteModel 进行基础连接测试
result = await checkRemoteModel({
modelName: formData.value.modelName,
baseUrl: formData.value.baseUrl,
apiKey: formData.value.apiKey || ''
})
break
default:
MessagePlugin.error(t('model.editor.unsupportedModelType'))
return
+1 -1
View File
@@ -46,7 +46,7 @@ import { MessagePlugin } from 'tdesign-vue-next'
import { useI18n } from 'vue-i18n'
interface Props {
modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM'
modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR'
selectedModelId?: string
disabled?: boolean
placeholder?: string
+19 -1
View File
@@ -1471,6 +1471,7 @@ export default {
graph: 'Knowledge Graph',
multimodal: 'Multimodal',
datasource: 'Data Sources',
asr: 'Speech Recognition',
share: 'Sharing'
},
basic: {
@@ -1676,6 +1677,15 @@ export default {
title: 'Multimodal Configuration',
description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images',
},
asr: {
title: 'Speech Recognition Configuration',
description: 'Configure speech recognition to upload audio files (mp3, wav, m4a, flac, ogg) and automatically transcribe to text',
label: 'Enable Speech Recognition',
desc: 'When enabled, audio files can be uploaded to the knowledge base and automatically transcribed to text',
modelLabel: 'ASR Model',
modelDescription: 'Speech-to-text model for audio transcription (e.g. OpenAI Whisper)',
modelPlaceholder: 'Select an ASR model',
},
advanced: {
title: 'Advanced Settings',
description: 'Configure question generation and other advanced features',
@@ -2076,17 +2086,20 @@ export default {
embedding: 'Configure embedding models for text vectorization',
rerank: 'Configure models for result re-ranking',
vllm: 'Configure vision-language models for multimodal understanding',
asr: 'Configure speech-to-text models for audio transcription',
default: 'Configure model information'
},
modelNamePlaceholder: {
local: 'e.g. llama2:latest',
remote: 'e.g. gpt-4, claude-3-opus',
localVllm: 'e.g. llava:latest',
remoteVllm: 'e.g. gpt-4-vision-preview'
remoteVllm: 'e.g. gpt-4-vision-preview',
remoteAsr: 'e.g. whisper-1'
},
baseUrlLabel: 'Base URL',
baseUrlPlaceholder: 'e.g. https://api.openai.com/v1',
baseUrlPlaceholderVllm: 'e.g. http://localhost:11434/v1',
baseUrlPlaceholderAsr: 'e.g. https://api.openai.com/v1',
apiKeyOptional: 'API Key (optional)',
apiKeyPlaceholder: 'Enter API Key',
connectionTest: 'Connection Test',
@@ -2582,6 +2595,11 @@ export default {
desc: 'Configure vision-language models for multimodal understanding',
empty: 'No VLLM models'
},
asr: {
title: 'ASR Speech Models',
desc: 'Configure speech-to-text models for audio transcription (e.g. OpenAI Whisper)',
empty: 'No ASR models'
},
toasts: {
nameRequired: 'Model name cannot be empty',
nameTooLong: 'Model name cannot exceed 100 characters',
+19 -1
View File
@@ -1534,6 +1534,7 @@ export default {
embedding: "텍스트 벡터화용 임베딩 모델 설정",
rerank: "결과 재정렬용 모델 설정",
vllm: "시각 이해 및 멀티모달용 비전 언어 모델 설정",
asr: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정",
default: "모델 정보 설정",
},
modelNamePlaceholder: {
@@ -1541,10 +1542,12 @@ export default {
remote: "예: gpt-4, claude-3-opus",
localVllm: "예: llava:latest",
remoteVllm: "예: gpt-4-vision-preview",
remoteAsr: "예: whisper-1",
},
baseUrlLabel: "Base URL",
baseUrlPlaceholder: "예: https://api.openai.com/v1",
baseUrlPlaceholderVllm: "예: http://localhost:11434/v1",
baseUrlPlaceholderAsr: "예: https://api.openai.com/v1",
apiKeyOptional: "API 키 (선택)",
apiKeyPlaceholder: "API 키 입력",
connectionTest: "연결 테스트",
@@ -1955,6 +1958,7 @@ export default {
faq: "FAQ 설정",
graph: "지식 그래프",
multimodal: "멀티모달",
asr: "음성 인식",
share: "공유관리",
storage: "스토리지 엔진",
datasource: "데이터 소스",
@@ -2170,7 +2174,16 @@ export default {
},
multimodal: {
title: "멀티모달 설정",
description: "멀티모달 콘텐츠 이해 기능을 설정합니다. 활성화하면 이미지 등 텍스트가 아닌 콘텐츠의 파싱 및 검색을 지원합니다.",
description: "멀티모달 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다",
},
asr: {
title: "음성 인식 설정",
description: "음성 인식 기능을 설정하여 오디오 파일(mp3, wav, m4a, flac, ogg)을 업로드하고 자동으로 텍스트로 전사합니다",
label: "음성 인식 활성화",
desc: "활성화하면 오디오 파일을 지식베이스에 업로드하고 자동으로 텍스트로 전사할 수 있습니다",
modelLabel: "ASR 모델",
modelDescription: "오디오 전사를 위한 음성 인식 모델 (예: OpenAI Whisper)",
modelPlaceholder: "ASR 모델을 선택해주세요",
},
advanced: {
title: "고급 설정",
@@ -2615,6 +2628,11 @@ export default {
desc: "시각 이해 및 멀티모달용 비전 언어 모델 설정",
empty: "VLLM 비전 모델 없음",
},
asr: {
title: "ASR 음성 모델",
desc: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정 (예: OpenAI Whisper)",
empty: "ASR 음성 모델 없음",
},
toasts: {
nameRequired: "모델 이름은 비워둘 수 없습니다",
nameTooLong: "모델 이름은 100자를 초과할 수 없습니다",
+20 -2
View File
@@ -1381,17 +1381,20 @@ export default {
embedding: 'Настройте модель встраивания для текстовой векторизации',
rerank: 'Настройте модель для повторного ранжирования результатов',
vllm: 'Настройте визуально-языковую модель для мультимодального понимания',
asr: 'Настройте модель распознавания речи для транскрибации аудио',
default: 'Настройте информацию о модели'
},
modelNamePlaceholder: {
local: 'например: llama2:latest',
remote: 'например: gpt-4, claude-3-opus',
localVllm: 'например: llava:latest',
remoteVllm: 'например: gpt-4-vision-preview'
remoteVllm: 'например: gpt-4-vision-preview',
remoteAsr: 'например: whisper-1'
},
baseUrlLabel: 'Base URL',
baseUrlPlaceholder: 'например: https://api.openai.com/v1',
baseUrlPlaceholderVllm: 'например: http://localhost:11434/v1',
baseUrlPlaceholderAsr: 'например: https://api.openai.com/v1',
apiKeyOptional: 'API Key (опционально)',
apiKeyPlaceholder: 'Введите API Key',
connectionTest: 'Проверка соединения',
@@ -1641,6 +1644,7 @@ export default {
faq: 'FAQ настройки',
graph: 'Граф знаний',
multimodal: 'Мультимодальность',
asr: 'Распознавание речи',
storage: 'Storage Engine',
datasource: 'Источники данных',
share: 'Sharing'
@@ -1836,7 +1840,16 @@ export default {
},
multimodal: {
title: 'Мультимодальная конфигурация',
description: 'Настройка понимания мультимодального контента для парсинга и поиска нетекстового содержимого, такого как изображения',
description: 'Настройте мультимодальное понимание контента для парсинга и поиска нетекстового контента, например изображений',
},
asr: {
title: 'Конфигурация распознавания речи',
description: 'Настройте распознавание речи для загрузки аудиофайлов (mp3, wav, m4a, flac, ogg) и автоматической транскрибации в текст',
label: 'Включить распознавание речи',
desc: 'При включении аудиофайлы могут быть загружены в базу знаний и автоматически транскрибированы в текст',
modelLabel: 'Модель ASR',
modelDescription: 'Модель распознавания речи для транскрибации аудио (например, OpenAI Whisper)',
modelPlaceholder: 'Выберите модель ASR',
},
advanced: {
title: 'Расширенные настройки',
@@ -2380,6 +2393,11 @@ export default {
desc: 'Визуально-языковые модели для мультимодального понимания',
empty: 'Нет VLLM моделей'
},
asr: {
title: 'ASR модели речи',
desc: 'Модели распознавания речи для транскрибации аудио (например, OpenAI Whisper)',
empty: 'Нет ASR моделей'
},
toasts: {
nameRequired: 'Название модели не может быть пустым',
nameTooLong: 'Название модели не может превышать 100 символов',
+18
View File
@@ -1528,6 +1528,7 @@ export default {
embedding: "配置用于文本向量化的嵌入模型",
rerank: "配置用于结果重排序的模型",
vllm: "配置用于视觉理解和多模态的视觉语言模型",
asr: "配置用于语音识别和音频转录的语音转文本模型",
default: "配置模型信息",
},
modelNamePlaceholder: {
@@ -1535,10 +1536,12 @@ export default {
remote: "例如:gpt-4, claude-3-opus",
localVllm: "例如:llava:latest",
remoteVllm: "例如:gpt-4-vision-preview",
remoteAsr: "例如:whisper-1",
},
baseUrlLabel: "Base URL",
baseUrlPlaceholder: "例如:https://api.openai.com/v1",
baseUrlPlaceholderVllm: "例如:http://localhost:11434/v1",
baseUrlPlaceholderAsr: "例如:https://api.openai.com/v1",
apiKeyOptional: "API Key(可选)",
apiKeyPlaceholder: "输入 API Key",
connectionTest: "连接测试",
@@ -1943,6 +1946,7 @@ export default {
graph: "知识图谱",
multimodal: "多模态",
datasource: "数据源",
asr: "语音识别",
share: "共享管理",
},
basic: {
@@ -2145,6 +2149,15 @@ export default {
title: "多模态配置",
description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索",
},
asr: {
title: "语音识别配置",
description: "配置语音识别功能,启用后支持上传音频文件(mp3、wav、m4a、flac、ogg)并自动转录为文本",
label: "启用语音识别",
desc: "启用后可以上传音频文件到知识库,系统将自动转录为文本进行处理",
modelLabel: "ASR 模型",
modelDescription: "用于语音转文本的语音识别模型(如 OpenAI Whisper)",
modelPlaceholder: "请选择 ASR 模型",
},
advanced: {
title: "高级设置",
description: "配置问题生成等高级功能",
@@ -2580,6 +2593,11 @@ export default {
desc: "配置用于视觉理解和多模态的视觉语言模型",
empty: "暂无 VLLM 视觉模型",
},
asr: {
title: "ASR 语音模型",
desc: "配置用于语音识别和音频转录的语音转文本模型(如 OpenAI Whisper)",
empty: "暂无 ASR 语音模型",
},
toasts: {
nameRequired: "模型名称不能为空",
nameTooLong: "模型名称不能超过100个字符",
@@ -1514,7 +1514,7 @@ async function createNewSession(value: string): Promise<void> {
ref="uploadInputRef"
type="file"
class="document-upload-input"
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'"
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'"
multiple
@change="handleDocumentUpload"
/>
@@ -202,6 +202,49 @@
</div>
</div>
<!-- ASR 语音识别设置 -->
<div v-if="!isFAQ" v-show="currentSection === 'asr'" class="section">
<div v-if="formData" class="kb-multimodal-settings">
<div class="section-header">
<h2>{{ $t('knowledgeEditor.asr.title') }}</h2>
<p class="section-description">{{ $t('knowledgeEditor.asr.description') }}</p>
</div>
<div class="settings-group">
<!-- ASR 开关 -->
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.asr.label') }}</label>
<p class="desc">{{ $t('knowledgeEditor.asr.desc') }}</p>
</div>
<div class="setting-control">
<t-switch
v-model="formData.asrConfig.enabled"
size="medium"
/>
</div>
</div>
<!-- ASR 模型选择 -->
<div v-if="formData.asrConfig.enabled" class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.asr.modelLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.asr.modelDescription') }}</p>
</div>
<div class="setting-control">
<ModelSelector
model-type="ASR"
:selected-model-id="formData.asrConfig.modelId"
:all-models="allModels"
@update:selected-model-id="(val: string) => { if (formData) formData.asrConfig.modelId = val }"
:placeholder="$t('knowledgeEditor.asr.modelPlaceholder')"
/>
</div>
</div>
</div>
</div>
</div>
<!-- 高级设置 -->
<div v-if="!isFAQ" v-show="currentSection === 'advanced'" class="section">
<KBAdvancedSettings
@@ -298,6 +341,7 @@ const navItems = computed(() => {
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
{ key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') },
{ key: 'asr', icon: 'sound', label: t('knowledgeEditor.sidebar.asr') },
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
)
if (props.mode === 'edit' && props.kbId) {
@@ -363,6 +407,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
enabled: false,
vllmModelId: ''
},
asrConfig: {
enabled: false,
modelId: '',
language: ''
},
nodeExtractConfig: {
enabled: false,
text: '',
@@ -443,6 +492,11 @@ const loadKBData = async () => {
enabled: !!kb.vlm_config?.enabled,
vllmModelId: kb.vlm_config?.model_id || ''
},
asrConfig: {
enabled: !!kb.asr_config?.enabled,
modelId: kb.asr_config?.model_id || '',
language: kb.asr_config?.language || ''
},
nodeExtractConfig: {
enabled: kb.extract_config?.enabled || false,
text: kb.extract_config?.text || '',
@@ -593,6 +647,15 @@ const buildSubmitData = () => {
: ''
}
// 添加ASR语音识别配置
data.asr_config = {
enabled: formData.value.asrConfig?.enabled || false,
model_id: formData.value.asrConfig?.enabled
? (formData.value.asrConfig?.modelId || '')
: '',
language: formData.value.asrConfig?.language || ''
}
// 存储引擎:仅传 provider,参数从全局设置读取
// Write to storage_provider_config (authoritative) + storage_config (legacy dual-write)
data.storage_provider_config = {
@@ -705,6 +768,7 @@ const doSubmit = async () => {
llmModelId: data.summary_model_id,
embeddingModelId: data.embedding_model_id,
vlm_config: data.vlm_config,
asr_config: data.asr_config,
documentSplitting: {
chunkSize: data.chunking_config.chunk_size,
chunkOverlap: data.chunking_config.chunk_overlap,
+64 -9
View File
@@ -221,6 +221,54 @@
</div>
</div>
<!-- STT 语音模型 -->
<div class="model-category-section" data-model-type="asr">
<div class="category-header">
<div class="header-info">
<h3>{{ $t('modelSettings.asr.title') }}</h3>
<p>{{ $t('modelSettings.asr.desc') }}</p>
</div>
<t-button size="small" theme="primary" @click="openAddDialog('asr')" class="add-model-btn">
<template #icon>
<t-icon name="add" class="add-icon" />
</template>
{{ $t('modelSettings.actions.addModel') }}
</t-button>
</div>
<div v-if="asrModels.length > 0" class="model-list-container">
<div v-for="model in asrModels" :key="model.id" class="model-card" :class="{ 'builtin-model': model.isBuiltin }">
<div class="model-info">
<div class="model-name">
{{ model.name }}
<t-tag v-if="model.isBuiltin" theme="primary" size="small">{{ $t('modelSettings.builtinTag') }}</t-tag>
</div>
<div class="model-meta">
<span class="source-tag">{{ model.source === 'local' ? 'Ollama' : $t('modelSettings.source.openaiCompatible') }}</span>
</div>
</div>
<div class="model-actions">
<t-dropdown
:options="getModelOptions('asr', model)"
@click="(data: any) => handleMenuAction(data, 'asr', model)"
placement="bottom-right"
attach="body"
>
<t-button variant="text" shape="square" size="small" class="more-btn">
<t-icon name="more" />
</t-button>
</t-dropdown>
</div>
</div>
</div>
<div v-else class="empty-state">
<p class="empty-text">{{ $t('modelSettings.asr.empty') }}</p>
<t-button theme="default" variant="outline" size="small" @click="openAddDialog('asr')">
{{ $t('modelSettings.actions.addModel') }}
</t-button>
</div>
</div>
<!-- 模型编辑器弹窗 -->
<ModelEditorDialog
v-model:visible="showDialog"
@@ -242,7 +290,7 @@ import { listModels, createModel, updateModel as updateModelAPI, deleteModel as
const { t } = useI18n()
const showDialog = ref(false)
const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm'>('chat')
const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'>('chat')
const editingModel = ref<any>(null)
const loading = ref(true)
@@ -268,12 +316,18 @@ const rerankModels = computed(() =>
.map(convertToLegacyFormat)
)
const vllmModels = computed(() =>
const vllmModels = computed(() =>
allModels.value
.filter(m => m.type === 'VLLM')
.map(convertToLegacyFormat)
)
const asrModels = computed(() =>
allModels.value
.filter(m => m.type === 'ASR')
.map(convertToLegacyFormat)
)
// 将后端模型格式转换为旧的前端格式
function convertToLegacyFormat(model: ModelConfig) {
return {
@@ -306,14 +360,14 @@ const loadModels = async () => {
}
// 打开添加对话框
const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm') => {
const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr') => {
currentModelType.value = type
editingModel.value = null
showDialog.value = true
}
// 编辑模型
const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
// 内置模型不能编辑
if (model.isBuiltin) {
MessagePlugin.warning(t('modelSettings.toasts.builtinCannotEdit'))
@@ -405,7 +459,7 @@ const handleModelSave = async (modelData: any) => {
}
// 删除模型
const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', modelId: string) => {
const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', modelId: string) => {
// 检查是否是内置模型
const model = allModels.value.find(m => m.id === modelId)
if (model?.is_builtin) {
@@ -425,7 +479,7 @@ const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model
}
// 获取模型操作菜单选项
const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
const options: any[] = []
// 内置模型不能编辑和删除
@@ -450,7 +504,7 @@ const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model:
}
// 处理菜单操作
const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
const value = data.value
if (value.indexOf('edit-') === 0) {
@@ -464,12 +518,13 @@ const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' |
}
// 获取后端模型类型
function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' {
function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR' {
const typeMap = {
chat: 'KnowledgeQA' as const,
embedding: 'Embedding' as const,
rerank: 'Rerank' as const,
vllm: 'VLLM' as const
vllm: 'VLLM' as const,
asr: 'ASR' as const
}
return typeMap[type]
}
+8
View File
@@ -209,6 +209,14 @@ func getContentTypeByExt(ext string) string {
return "image/svg+xml"
case ".mp3":
return "audio/mpeg"
case ".wav":
return "audio/wav"
case ".m4a":
return "audio/mp4"
case ".flac":
return "audio/flac"
case ".ogg":
return "audio/ogg"
case ".mp4":
return "video/mp4"
default:
+85 -1
View File
@@ -259,6 +259,15 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
logger.Info(ctx, "Image multimodal configuration validation passed")
}
// 检查音频ASR配置完整性 - 只在音频文件时校验
if IsAudioType(getFileType(fileName)) {
if !kb.ASRConfig.IsASREnabled() {
logger.Error(ctx, "ASR model is not configured")
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
}
logger.Info(ctx, "Audio ASR configuration validation passed")
}
// Validate file type
logger.Infof(ctx, "Checking file type: %s", fileName)
if !isValidFileType(fileName) {
@@ -596,6 +605,11 @@ var allowedFileURLExtensions = map[string]bool{
"pdf": true,
"docx": true,
"doc": true,
"mp3": true,
"wav": true,
"m4a": true,
"flac": true,
"ogg": true,
}
// maxFileURLSize is the maximum allowed file size for file URL import (10MB)
@@ -2987,7 +3001,8 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
// isValidFileType checks if a file type is supported
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json":
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
"mp3", "wav", "m4a", "flac", "ogg":
return true
default:
return false
@@ -7399,6 +7414,16 @@ func IsImageType(fileType string) bool {
}
}
// IsAudioType checks if a file type is an audio format
func IsAudioType(fileType string) bool {
switch strings.ToLower(fileType) {
case "mp3", "wav", "m4a", "flac", "ogg":
return true
default:
return false
}
}
// downloadFileFromURL downloads a remote file to a temp file and returns its binary content.
// payloadFileName and payloadFileType are in/out pointers: if they point to an empty string,
// the function resolves the value from Content-Disposition / URL path and writes it back.
@@ -7641,6 +7666,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
// 检查音频ASR配置(仅对文件导入)
if payload.FilePath != "" && IsAudioType(payload.FileType) && !kb.ASRConfig.IsASREnabled() {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument audio without ASR model configured")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "上传音频文件需要设置ASR语音识别模型"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
var convertResult *types.ReadResult
var chunks []types.ParsedChunk
@@ -7764,6 +7800,54 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
}
// Step 1.5: ASR transcription for audio files
if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 {
if !kb.ASRConfig.IsASREnabled() {
logger.Error(ctx, "Audio file detected but ASR is not configured")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "ASR model is not configured for audio transcription"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes",
knowledge.ID, len(convertResult.AudioData))
asrModel, err := s.modelService.GetASRModel(ctx, kb.ASRConfig.ModelID)
if err != nil {
logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to get ASR model: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
transcribedText, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
if err != nil {
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("audio transcription failed: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("audio transcription failed: %w", err)
}
if transcribedText == "" {
logger.Warn(ctx, "[ASR] Transcription returned empty text")
transcribedText = "[No speech detected in audio file]"
}
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(transcribedText))
// Replace the audio placeholder with the transcribed text
convertResult.MarkdownContent = transcribedText
convertResult.IsAudio = false
convertResult.AudioData = nil
}
// Step 2: Store images and update markdown references
var storedImages []docparser.StoredImage
+48
View File
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
@@ -458,3 +459,50 @@ func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM
// Note: default model selection logic has been removed; models no longer
// maintain a per-type default flag at the service layer.
// GetASRModel retrieves and initializes an automatic speech recognition model instance.
func (s *modelService) GetASRModel(ctx context.Context, modelId string) (asr.ASR, error) {
if modelId == "" {
return nil, errors.New("model ID cannot be empty")
}
tenantID := types.MustTenantIDFromContext(ctx)
model, err := s.repo.GetByID(ctx, tenantID, modelId)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": modelId,
"tenant_id": tenantID,
})
return nil, err
}
if model == nil {
return nil, ErrModelNotFound
}
logger.Infof(ctx, "Getting ASR model: %s, source: %s", model.Name, model.Source)
ifType := model.Parameters.InterfaceType
if ifType == "" {
ifType = "openai"
}
sttModel, err := asr.NewASR(&asr.Config{
ModelID: model.ID,
APIKey: model.Parameters.APIKey,
BaseURL: model.Parameters.BaseURL,
ModelName: model.Name,
Source: model.Source,
InterfaceType: ifType,
})
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": model.ID,
"model_name": model.Name,
})
return nil, err
}
return sttModel, nil
}
@@ -24,10 +24,17 @@ var imageFormats = map[string]bool{
"bmp": true, "tiff": true, "webp": true,
}
var audioFormats = map[string]bool{
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
}
func init() {
for k := range imageFormats {
simpleFormats[k] = true
}
for k := range audioFormats {
simpleFormats[k] = true
}
}
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
@@ -65,6 +72,8 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
return &types.ReadResult{MarkdownContent: md}, nil
case imageFormats[ft]:
return imageToResult(req.FileName, req.FileContent), nil
case audioFormats[ft]:
return audioToResult(req.FileName, req.FileContent), nil
default:
return nil, fmt.Errorf("unsupported simple format: %s", ft)
}
@@ -99,6 +108,28 @@ func IsImageFormat(fileType string) bool {
return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// IsAudioFormat returns true if the file type is a recognized audio format.
func IsAudioFormat(fileType string) bool {
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// audioToResult wraps a standalone audio file. The actual transcription is
// handled by the ASR model in the knowledge service pipeline. Here we just
// return a placeholder markdown with the raw bytes preserved for upstream
// processing.
func audioToResult(fileName string, data []byte) *types.ReadResult {
if fileName == "" {
fileName = "audio.mp3"
}
// Return a placeholder; the knowledge service will replace this with
// the ASR transcription result.
return &types.ReadResult{
MarkdownContent: fmt.Sprintf("[Audio file: %s]", fileName),
IsAudio: true,
AudioData: data,
}
}
// ensureOriginalImageRef checks whether the input file is an image and, if the
// returned markdown does not already contain a markdown image reference for it,
// prepends one and appends the raw bytes to imageRefs. This guarantees that
@@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string {
return "Simple format & image parsing (no external service required)"
}
func (e *simpleEngine) FileTypes(_ bool) []string {
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"}
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
}
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
return true, ""
+44
View File
@@ -0,0 +1,44 @@
package asr
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
)
// ASR defines the interface for Automatic Speech Recognition model operations.
type ASR interface {
// Transcribe sends audio bytes to the ASR model and returns the transcribed text.
Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error)
GetModelName() string
GetModelID() string
}
// Config holds the configuration needed to create an ASR instance.
type Config struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
InterfaceType string // "openai" (default)
Language string // optional: specify language for transcription
}
// NewASR creates an ASR instance based on the provided configuration.
func NewASR(config *Config) (ASR, error) {
ifType := strings.ToLower(config.InterfaceType)
if ifType == "" {
ifType = "openai"
}
switch ifType {
case "openai":
return NewOpenAIASR(config)
default:
return nil, fmt.Errorf("unsupported ASR interface type: %s", ifType)
}
}
+119
View File
@@ -0,0 +1,119 @@
package asr
import (
"bytes"
"context"
"fmt"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
openai "github.com/sashabaranov/go-openai"
)
const (
asrDefaultTimeout = 300 * time.Second // audio transcription can be slow
)
// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.
type OpenAIASR struct {
modelName string
modelID string
client *openai.Client
baseURL string
language string
}
// NewOpenAIASR creates an OpenAI-compatible ASR instance.
func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
apiCfg := openai.DefaultConfig(config.APIKey)
if config.BaseURL != "" {
apiCfg.BaseURL = config.BaseURL
}
apiCfg.HTTPClient = &http.Client{Timeout: asrDefaultTimeout}
return &OpenAIASR{
modelName: config.ModelName,
modelID: config.ModelID,
client: openai.NewClientWithConfig(apiCfg),
baseURL: config.BaseURL,
language: config.Language,
}, nil
}
// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error) {
if len(audioBytes) == 0 {
return "", fmt.Errorf("audio bytes are empty")
}
// Ensure fileName has an extension for proper MIME type detection
if fileName == "" {
fileName = "audio.mp3"
}
logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",
s.modelName, s.baseURL, len(audioBytes), fileName)
req := openai.AudioRequest{
Model: s.modelName,
FilePath: fileName,
Reader: bytes.NewReader(audioBytes),
Format: openai.AudioResponseFormatJSON,
}
if s.language != "" {
req.Language = s.language
}
resp, err := s.client.CreateTranscription(ctx, req)
if err != nil {
return "", fmt.Errorf("ASR transcription request failed: %w", err)
}
text := strings.TrimSpace(resp.Text)
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
return text, nil
}
func (s *OpenAIASR) GetModelName() string { return s.modelName }
func (s *OpenAIASR) GetModelID() string { return s.modelID }
// DetectAudioFormat returns a file extension hint for the given audio bytes.
func DetectAudioFormat(data []byte, fileName string) string {
if fileName != "" {
ext := strings.ToLower(filepath.Ext(fileName))
if ext != "" {
return ext
}
}
// Try to detect from magic bytes
if len(data) >= 4 {
// MP3: starts with ID3 or 0xFF 0xFB
if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {
return ".mp3"
}
if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
return ".mp3"
}
// FLAC: starts with "fLaC"
if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {
return ".flac"
}
// OGG: starts with "OggS"
if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {
return ".ogg"
}
// WAV: starts with "RIFF"
if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {
return ".wav"
}
}
// M4A: check for ftyp box
if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {
return ".m4a"
}
return ".mp3" // default
}
+2
View File
@@ -32,12 +32,14 @@ func (p *AliyunProvider) Info() ProviderInfo {
types.ModelTypeEmbedding: AliyunChatBaseURL,
types.ModelTypeRerank: AliyunRerankBaseURL,
types.ModelTypeVLLM: AliyunChatBaseURL,
types.ModelTypeASR: AliyunChatBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: true,
}
+1
View File
@@ -25,6 +25,7 @@ func (p *GenericProvider) Info() ProviderInfo {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: false, // 可能需要也可能不需要
}
+2
View File
@@ -31,12 +31,14 @@ func (p *GPUStackProvider) Info() ProviderInfo {
types.ModelTypeEmbedding: GPUStackBaseURL,
types.ModelTypeRerank: GPUStackRerankBaseURL,
types.ModelTypeVLLM: GPUStackBaseURL,
types.ModelTypeASR: GPUStackBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: true, // GPUStack 需要 API Key
}
+2
View File
@@ -28,12 +28,14 @@ func (p *OpenAIProvider) Info() ProviderInfo {
types.ModelTypeEmbedding: OpenAIBaseURL,
types.ModelTypeRerank: OpenAIBaseURL,
types.ModelTypeVLLM: OpenAIBaseURL,
types.ModelTypeASR: OpenAIBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: true,
}
+2
View File
@@ -28,12 +28,14 @@ func (p *SiliconFlowProvider) Info() ProviderInfo {
types.ModelTypeEmbedding: SiliconFlowBaseURL,
types.ModelTypeRerank: SiliconFlowBaseURL,
types.ModelTypeVLLM: SiliconFlowBaseURL,
types.ModelTypeASR: SiliconFlowBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: true,
}
+2
View File
@@ -20,6 +20,8 @@ type ReadResult struct {
ImageDirPath string
Metadata map[string]string
Error string
IsAudio bool // true when the result contains raw audio data needing ASR transcription
AudioData []byte // raw audio bytes for ASR processing
}
// ImageRef represents an image reference extracted from the document.
+3
View File
@@ -6,6 +6,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -32,6 +33,8 @@ type ModelService interface {
GetChatModel(ctx context.Context, modelId string) (chat.Chat, error)
// GetVLMModel gets a vision language model
GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error)
// GetASRModel gets an automatic speech recognition model
GetASRModel(ctx context.Context, modelId string) (asr.ASR, error)
}
// ModelRepository defines the model repository interface
+31
View File
@@ -60,6 +60,8 @@ type KnowledgeBase struct {
SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"`
// VLM config
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
// ASR config (Automatic Speech Recognition)
ASRConfig ASRConfig `yaml:"asr_config" json:"asr_config" gorm:"type:json"`
// Storage provider config (new): only stores provider selection; credentials from tenant StorageEngineConfig
StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"`
// Deprecated: legacy COS config column. Kept for backward compatibility with old data.
@@ -386,6 +388,35 @@ func (c *VLMConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c)
}
// ASRConfig represents the ASR (Automatic Speech Recognition) configuration
type ASRConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
ModelID string `yaml:"model_id" json:"model_id"`
Language string `yaml:"language" json:"language"` // optional: language hint for transcription
}
// IsASREnabled checks if ASR is enabled with a valid model
func (c ASRConfig) IsASREnabled() bool {
return c.Enabled && c.ModelID != ""
}
// Value implements the driver.Valuer interface, used to convert ASRConfig to database value
func (c ASRConfig) Value() (driver.Value, error) {
return json.Marshal(c)
}
// Scan implements the sql.Scanner interface, used to convert database value to ASRConfig
func (c *ASRConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(b, c)
}
// ExtractConfig represents the extract configuration for a knowledge base
type ExtractConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
+1
View File
@@ -18,6 +18,7 @@ const (
ModelTypeRerank ModelType = "Rerank" // Rerank model
ModelTypeKnowledgeQA ModelType = "KnowledgeQA" // KnowledgeQA model
ModelTypeVLLM ModelType = "VLLM" // VLLM model
ModelTypeASR ModelType = "ASR" // ASR (Automatic Speech Recognition) model
)
// ModelStatus represents the status of the model
@@ -0,0 +1,7 @@
-- Migration: 000025_add_asr_config (rollback)
-- Description: Remove asr_config column from knowledge_bases.
DO $$ BEGIN RAISE NOTICE '[Migration 000025] Removing asr_config column from knowledge_bases'; END $$;
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS asr_config;
DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column removed successfully'; END $$;
@@ -0,0 +1,10 @@
-- Migration: 000025_add_asr_config
-- Description: Add asr_config column to knowledge_bases for ASR (Automatic Speech Recognition) configuration.
-- ASR config stores the model ID and language settings for audio transcription.
DO $$ BEGIN RAISE NOTICE '[Migration 000025] Adding asr_config column to knowledge_bases'; END $$;
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS asr_config JSONB;
COMMENT ON COLUMN knowledge_bases.asr_config IS 'ASR (Automatic Speech Recognition) configuration: {"enabled": bool, "model_id": string, "language": string}';
DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column added successfully'; END $$;