feat(asr): integrate Automatic Speech Recognition (ASR) support
- Added ASR configuration options to the knowledge base and model settings. - Implemented ASR model selection and transcription capabilities in the knowledge processing pipeline. - Enhanced file upload support to include audio formats for ASR processing. - Updated localization files to include ASR-related labels and descriptions. - Refactored model handling to accommodate ASR as a new model type.
This commit is contained in:
@@ -81,6 +81,11 @@ export interface KBModelConfigRequest {
|
||||
enabled: boolean
|
||||
model_id?: string
|
||||
}
|
||||
asr_config?: {
|
||||
enabled: boolean
|
||||
model_id?: string
|
||||
language?: string
|
||||
}
|
||||
documentSplitting: {
|
||||
chunkSize: number
|
||||
chunkOverlap: number
|
||||
|
||||
@@ -21,6 +21,11 @@ export function createKnowledgeBase(data: {
|
||||
};
|
||||
storage_provider_config?: { provider: string };
|
||||
storage_config?: any; // legacy, kept for backward compat (dual-write)
|
||||
asr_config?: {
|
||||
enabled: boolean;
|
||||
model_id?: string;
|
||||
language?: string;
|
||||
};
|
||||
extract_config?: any;
|
||||
faq_config?: { index_mode: string; question_index_mode?: string };
|
||||
}) {
|
||||
|
||||
@@ -8,7 +8,7 @@ export interface ModelConfig {
|
||||
id?: string;
|
||||
tenant_id?: number;
|
||||
name: string;
|
||||
type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM';
|
||||
type: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR';
|
||||
source: 'local' | 'remote';
|
||||
description?: string;
|
||||
parameters: {
|
||||
|
||||
@@ -280,7 +280,7 @@ interface ModelFormData {
|
||||
|
||||
interface Props {
|
||||
visible: boolean
|
||||
modelType: 'chat' | 'embedding' | 'rerank' | 'vllm'
|
||||
modelType: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'
|
||||
modelData?: ModelFormData | null
|
||||
}
|
||||
|
||||
@@ -303,29 +303,31 @@ const loadingProviders = ref(false)
|
||||
|
||||
// 硬编码的后备 Provider 配置 (当 API 不可用时使用)
|
||||
const fallbackProviderOptions = computed(() => [
|
||||
{
|
||||
value: 'openai',
|
||||
label: t('model.editor.providers.openai.label'),
|
||||
{
|
||||
value: 'openai',
|
||||
label: t('model.editor.providers.openai.label'),
|
||||
defaultUrls: {
|
||||
chat: 'https://api.openai.com/v1',
|
||||
embedding: 'https://api.openai.com/v1',
|
||||
rerank: 'https://api.openai.com/v1',
|
||||
vllm: 'https://api.openai.com/v1'
|
||||
vllm: 'https://api.openai.com/v1',
|
||||
asr: 'https://api.openai.com/v1'
|
||||
},
|
||||
description: t('model.editor.providers.openai.description'),
|
||||
modelTypes: ['chat', 'embedding', 'vllm']
|
||||
modelTypes: ['chat', 'embedding', 'vllm', 'asr']
|
||||
},
|
||||
{
|
||||
value: 'aliyun',
|
||||
label: t('model.editor.providers.aliyun.label'),
|
||||
{
|
||||
value: 'aliyun',
|
||||
label: t('model.editor.providers.aliyun.label'),
|
||||
defaultUrls: {
|
||||
chat: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
embedding: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
rerank: 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank',
|
||||
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
|
||||
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
asr: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
|
||||
},
|
||||
description: t('model.editor.providers.aliyun.description'),
|
||||
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
|
||||
modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr']
|
||||
},
|
||||
{
|
||||
value: 'zhipu',
|
||||
@@ -394,10 +396,10 @@ const fallbackProviderOptions = computed(() => [
|
||||
},
|
||||
{
|
||||
value: 'generic',
|
||||
label: t('model.editor.providers.generic.label'),
|
||||
label: t('model.editor.providers.generic.label'),
|
||||
defaultUrls: {},
|
||||
description: t('model.editor.providers.generic.description'),
|
||||
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
|
||||
modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr']
|
||||
},
|
||||
])
|
||||
|
||||
@@ -536,15 +538,22 @@ const getModelNamePlaceholder = () => {
|
||||
? t('model.editor.modelNamePlaceholder.localVllm')
|
||||
: t('model.editor.modelNamePlaceholder.remoteVllm')
|
||||
}
|
||||
if (props.modelType === 'asr') {
|
||||
return t('model.editor.modelNamePlaceholder.remoteAsr')
|
||||
}
|
||||
return formData.value.source === 'local'
|
||||
? t('model.editor.modelNamePlaceholder.local')
|
||||
: t('model.editor.modelNamePlaceholder.remote')
|
||||
}
|
||||
|
||||
const getBaseUrlPlaceholder = () => {
|
||||
return props.modelType === 'vllm'
|
||||
? t('model.editor.baseUrlPlaceholderVllm')
|
||||
: t('model.editor.baseUrlPlaceholder')
|
||||
if (props.modelType === 'vllm') {
|
||||
return t('model.editor.baseUrlPlaceholderVllm')
|
||||
}
|
||||
if (props.modelType === 'asr') {
|
||||
return t('model.editor.baseUrlPlaceholderAsr')
|
||||
}
|
||||
return t('model.editor.baseUrlPlaceholder')
|
||||
}
|
||||
|
||||
// 检查Ollama服务状态
|
||||
@@ -840,7 +849,17 @@ const checkRemoteAPI = async () => {
|
||||
apiKey: formData.value.apiKey || ''
|
||||
})
|
||||
break
|
||||
|
||||
|
||||
case 'asr':
|
||||
// ASR 模型(语音识别)
|
||||
// ASR 使用 checkRemoteModel 进行基础连接测试
|
||||
result = await checkRemoteModel({
|
||||
modelName: formData.value.modelName,
|
||||
baseUrl: formData.value.baseUrl,
|
||||
apiKey: formData.value.apiKey || ''
|
||||
})
|
||||
break
|
||||
|
||||
default:
|
||||
MessagePlugin.error(t('model.editor.unsupportedModelType'))
|
||||
return
|
||||
|
||||
@@ -46,7 +46,7 @@ import { MessagePlugin } from 'tdesign-vue-next'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
interface Props {
|
||||
modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM'
|
||||
modelType: 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR'
|
||||
selectedModelId?: string
|
||||
disabled?: boolean
|
||||
placeholder?: string
|
||||
|
||||
@@ -1471,6 +1471,7 @@ export default {
|
||||
graph: 'Knowledge Graph',
|
||||
multimodal: 'Multimodal',
|
||||
datasource: 'Data Sources',
|
||||
asr: 'Speech Recognition',
|
||||
share: 'Sharing'
|
||||
},
|
||||
basic: {
|
||||
@@ -1676,6 +1677,15 @@ export default {
|
||||
title: 'Multimodal Configuration',
|
||||
description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images',
|
||||
},
|
||||
asr: {
|
||||
title: 'Speech Recognition Configuration',
|
||||
description: 'Configure speech recognition to upload audio files (mp3, wav, m4a, flac, ogg) and automatically transcribe to text',
|
||||
label: 'Enable Speech Recognition',
|
||||
desc: 'When enabled, audio files can be uploaded to the knowledge base and automatically transcribed to text',
|
||||
modelLabel: 'ASR Model',
|
||||
modelDescription: 'Speech-to-text model for audio transcription (e.g. OpenAI Whisper)',
|
||||
modelPlaceholder: 'Select an ASR model',
|
||||
},
|
||||
advanced: {
|
||||
title: 'Advanced Settings',
|
||||
description: 'Configure question generation and other advanced features',
|
||||
@@ -2076,17 +2086,20 @@ export default {
|
||||
embedding: 'Configure embedding models for text vectorization',
|
||||
rerank: 'Configure models for result re-ranking',
|
||||
vllm: 'Configure vision-language models for multimodal understanding',
|
||||
asr: 'Configure speech-to-text models for audio transcription',
|
||||
default: 'Configure model information'
|
||||
},
|
||||
modelNamePlaceholder: {
|
||||
local: 'e.g. llama2:latest',
|
||||
remote: 'e.g. gpt-4, claude-3-opus',
|
||||
localVllm: 'e.g. llava:latest',
|
||||
remoteVllm: 'e.g. gpt-4-vision-preview'
|
||||
remoteVllm: 'e.g. gpt-4-vision-preview',
|
||||
remoteAsr: 'e.g. whisper-1'
|
||||
},
|
||||
baseUrlLabel: 'Base URL',
|
||||
baseUrlPlaceholder: 'e.g. https://api.openai.com/v1',
|
||||
baseUrlPlaceholderVllm: 'e.g. http://localhost:11434/v1',
|
||||
baseUrlPlaceholderAsr: 'e.g. https://api.openai.com/v1',
|
||||
apiKeyOptional: 'API Key (optional)',
|
||||
apiKeyPlaceholder: 'Enter API Key',
|
||||
connectionTest: 'Connection Test',
|
||||
@@ -2582,6 +2595,11 @@ export default {
|
||||
desc: 'Configure vision-language models for multimodal understanding',
|
||||
empty: 'No VLLM models'
|
||||
},
|
||||
asr: {
|
||||
title: 'ASR Speech Models',
|
||||
desc: 'Configure speech-to-text models for audio transcription (e.g. OpenAI Whisper)',
|
||||
empty: 'No ASR models'
|
||||
},
|
||||
toasts: {
|
||||
nameRequired: 'Model name cannot be empty',
|
||||
nameTooLong: 'Model name cannot exceed 100 characters',
|
||||
|
||||
@@ -1534,6 +1534,7 @@ export default {
|
||||
embedding: "텍스트 벡터화용 임베딩 모델 설정",
|
||||
rerank: "결과 재정렬용 모델 설정",
|
||||
vllm: "시각 이해 및 멀티모달용 비전 언어 모델 설정",
|
||||
asr: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정",
|
||||
default: "모델 정보 설정",
|
||||
},
|
||||
modelNamePlaceholder: {
|
||||
@@ -1541,10 +1542,12 @@ export default {
|
||||
remote: "예: gpt-4, claude-3-opus",
|
||||
localVllm: "예: llava:latest",
|
||||
remoteVllm: "예: gpt-4-vision-preview",
|
||||
remoteAsr: "예: whisper-1",
|
||||
},
|
||||
baseUrlLabel: "Base URL",
|
||||
baseUrlPlaceholder: "예: https://api.openai.com/v1",
|
||||
baseUrlPlaceholderVllm: "예: http://localhost:11434/v1",
|
||||
baseUrlPlaceholderAsr: "예: https://api.openai.com/v1",
|
||||
apiKeyOptional: "API 키 (선택)",
|
||||
apiKeyPlaceholder: "API 키 입력",
|
||||
connectionTest: "연결 테스트",
|
||||
@@ -1955,6 +1958,7 @@ export default {
|
||||
faq: "FAQ 설정",
|
||||
graph: "지식 그래프",
|
||||
multimodal: "멀티모달",
|
||||
asr: "음성 인식",
|
||||
share: "공유관리",
|
||||
storage: "스토리지 엔진",
|
||||
datasource: "데이터 소스",
|
||||
@@ -2170,7 +2174,16 @@ export default {
|
||||
},
|
||||
multimodal: {
|
||||
title: "멀티모달 설정",
|
||||
description: "멀티모달 콘텐츠 이해 기능을 설정합니다. 활성화하면 이미지 등 텍스트가 아닌 콘텐츠의 파싱 및 검색을 지원합니다.",
|
||||
description: "멀티모달 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다",
|
||||
},
|
||||
asr: {
|
||||
title: "음성 인식 설정",
|
||||
description: "음성 인식 기능을 설정하여 오디오 파일(mp3, wav, m4a, flac, ogg)을 업로드하고 자동으로 텍스트로 전사합니다",
|
||||
label: "음성 인식 활성화",
|
||||
desc: "활성화하면 오디오 파일을 지식베이스에 업로드하고 자동으로 텍스트로 전사할 수 있습니다",
|
||||
modelLabel: "ASR 모델",
|
||||
modelDescription: "오디오 전사를 위한 음성 인식 모델 (예: OpenAI Whisper)",
|
||||
modelPlaceholder: "ASR 모델을 선택해주세요",
|
||||
},
|
||||
advanced: {
|
||||
title: "고급 설정",
|
||||
@@ -2615,6 +2628,11 @@ export default {
|
||||
desc: "시각 이해 및 멀티모달용 비전 언어 모델 설정",
|
||||
empty: "VLLM 비전 모델 없음",
|
||||
},
|
||||
asr: {
|
||||
title: "ASR 음성 모델",
|
||||
desc: "음성 인식 및 오디오 전사를 위한 음성 인식 모델 설정 (예: OpenAI Whisper)",
|
||||
empty: "ASR 음성 모델 없음",
|
||||
},
|
||||
toasts: {
|
||||
nameRequired: "모델 이름은 비워둘 수 없습니다",
|
||||
nameTooLong: "모델 이름은 100자를 초과할 수 없습니다",
|
||||
|
||||
@@ -1381,17 +1381,20 @@ export default {
|
||||
embedding: 'Настройте модель встраивания для текстовой векторизации',
|
||||
rerank: 'Настройте модель для повторного ранжирования результатов',
|
||||
vllm: 'Настройте визуально-языковую модель для мультимодального понимания',
|
||||
asr: 'Настройте модель распознавания речи для транскрибации аудио',
|
||||
default: 'Настройте информацию о модели'
|
||||
},
|
||||
modelNamePlaceholder: {
|
||||
local: 'например: llama2:latest',
|
||||
remote: 'например: gpt-4, claude-3-opus',
|
||||
localVllm: 'например: llava:latest',
|
||||
remoteVllm: 'например: gpt-4-vision-preview'
|
||||
remoteVllm: 'например: gpt-4-vision-preview',
|
||||
remoteAsr: 'например: whisper-1'
|
||||
},
|
||||
baseUrlLabel: 'Base URL',
|
||||
baseUrlPlaceholder: 'например: https://api.openai.com/v1',
|
||||
baseUrlPlaceholderVllm: 'например: http://localhost:11434/v1',
|
||||
baseUrlPlaceholderAsr: 'например: https://api.openai.com/v1',
|
||||
apiKeyOptional: 'API Key (опционально)',
|
||||
apiKeyPlaceholder: 'Введите API Key',
|
||||
connectionTest: 'Проверка соединения',
|
||||
@@ -1641,6 +1644,7 @@ export default {
|
||||
faq: 'FAQ настройки',
|
||||
graph: 'Граф знаний',
|
||||
multimodal: 'Мультимодальность',
|
||||
asr: 'Распознавание речи',
|
||||
storage: 'Storage Engine',
|
||||
datasource: 'Источники данных',
|
||||
share: 'Sharing'
|
||||
@@ -1836,7 +1840,16 @@ export default {
|
||||
},
|
||||
multimodal: {
|
||||
title: 'Мультимодальная конфигурация',
|
||||
description: 'Настройка понимания мультимодального контента для парсинга и поиска нетекстового содержимого, такого как изображения',
|
||||
description: 'Настройте мультимодальное понимание контента для парсинга и поиска нетекстового контента, например изображений',
|
||||
},
|
||||
asr: {
|
||||
title: 'Конфигурация распознавания речи',
|
||||
description: 'Настройте распознавание речи для загрузки аудиофайлов (mp3, wav, m4a, flac, ogg) и автоматической транскрибации в текст',
|
||||
label: 'Включить распознавание речи',
|
||||
desc: 'При включении аудиофайлы могут быть загружены в базу знаний и автоматически транскрибированы в текст',
|
||||
modelLabel: 'Модель ASR',
|
||||
modelDescription: 'Модель распознавания речи для транскрибации аудио (например, OpenAI Whisper)',
|
||||
modelPlaceholder: 'Выберите модель ASR',
|
||||
},
|
||||
advanced: {
|
||||
title: 'Расширенные настройки',
|
||||
@@ -2380,6 +2393,11 @@ export default {
|
||||
desc: 'Визуально-языковые модели для мультимодального понимания',
|
||||
empty: 'Нет VLLM моделей'
|
||||
},
|
||||
asr: {
|
||||
title: 'ASR модели речи',
|
||||
desc: 'Модели распознавания речи для транскрибации аудио (например, OpenAI Whisper)',
|
||||
empty: 'Нет ASR моделей'
|
||||
},
|
||||
toasts: {
|
||||
nameRequired: 'Название модели не может быть пустым',
|
||||
nameTooLong: 'Название модели не может превышать 100 символов',
|
||||
|
||||
@@ -1528,6 +1528,7 @@ export default {
|
||||
embedding: "配置用于文本向量化的嵌入模型",
|
||||
rerank: "配置用于结果重排序的模型",
|
||||
vllm: "配置用于视觉理解和多模态的视觉语言模型",
|
||||
asr: "配置用于语音识别和音频转录的语音转文本模型",
|
||||
default: "配置模型信息",
|
||||
},
|
||||
modelNamePlaceholder: {
|
||||
@@ -1535,10 +1536,12 @@ export default {
|
||||
remote: "例如:gpt-4, claude-3-opus",
|
||||
localVllm: "例如:llava:latest",
|
||||
remoteVllm: "例如:gpt-4-vision-preview",
|
||||
remoteAsr: "例如:whisper-1",
|
||||
},
|
||||
baseUrlLabel: "Base URL",
|
||||
baseUrlPlaceholder: "例如:https://api.openai.com/v1",
|
||||
baseUrlPlaceholderVllm: "例如:http://localhost:11434/v1",
|
||||
baseUrlPlaceholderAsr: "例如:https://api.openai.com/v1",
|
||||
apiKeyOptional: "API Key(可选)",
|
||||
apiKeyPlaceholder: "输入 API Key",
|
||||
connectionTest: "连接测试",
|
||||
@@ -1943,6 +1946,7 @@ export default {
|
||||
graph: "知识图谱",
|
||||
multimodal: "多模态",
|
||||
datasource: "数据源",
|
||||
asr: "语音识别",
|
||||
share: "共享管理",
|
||||
},
|
||||
basic: {
|
||||
@@ -2145,6 +2149,15 @@ export default {
|
||||
title: "多模态配置",
|
||||
description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索",
|
||||
},
|
||||
asr: {
|
||||
title: "语音识别配置",
|
||||
description: "配置语音识别功能,启用后支持上传音频文件(mp3、wav、m4a、flac、ogg)并自动转录为文本",
|
||||
label: "启用语音识别",
|
||||
desc: "启用后可以上传音频文件到知识库,系统将自动转录为文本进行处理",
|
||||
modelLabel: "ASR 模型",
|
||||
modelDescription: "用于语音转文本的语音识别模型(如 OpenAI Whisper)",
|
||||
modelPlaceholder: "请选择 ASR 模型",
|
||||
},
|
||||
advanced: {
|
||||
title: "高级设置",
|
||||
description: "配置问题生成等高级功能",
|
||||
@@ -2580,6 +2593,11 @@ export default {
|
||||
desc: "配置用于视觉理解和多模态的视觉语言模型",
|
||||
empty: "暂无 VLLM 视觉模型",
|
||||
},
|
||||
asr: {
|
||||
title: "ASR 语音模型",
|
||||
desc: "配置用于语音识别和音频转录的语音转文本模型(如 OpenAI Whisper)",
|
||||
empty: "暂无 ASR 语音模型",
|
||||
},
|
||||
toasts: {
|
||||
nameRequired: "模型名称不能为空",
|
||||
nameTooLong: "模型名称不能超过100个字符",
|
||||
|
||||
@@ -1514,7 +1514,7 @@ async function createNewSession(value: string): Promise<void> {
|
||||
ref="uploadInputRef"
|
||||
type="file"
|
||||
class="document-upload-input"
|
||||
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'"
|
||||
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'"
|
||||
multiple
|
||||
@change="handleDocumentUpload"
|
||||
/>
|
||||
|
||||
@@ -202,6 +202,49 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ASR 语音识别设置 -->
|
||||
<div v-if="!isFAQ" v-show="currentSection === 'asr'" class="section">
|
||||
<div v-if="formData" class="kb-multimodal-settings">
|
||||
<div class="section-header">
|
||||
<h2>{{ $t('knowledgeEditor.asr.title') }}</h2>
|
||||
<p class="section-description">{{ $t('knowledgeEditor.asr.description') }}</p>
|
||||
</div>
|
||||
|
||||
<div class="settings-group">
|
||||
<!-- ASR 开关 -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.asr.label') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.asr.desc') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-switch
|
||||
v-model="formData.asrConfig.enabled"
|
||||
size="medium"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ASR 模型选择 -->
|
||||
<div v-if="formData.asrConfig.enabled" class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.asr.modelLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.asr.modelDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<ModelSelector
|
||||
model-type="ASR"
|
||||
:selected-model-id="formData.asrConfig.modelId"
|
||||
:all-models="allModels"
|
||||
@update:selected-model-id="(val: string) => { if (formData) formData.asrConfig.modelId = val }"
|
||||
:placeholder="$t('knowledgeEditor.asr.modelPlaceholder')"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 高级设置 -->
|
||||
<div v-if="!isFAQ" v-show="currentSection === 'advanced'" class="section">
|
||||
<KBAdvancedSettings
|
||||
@@ -298,6 +341,7 @@ const navItems = computed(() => {
|
||||
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
|
||||
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
|
||||
{ key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') },
|
||||
{ key: 'asr', icon: 'sound', label: t('knowledgeEditor.sidebar.asr') },
|
||||
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
|
||||
)
|
||||
if (props.mode === 'edit' && props.kbId) {
|
||||
@@ -363,6 +407,11 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
||||
enabled: false,
|
||||
vllmModelId: ''
|
||||
},
|
||||
asrConfig: {
|
||||
enabled: false,
|
||||
modelId: '',
|
||||
language: ''
|
||||
},
|
||||
nodeExtractConfig: {
|
||||
enabled: false,
|
||||
text: '',
|
||||
@@ -443,6 +492,11 @@ const loadKBData = async () => {
|
||||
enabled: !!kb.vlm_config?.enabled,
|
||||
vllmModelId: kb.vlm_config?.model_id || ''
|
||||
},
|
||||
asrConfig: {
|
||||
enabled: !!kb.asr_config?.enabled,
|
||||
modelId: kb.asr_config?.model_id || '',
|
||||
language: kb.asr_config?.language || ''
|
||||
},
|
||||
nodeExtractConfig: {
|
||||
enabled: kb.extract_config?.enabled || false,
|
||||
text: kb.extract_config?.text || '',
|
||||
@@ -593,6 +647,15 @@ const buildSubmitData = () => {
|
||||
: ''
|
||||
}
|
||||
|
||||
// 添加ASR语音识别配置
|
||||
data.asr_config = {
|
||||
enabled: formData.value.asrConfig?.enabled || false,
|
||||
model_id: formData.value.asrConfig?.enabled
|
||||
? (formData.value.asrConfig?.modelId || '')
|
||||
: '',
|
||||
language: formData.value.asrConfig?.language || ''
|
||||
}
|
||||
|
||||
// 存储引擎:仅传 provider,参数从全局设置读取
|
||||
// Write to storage_provider_config (authoritative) + storage_config (legacy dual-write)
|
||||
data.storage_provider_config = {
|
||||
@@ -705,6 +768,7 @@ const doSubmit = async () => {
|
||||
llmModelId: data.summary_model_id,
|
||||
embeddingModelId: data.embedding_model_id,
|
||||
vlm_config: data.vlm_config,
|
||||
asr_config: data.asr_config,
|
||||
documentSplitting: {
|
||||
chunkSize: data.chunking_config.chunk_size,
|
||||
chunkOverlap: data.chunking_config.chunk_overlap,
|
||||
|
||||
@@ -221,6 +221,54 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- STT 语音模型 -->
|
||||
<div class="model-category-section" data-model-type="asr">
|
||||
<div class="category-header">
|
||||
<div class="header-info">
|
||||
<h3>{{ $t('modelSettings.asr.title') }}</h3>
|
||||
<p>{{ $t('modelSettings.asr.desc') }}</p>
|
||||
</div>
|
||||
<t-button size="small" theme="primary" @click="openAddDialog('asr')" class="add-model-btn">
|
||||
<template #icon>
|
||||
<t-icon name="add" class="add-icon" />
|
||||
</template>
|
||||
{{ $t('modelSettings.actions.addModel') }}
|
||||
</t-button>
|
||||
</div>
|
||||
|
||||
<div v-if="asrModels.length > 0" class="model-list-container">
|
||||
<div v-for="model in asrModels" :key="model.id" class="model-card" :class="{ 'builtin-model': model.isBuiltin }">
|
||||
<div class="model-info">
|
||||
<div class="model-name">
|
||||
{{ model.name }}
|
||||
<t-tag v-if="model.isBuiltin" theme="primary" size="small">{{ $t('modelSettings.builtinTag') }}</t-tag>
|
||||
</div>
|
||||
<div class="model-meta">
|
||||
<span class="source-tag">{{ model.source === 'local' ? 'Ollama' : $t('modelSettings.source.openaiCompatible') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="model-actions">
|
||||
<t-dropdown
|
||||
:options="getModelOptions('asr', model)"
|
||||
@click="(data: any) => handleMenuAction(data, 'asr', model)"
|
||||
placement="bottom-right"
|
||||
attach="body"
|
||||
>
|
||||
<t-button variant="text" shape="square" size="small" class="more-btn">
|
||||
<t-icon name="more" />
|
||||
</t-button>
|
||||
</t-dropdown>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="empty-state">
|
||||
<p class="empty-text">{{ $t('modelSettings.asr.empty') }}</p>
|
||||
<t-button theme="default" variant="outline" size="small" @click="openAddDialog('asr')">
|
||||
{{ $t('modelSettings.actions.addModel') }}
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 模型编辑器弹窗 -->
|
||||
<ModelEditorDialog
|
||||
v-model:visible="showDialog"
|
||||
@@ -242,7 +290,7 @@ import { listModels, createModel, updateModel as updateModelAPI, deleteModel as
|
||||
const { t } = useI18n()
|
||||
|
||||
const showDialog = ref(false)
|
||||
const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm'>('chat')
|
||||
const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'>('chat')
|
||||
const editingModel = ref<any>(null)
|
||||
const loading = ref(true)
|
||||
|
||||
@@ -268,12 +316,18 @@ const rerankModels = computed(() =>
|
||||
.map(convertToLegacyFormat)
|
||||
)
|
||||
|
||||
const vllmModels = computed(() =>
|
||||
const vllmModels = computed(() =>
|
||||
allModels.value
|
||||
.filter(m => m.type === 'VLLM')
|
||||
.map(convertToLegacyFormat)
|
||||
)
|
||||
|
||||
const asrModels = computed(() =>
|
||||
allModels.value
|
||||
.filter(m => m.type === 'ASR')
|
||||
.map(convertToLegacyFormat)
|
||||
)
|
||||
|
||||
// 将后端模型格式转换为旧的前端格式
|
||||
function convertToLegacyFormat(model: ModelConfig) {
|
||||
return {
|
||||
@@ -306,14 +360,14 @@ const loadModels = async () => {
|
||||
}
|
||||
|
||||
// 打开添加对话框
|
||||
const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm') => {
|
||||
const openAddDialog = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr') => {
|
||||
currentModelType.value = type
|
||||
editingModel.value = null
|
||||
showDialog.value = true
|
||||
}
|
||||
|
||||
// 编辑模型
|
||||
const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
|
||||
const editModel = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
|
||||
// 内置模型不能编辑
|
||||
if (model.isBuiltin) {
|
||||
MessagePlugin.warning(t('modelSettings.toasts.builtinCannotEdit'))
|
||||
@@ -405,7 +459,7 @@ const handleModelSave = async (modelData: any) => {
|
||||
}
|
||||
|
||||
// 删除模型
|
||||
const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', modelId: string) => {
|
||||
const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', modelId: string) => {
|
||||
// 检查是否是内置模型
|
||||
const model = allModels.value.find(m => m.id === modelId)
|
||||
if (model?.is_builtin) {
|
||||
@@ -425,7 +479,7 @@ const deleteModel = async (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model
|
||||
}
|
||||
|
||||
// 获取模型操作菜单选项
|
||||
const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
|
||||
const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
|
||||
const options: any[] = []
|
||||
|
||||
// 内置模型不能编辑和删除
|
||||
@@ -450,7 +504,7 @@ const getModelOptions = (type: 'chat' | 'embedding' | 'rerank' | 'vllm', model:
|
||||
}
|
||||
|
||||
// 处理菜单操作
|
||||
const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm', model: any) => {
|
||||
const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr', model: any) => {
|
||||
const value = data.value
|
||||
|
||||
if (value.indexOf('edit-') === 0) {
|
||||
@@ -464,12 +518,13 @@ const handleMenuAction = (data: { value: string }, type: 'chat' | 'embedding' |
|
||||
}
|
||||
|
||||
// 获取后端模型类型
|
||||
function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' {
|
||||
function getModelType(type: 'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'): 'KnowledgeQA' | 'Embedding' | 'Rerank' | 'VLLM' | 'ASR' {
|
||||
const typeMap = {
|
||||
chat: 'KnowledgeQA' as const,
|
||||
embedding: 'Embedding' as const,
|
||||
rerank: 'Rerank' as const,
|
||||
vllm: 'VLLM' as const
|
||||
vllm: 'VLLM' as const,
|
||||
asr: 'ASR' as const
|
||||
}
|
||||
return typeMap[type]
|
||||
}
|
||||
|
||||
@@ -209,6 +209,14 @@ func getContentTypeByExt(ext string) string {
|
||||
return "image/svg+xml"
|
||||
case ".mp3":
|
||||
return "audio/mpeg"
|
||||
case ".wav":
|
||||
return "audio/wav"
|
||||
case ".m4a":
|
||||
return "audio/mp4"
|
||||
case ".flac":
|
||||
return "audio/flac"
|
||||
case ".ogg":
|
||||
return "audio/ogg"
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
default:
|
||||
|
||||
@@ -259,6 +259,15 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
logger.Info(ctx, "Image multimodal configuration validation passed")
|
||||
}
|
||||
|
||||
// 检查音频ASR配置完整性 - 只在音频文件时校验
|
||||
if IsAudioType(getFileType(fileName)) {
|
||||
if !kb.ASRConfig.IsASREnabled() {
|
||||
logger.Error(ctx, "ASR model is not configured")
|
||||
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
|
||||
}
|
||||
logger.Info(ctx, "Audio ASR configuration validation passed")
|
||||
}
|
||||
|
||||
// Validate file type
|
||||
logger.Infof(ctx, "Checking file type: %s", fileName)
|
||||
if !isValidFileType(fileName) {
|
||||
@@ -596,6 +605,11 @@ var allowedFileURLExtensions = map[string]bool{
|
||||
"pdf": true,
|
||||
"docx": true,
|
||||
"doc": true,
|
||||
"mp3": true,
|
||||
"wav": true,
|
||||
"m4a": true,
|
||||
"flac": true,
|
||||
"ogg": true,
|
||||
}
|
||||
|
||||
// maxFileURLSize is the maximum allowed file size for file URL import (10MB)
|
||||
@@ -2987,7 +3001,8 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
// isValidFileType checks if a file type is supported
|
||||
func isValidFileType(filename string) bool {
|
||||
switch strings.ToLower(getFileType(filename)) {
|
||||
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json":
|
||||
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
|
||||
"mp3", "wav", "m4a", "flac", "ogg":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -7399,6 +7414,16 @@ func IsImageType(fileType string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// IsAudioType checks if a file type is an audio format
|
||||
func IsAudioType(fileType string) bool {
|
||||
switch strings.ToLower(fileType) {
|
||||
case "mp3", "wav", "m4a", "flac", "ogg":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// downloadFileFromURL downloads a remote file to a temp file and returns its binary content.
|
||||
// payloadFileName and payloadFileType are in/out pointers: if they point to an empty string,
|
||||
// the function resolves the value from Content-Disposition / URL path and writes it back.
|
||||
@@ -7641,6 +7666,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查音频ASR配置(仅对文件导入)
|
||||
if payload.FilePath != "" && IsAudioType(payload.FileType) && !kb.ASRConfig.IsASREnabled() {
|
||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
||||
Errorf("processDocument audio without ASR model configured")
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "上传音频文件需要设置ASR语音识别模型"
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
|
||||
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
|
||||
var convertResult *types.ReadResult
|
||||
var chunks []types.ParsedChunk
|
||||
@@ -7764,6 +7800,54 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1.5: ASR transcription for audio files
|
||||
if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 {
|
||||
if !kb.ASRConfig.IsASREnabled() {
|
||||
logger.Error(ctx, "Audio file detected but ASR is not configured")
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "ASR model is not configured for audio transcription"
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes",
|
||||
knowledge.ID, len(convertResult.AudioData))
|
||||
|
||||
asrModel, err := s.modelService.GetASRModel(ctx, kb.ASRConfig.ModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err)
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = fmt.Sprintf("failed to get ASR model: %v", err)
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
|
||||
transcribedText, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
|
||||
if isLastRetry {
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = fmt.Sprintf("audio transcription failed: %v", err)
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
}
|
||||
return fmt.Errorf("audio transcription failed: %w", err)
|
||||
}
|
||||
|
||||
if transcribedText == "" {
|
||||
logger.Warn(ctx, "[ASR] Transcription returned empty text")
|
||||
transcribedText = "[No speech detected in audio file]"
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(transcribedText))
|
||||
// Replace the audio placeholder with the transcribed text
|
||||
convertResult.MarkdownContent = transcribedText
|
||||
convertResult.IsAudio = false
|
||||
convertResult.AudioData = nil
|
||||
}
|
||||
|
||||
// Step 2: Store images and update markdown references
|
||||
var storedImages []docparser.StoredImage
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/asr"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -458,3 +459,50 @@ func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM
|
||||
|
||||
// Note: default model selection logic has been removed; models no longer
|
||||
// maintain a per-type default flag at the service layer.
|
||||
|
||||
// GetASRModel retrieves and initializes an automatic speech recognition model instance.
|
||||
func (s *modelService) GetASRModel(ctx context.Context, modelId string) (asr.ASR, error) {
|
||||
if modelId == "" {
|
||||
return nil, errors.New("model ID cannot be empty")
|
||||
}
|
||||
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
|
||||
model, err := s.repo.GetByID(ctx, tenantID, modelId)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": modelId,
|
||||
"tenant_id": tenantID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if model == nil {
|
||||
return nil, ErrModelNotFound
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Getting ASR model: %s, source: %s", model.Name, model.Source)
|
||||
|
||||
ifType := model.Parameters.InterfaceType
|
||||
if ifType == "" {
|
||||
ifType = "openai"
|
||||
}
|
||||
|
||||
sttModel, err := asr.NewASR(&asr.Config{
|
||||
ModelID: model.ID,
|
||||
APIKey: model.Parameters.APIKey,
|
||||
BaseURL: model.Parameters.BaseURL,
|
||||
ModelName: model.Name,
|
||||
Source: model.Source,
|
||||
InterfaceType: ifType,
|
||||
})
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"model_name": model.Name,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sttModel, nil
|
||||
}
|
||||
|
||||
@@ -24,10 +24,17 @@ var imageFormats = map[string]bool{
|
||||
"bmp": true, "tiff": true, "webp": true,
|
||||
}
|
||||
|
||||
var audioFormats = map[string]bool{
|
||||
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for k := range imageFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
for k := range audioFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
}
|
||||
|
||||
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
|
||||
@@ -65,6 +72,8 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
|
||||
return &types.ReadResult{MarkdownContent: md}, nil
|
||||
case imageFormats[ft]:
|
||||
return imageToResult(req.FileName, req.FileContent), nil
|
||||
case audioFormats[ft]:
|
||||
return audioToResult(req.FileName, req.FileContent), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported simple format: %s", ft)
|
||||
}
|
||||
@@ -99,6 +108,28 @@ func IsImageFormat(fileType string) bool {
|
||||
return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// IsAudioFormat returns true if the file type is a recognized audio format.
|
||||
func IsAudioFormat(fileType string) bool {
|
||||
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// audioToResult wraps a standalone audio file. The actual transcription is
|
||||
// handled by the ASR model in the knowledge service pipeline. Here we just
|
||||
// return a placeholder markdown with the raw bytes preserved for upstream
|
||||
// processing.
|
||||
func audioToResult(fileName string, data []byte) *types.ReadResult {
|
||||
if fileName == "" {
|
||||
fileName = "audio.mp3"
|
||||
}
|
||||
// Return a placeholder; the knowledge service will replace this with
|
||||
// the ASR transcription result.
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: fmt.Sprintf("[Audio file: %s]", fileName),
|
||||
IsAudio: true,
|
||||
AudioData: data,
|
||||
}
|
||||
}
|
||||
|
||||
// ensureOriginalImageRef checks whether the input file is an image and, if the
|
||||
// returned markdown does not already contain a markdown image reference for it,
|
||||
// prepends one and appends the raw bytes to imageRefs. This guarantees that
|
||||
|
||||
@@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string {
|
||||
return "Simple format & image parsing (no external service required)"
|
||||
}
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"}
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
}
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ASR defines the interface for Automatic Speech Recognition model operations.
|
||||
type ASR interface {
|
||||
// Transcribe sends audio bytes to the ASR model and returns the transcribed text.
|
||||
Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error)
|
||||
|
||||
GetModelName() string
|
||||
GetModelID() string
|
||||
}
|
||||
|
||||
// Config holds the configuration needed to create an ASR instance.
|
||||
type Config struct {
|
||||
Source types.ModelSource
|
||||
BaseURL string
|
||||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
InterfaceType string // "openai" (default)
|
||||
Language string // optional: specify language for transcription
|
||||
}
|
||||
|
||||
// NewASR creates an ASR instance based on the provided configuration.
|
||||
func NewASR(config *Config) (ASR, error) {
|
||||
ifType := strings.ToLower(config.InterfaceType)
|
||||
if ifType == "" {
|
||||
ifType = "openai"
|
||||
}
|
||||
|
||||
switch ifType {
|
||||
case "openai":
|
||||
return NewOpenAIASR(config)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported ASR interface type: %s", ifType)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
openai "github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
const (
|
||||
asrDefaultTimeout = 300 * time.Second // audio transcription can be slow
|
||||
)
|
||||
|
||||
// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.
|
||||
type OpenAIASR struct {
|
||||
modelName string
|
||||
modelID string
|
||||
client *openai.Client
|
||||
baseURL string
|
||||
language string
|
||||
}
|
||||
|
||||
// NewOpenAIASR creates an OpenAI-compatible ASR instance.
|
||||
func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
|
||||
apiCfg := openai.DefaultConfig(config.APIKey)
|
||||
if config.BaseURL != "" {
|
||||
apiCfg.BaseURL = config.BaseURL
|
||||
}
|
||||
apiCfg.HTTPClient = &http.Client{Timeout: asrDefaultTimeout}
|
||||
|
||||
return &OpenAIASR{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
client: openai.NewClientWithConfig(apiCfg),
|
||||
baseURL: config.BaseURL,
|
||||
language: config.Language,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
|
||||
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error) {
|
||||
if len(audioBytes) == 0 {
|
||||
return "", fmt.Errorf("audio bytes are empty")
|
||||
}
|
||||
|
||||
// Ensure fileName has an extension for proper MIME type detection
|
||||
if fileName == "" {
|
||||
fileName = "audio.mp3"
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",
|
||||
s.modelName, s.baseURL, len(audioBytes), fileName)
|
||||
|
||||
req := openai.AudioRequest{
|
||||
Model: s.modelName,
|
||||
FilePath: fileName,
|
||||
Reader: bytes.NewReader(audioBytes),
|
||||
Format: openai.AudioResponseFormatJSON,
|
||||
}
|
||||
|
||||
if s.language != "" {
|
||||
req.Language = s.language
|
||||
}
|
||||
|
||||
resp, err := s.client.CreateTranscription(ctx, req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ASR transcription request failed: %w", err)
|
||||
}
|
||||
|
||||
text := strings.TrimSpace(resp.Text)
|
||||
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (s *OpenAIASR) GetModelName() string { return s.modelName }
|
||||
func (s *OpenAIASR) GetModelID() string { return s.modelID }
|
||||
|
||||
// DetectAudioFormat returns a file extension hint for the given audio bytes.
|
||||
func DetectAudioFormat(data []byte, fileName string) string {
|
||||
if fileName != "" {
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext != "" {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
// Try to detect from magic bytes
|
||||
if len(data) >= 4 {
|
||||
// MP3: starts with ID3 or 0xFF 0xFB
|
||||
if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {
|
||||
return ".mp3"
|
||||
}
|
||||
if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
|
||||
return ".mp3"
|
||||
}
|
||||
// FLAC: starts with "fLaC"
|
||||
if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {
|
||||
return ".flac"
|
||||
}
|
||||
// OGG: starts with "OggS"
|
||||
if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {
|
||||
return ".ogg"
|
||||
}
|
||||
// WAV: starts with "RIFF"
|
||||
if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {
|
||||
return ".wav"
|
||||
}
|
||||
}
|
||||
// M4A: check for ftyp box
|
||||
if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {
|
||||
return ".m4a"
|
||||
}
|
||||
return ".mp3" // default
|
||||
}
|
||||
@@ -32,12 +32,14 @@ func (p *AliyunProvider) Info() ProviderInfo {
|
||||
types.ModelTypeEmbedding: AliyunChatBaseURL,
|
||||
types.ModelTypeRerank: AliyunRerankBaseURL,
|
||||
types.ModelTypeVLLM: AliyunChatBaseURL,
|
||||
types.ModelTypeASR: AliyunChatBaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RequiresAuth: true,
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ func (p *GenericProvider) Info() ProviderInfo {
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RequiresAuth: false, // 可能需要也可能不需要
|
||||
}
|
||||
|
||||
@@ -31,12 +31,14 @@ func (p *GPUStackProvider) Info() ProviderInfo {
|
||||
types.ModelTypeEmbedding: GPUStackBaseURL,
|
||||
types.ModelTypeRerank: GPUStackRerankBaseURL,
|
||||
types.ModelTypeVLLM: GPUStackBaseURL,
|
||||
types.ModelTypeASR: GPUStackBaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RequiresAuth: true, // GPUStack 需要 API Key
|
||||
}
|
||||
|
||||
@@ -28,12 +28,14 @@ func (p *OpenAIProvider) Info() ProviderInfo {
|
||||
types.ModelTypeEmbedding: OpenAIBaseURL,
|
||||
types.ModelTypeRerank: OpenAIBaseURL,
|
||||
types.ModelTypeVLLM: OpenAIBaseURL,
|
||||
types.ModelTypeASR: OpenAIBaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RequiresAuth: true,
|
||||
}
|
||||
|
||||
@@ -28,12 +28,14 @@ func (p *SiliconFlowProvider) Info() ProviderInfo {
|
||||
types.ModelTypeEmbedding: SiliconFlowBaseURL,
|
||||
types.ModelTypeRerank: SiliconFlowBaseURL,
|
||||
types.ModelTypeVLLM: SiliconFlowBaseURL,
|
||||
types.ModelTypeASR: SiliconFlowBaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RequiresAuth: true,
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ type ReadResult struct {
|
||||
ImageDirPath string
|
||||
Metadata map[string]string
|
||||
Error string
|
||||
IsAudio bool // true when the result contains raw audio data needing ASR transcription
|
||||
AudioData []byte // raw audio bytes for ASR processing
|
||||
}
|
||||
|
||||
// ImageRef represents an image reference extracted from the document.
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/asr"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
@@ -32,6 +33,8 @@ type ModelService interface {
|
||||
GetChatModel(ctx context.Context, modelId string) (chat.Chat, error)
|
||||
// GetVLMModel gets a vision language model
|
||||
GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error)
|
||||
// GetASRModel gets an automatic speech recognition model
|
||||
GetASRModel(ctx context.Context, modelId string) (asr.ASR, error)
|
||||
}
|
||||
|
||||
// ModelRepository defines the model repository interface
|
||||
|
||||
@@ -60,6 +60,8 @@ type KnowledgeBase struct {
|
||||
SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"`
|
||||
// VLM config
|
||||
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
|
||||
// ASR config (Automatic Speech Recognition)
|
||||
ASRConfig ASRConfig `yaml:"asr_config" json:"asr_config" gorm:"type:json"`
|
||||
// Storage provider config (new): only stores provider selection; credentials from tenant StorageEngineConfig
|
||||
StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"`
|
||||
// Deprecated: legacy COS config column. Kept for backward compatibility with old data.
|
||||
@@ -386,6 +388,35 @@ func (c *VLMConfig) Scan(value interface{}) error {
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// ASRConfig represents the ASR (Automatic Speech Recognition) configuration
|
||||
type ASRConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
ModelID string `yaml:"model_id" json:"model_id"`
|
||||
Language string `yaml:"language" json:"language"` // optional: language hint for transcription
|
||||
}
|
||||
|
||||
// IsASREnabled checks if ASR is enabled with a valid model
|
||||
func (c ASRConfig) IsASREnabled() bool {
|
||||
return c.Enabled && c.ModelID != ""
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert ASRConfig to database value
|
||||
func (c ASRConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface, used to convert database value to ASRConfig
|
||||
func (c *ASRConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// ExtractConfig represents the extract configuration for a knowledge base
|
||||
type ExtractConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
ModelTypeRerank ModelType = "Rerank" // Rerank model
|
||||
ModelTypeKnowledgeQA ModelType = "KnowledgeQA" // KnowledgeQA model
|
||||
ModelTypeVLLM ModelType = "VLLM" // VLLM model
|
||||
ModelTypeASR ModelType = "ASR" // ASR (Automatic Speech Recognition) model
|
||||
)
|
||||
|
||||
// ModelStatus represents the status of the model
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration: 000025_add_asr_config (rollback)
|
||||
-- Description: Remove asr_config column from knowledge_bases.
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000025] Removing asr_config column from knowledge_bases'; END $$;
|
||||
|
||||
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS asr_config;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column removed successfully'; END $$;
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Migration: 000025_add_asr_config
|
||||
-- Description: Add asr_config column to knowledge_bases for ASR (Automatic Speech Recognition) configuration.
|
||||
-- ASR config stores the model ID and language settings for audio transcription.
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000025] Adding asr_config column to knowledge_bases'; END $$;
|
||||
|
||||
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS asr_config JSONB;
|
||||
|
||||
COMMENT ON COLUMN knowledge_bases.asr_config IS 'ASR (Automatic Speech Recognition) configuration: {"enabled": bool, "model_id": string, "language": string}';
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column added successfully'; END $$;
|
||||
Reference in New Issue
Block a user