feat(asr): add ASR model connection check and update related components

- Implemented a new function to check ASR model connectivity via the /v1/audio/transcriptions endpoint.
- Updated ModelEditorDialog.vue to integrate ASR model checks and adjust UI elements accordingly.
- Enhanced localization files to reflect changes in ASR terminology and descriptions.
- Added ASR configuration options to the knowledge base and updated related migration scripts.
- Refactored model handling to support ASR as a distinct model type in the application.
This commit is contained in:
wizardchen
2026-04-02 20:53:44 +08:00
committed by lyingbug
parent d3c4992e01
commit c93f4e94ae
17 changed files with 231 additions and 83 deletions
+21
View File
@@ -313,6 +313,27 @@ export function checkRerankModel(modelConfig: {
});
}
// 检查 ASR 模型连接(通过 /v1/audio/transcriptions 端点测试)
export function checkASRModel(modelConfig: {
modelName: string;
baseUrl: string;
apiKey?: string;
}): Promise<{
available: boolean;
message?: string;
}> {
return new Promise((resolve, reject) => {
post('/api/v1/initialization/asr/check', modelConfig)
.then((response: any) => {
resolve(response.data || {});
})
.catch((error: any) => {
console.error('Failed to check ASR model:', error);
reject(error);
});
});
}
export function testMultimodalFunction(testData: {
image: File;
vlm_model: string;
+44 -27
View File
@@ -124,10 +124,11 @@
<!-- 厂商选择器 -->
<div class="form-item">
<label class="form-label">{{ $t('model.editor.providerLabel') }}</label>
<t-select
v-model="formData.provider"
<t-select
v-model="formData.provider"
:placeholder="$t('model.editor.providerPlaceholder')"
@change="handleProviderChange"
:popup-props="{ overlayClassName: 'provider-select-popup' }"
>
<t-option
v-for="opt in providerOptions"
@@ -260,7 +261,7 @@
<script setup lang="ts">
import { ref, watch, computed, onUnmounted, nextTick } from 'vue'
import { MessagePlugin } from 'tdesign-vue-next'
import { checkOllamaModels, checkRemoteModel, testEmbeddingModel, checkRerankModel, listOllamaModels, downloadOllamaModel, getDownloadProgress, checkOllamaStatus, listModelProviders, type OllamaModelInfo, type ModelProviderOption } from '@/api/initialization'
import { checkOllamaModels, checkRemoteModel, testEmbeddingModel, checkRerankModel, checkASRModel, listOllamaModels, downloadOllamaModel, getDownloadProgress, checkOllamaStatus, listModelProviders, type OllamaModelInfo, type ModelProviderOption } from '@/api/initialization'
import { useI18n } from 'vue-i18n'
import { useUIStore } from '@/stores/ui'
@@ -323,11 +324,10 @@ const fallbackProviderOptions = computed(() => [
chat: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
embedding: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
rerank: 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank',
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
asr: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
vllm: 'https://dashscope.aliyuncs.com/compatible-mode/v1'
},
description: t('model.editor.providers.aliyun.description'),
modelTypes: ['chat', 'embedding', 'rerank', 'vllm', 'asr']
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
},
{
value: 'zhipu',
@@ -851,9 +851,8 @@ const checkRemoteAPI = async () => {
break
case 'asr':
// ASR 模型(语音识别)
// ASR 使用 checkRemoteModel 进行基础连接测试
result = await checkRemoteModel({
// ASR 模型(语音识别)— 使用专用的 ASR 测试接口(/v1/audio/transcriptions)
result = await checkASRModel({
modelName: formData.value.modelName,
baseUrl: formData.value.baseUrl,
apiKey: formData.value.apiKey || ''
@@ -1252,24 +1251,8 @@ const handleOverlayMouseUp = () => {
}
}
// 厂商选择器样式
.provider-option {
display: flex;
flex-direction: column;
gap: 2px;
padding: 4px 0;
.provider-name {
font-size: 14px;
font-weight: 500;
color: var(--td-text-color-primary);
}
.provider-desc {
font-size: 12px;
color: var(--td-text-color-placeholder);
}
}
// 厂商选择器样式 — 移至非 scoped 块,因为 t-select popup 渲染到 body 下
// .provider-option 样式见文件末尾
// 单选按钮组
:deep(.t-radio-group) {
@@ -1641,3 +1624,37 @@ const handleOverlayMouseUp = () => {
}
}
</style>
<!-- 非 scoped 样式:t-select popup 渲染到 body 下,scoped 样式无法覆盖 -->
<style lang="less">
.provider-select-popup {
// 覆盖 TDesign option 默认固定高度,让两行内容正常展示
.t-select-option {
height: auto !important;
padding: 0 8px;
}
.provider-option {
display: flex;
flex-direction: column;
gap: 2px;
padding: 6px 0;
.provider-name {
font-size: 14px;
font-weight: 500;
color: var(--td-text-color-primary);
line-height: 22px;
}
.provider-desc {
font-size: 12px;
color: var(--td-text-color-placeholder);
line-height: 18px;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
}
}
</style>
+6 -5
View File
@@ -1469,9 +1469,9 @@ export default {
advanced: 'Advanced Settings',
faq: 'FAQ Settings',
graph: 'Knowledge Graph',
multimodal: 'Multimodal',
multimodal: 'Image Processing',
asr: 'Audio Processing',
datasource: 'Data Sources',
asr: 'Speech Recognition',
share: 'Sharing'
},
basic: {
@@ -1674,11 +1674,11 @@ export default {
childChunkSizeDescription: 'Size of child chunks used for embedding matching (64-1024)'
},
multimodal: {
title: 'Multimodal Configuration',
description: 'Configure multimodal content understanding for parsing and retrieving non-text content like images',
title: 'Image Processing Configuration',
description: 'Configure image content understanding for parsing and retrieving non-text content like images',
},
asr: {
title: 'Speech Recognition Configuration',
title: 'Audio Processing Configuration',
description: 'Configure speech recognition to upload audio files (mp3, wav, m4a, flac, ogg) and automatically transcribe to text',
label: 'Enable Speech Recognition',
desc: 'When enabled, audio files can be uploaded to the knowledge base and automatically transcribed to text',
@@ -3097,6 +3097,7 @@ export default {
fileTypeText: 'Plain Text',
fileTypeJson: 'JSON Files',
fileTypeImage: 'Images',
fileTypeAudio: 'Audio Files',
engines: {
builtin: {
name: 'Built-in',
+6 -5
View File
@@ -1957,8 +1957,8 @@ export default {
advanced: "고급 설정",
faq: "FAQ 설정",
graph: "지식 그래프",
multimodal: "멀티모달",
asr: "음성 인식",
multimodal: "이미지 처리",
asr: "오디오 처리",
share: "공유관리",
storage: "스토리지 엔진",
datasource: "데이터 소스",
@@ -2173,11 +2173,11 @@ export default {
childChunkSizeDescription: "임베딩 매칭에 사용되는 자식 청크의 문자 수 (64-1024)",
},
multimodal: {
title: "멀티모달 설정",
description: "멀티모달 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다",
title: "이미지 처리 설정",
description: "이미지 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다",
},
asr: {
title: "음성 인식 설정",
title: "오디오 처리 설정",
description: "음성 인식 기능을 설정하여 오디오 파일(mp3, wav, m4a, flac, ogg)을 업로드하고 자동으로 텍스트로 전사합니다",
label: "음성 인식 활성화",
desc: "활성화하면 오디오 파일을 지식베이스에 업로드하고 자동으로 텍스트로 전사할 수 있습니다",
@@ -3151,6 +3151,7 @@ export default {
fileTypeText: '일반 텍스트',
fileTypeJson: 'JSON 파일',
fileTypeImage: '이미지',
fileTypeAudio: '오디오 파일',
engines: {
builtin: {
name: '내장',
+6 -5
View File
@@ -1643,8 +1643,8 @@ export default {
advanced: 'Дополнительные настройки',
faq: 'FAQ настройки',
graph: 'Граф знаний',
multimodal: 'Мультимодальность',
asr: 'Распознавание речи',
multimodal: 'Обработка изображений',
asr: 'Обработка аудио',
storage: 'Storage Engine',
datasource: 'Источники данных',
share: 'Sharing'
@@ -1839,11 +1839,11 @@ export default {
childChunkSizeDescription: 'Размер дочерних блоков для поиска по эмбеддингам (64-1024)'
},
multimodal: {
title: 'Мультимодальная конфигурация',
description: 'Настройте мультимодальное понимание контента для парсинга и поиска нетекстового контента, например изображений',
title: 'Обработка изображений',
description: 'Настройте понимание изображений для парсинга и поиска нетекстового контента',
},
asr: {
title: 'Конфигурация распознавания речи',
title: 'Обработка аудио',
description: 'Настройте распознавание речи для загрузки аудиофайлов (mp3, wav, m4a, flac, ogg) и автоматической транскрибации в текст',
label: 'Включить распознавание речи',
desc: 'При включении аудиофайлы могут быть загружены в базу знаний и автоматически транскрибированы в текст',
@@ -2767,6 +2767,7 @@ export default {
fileTypeText: 'Текстовые файлы',
fileTypeJson: 'Файлы JSON',
fileTypeImage: 'Изображения',
fileTypeAudio: 'Аудиофайлы',
engines: {
builtin: {
name: 'Встроенный',
+6 -5
View File
@@ -1944,9 +1944,9 @@ export default {
advanced: "高级设置",
faq: "FAQ 设置",
graph: "知识图谱",
multimodal: "多模态",
multimodal: "图像处理",
asr: "音频处理",
datasource: "数据源",
asr: "语音识别",
share: "共享管理",
},
basic: {
@@ -2146,11 +2146,11 @@ export default {
childChunkSizeDescription: "用于向量匹配的子块字符数(64-1024)",
},
multimodal: {
title: "多模态配置",
description: "配置多模态内容理解能力,启用后支持图片等非文本内容的解析和检索",
title: "图像处理配置",
description: "配置图像内容理解能力,启用后支持图片等非文本内容的解析和检索",
},
asr: {
title: "语音识别配置",
title: "音频处理配置",
description: "配置语音识别功能,启用后支持上传音频文件(mp3、wav、m4a、flac、ogg)并自动转录为文本",
label: "启用语音识别",
desc: "启用后可以上传音频文件到知识库,系统将自动转录为文本进行处理",
@@ -3095,6 +3095,7 @@ export default {
fileTypeText: "纯文本",
fileTypeJson: "JSON 文件",
fileTypeImage: "图片",
fileTypeAudio: "音频文件",
engines: {
builtin: {
name: "内置",
@@ -202,7 +202,7 @@
</div>
</div>
<!-- ASR 语音识别设置 -->
<!-- 音频处理(ASR)设置 -->
<div v-if="!isFAQ" v-show="currentSection === 'asr'" class="section">
<div v-if="formData" class="kb-multimodal-settings">
<div class="section-header">
@@ -337,11 +337,11 @@ const navItems = computed(() => {
} else {
items.push(
{ key: 'parser', icon: 'file-search', label: t('settings.parserEngine') },
{ key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') },
{ key: 'asr', icon: 'sound', label: t('knowledgeEditor.sidebar.asr') },
{ key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') },
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
{ key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') },
{ key: 'asr', icon: 'sound', label: t('knowledgeEditor.sidebar.asr') },
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
)
if (props.mode === 'edit' && props.kbId) {
@@ -159,6 +159,7 @@ const fileTypeGroups = computed(() => {
const txtExts = ['txt'].filter(e => ft.has(e))
const jsonExts = ['json'].filter(e => ft.has(e))
const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e))
const audioExts = ['mp3', 'wav', 'm4a', 'flac', 'ogg'].filter(e => ft.has(e))
if (pdfExts.length) groups.push({ key: 'pdf', label: t('kbSettings.parser.fileTypePdf'), icon: 'file-pdf', extensions: pdfExts })
if (officeExts.length) groups.push({ key: 'office', label: t('kbSettings.parser.fileTypeWord'), icon: 'file-word', extensions: officeExts })
@@ -169,6 +170,7 @@ const fileTypeGroups = computed(() => {
if (txtExts.length) groups.push({ key: 'text', label: t('kbSettings.parser.fileTypeText'), icon: 'file', extensions: txtExts })
if (jsonExts.length) groups.push({ key: 'json', label: t('kbSettings.parser.fileTypeJson'), icon: 'file-code', extensions: jsonExts })
if (imageExts.length) groups.push({ key: 'image', label: t('kbSettings.parser.fileTypeImage'), icon: 'image', extensions: imageExts })
if (audioExts.length) groups.push({ key: 'audio', label: t('kbSettings.parser.fileTypeAudio'), icon: 'sound', extensions: audioExts })
return groups
})
+5 -11
View File
@@ -483,18 +483,12 @@ func (s *modelService) GetASRModel(ctx context.Context, modelId string) (asr.ASR
logger.Infof(ctx, "Getting ASR model: %s, source: %s", model.Name, model.Source)
ifType := model.Parameters.InterfaceType
if ifType == "" {
ifType = "openai"
}
sttModel, err := asr.NewASR(&asr.Config{
ModelID: model.ID,
APIKey: model.Parameters.APIKey,
BaseURL: model.Parameters.BaseURL,
ModelName: model.Name,
Source: model.Source,
InterfaceType: ifType,
ModelID: model.ID,
APIKey: model.Parameters.APIKey,
BaseURL: model.Parameters.BaseURL,
ModelName: model.Name,
Source: model.Source,
})
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
+119
View File
@@ -2,6 +2,7 @@ package handler
import (
"context"
_ "embed"
"encoding/json"
"fmt"
"io"
@@ -17,6 +18,7 @@ import (
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
@@ -29,6 +31,9 @@ import (
"github.com/ollama/ollama/api"
)
//go:embed testdata/asr_test.wav
var asrTestWAV []byte
// DownloadTask 下载任务信息
type DownloadTask struct {
ID string `json:"id"`
@@ -89,6 +94,7 @@ type KBModelConfigRequest struct {
LLMModelID string `json:"llmModelId" binding:"required"`
EmbeddingModelID string `json:"embeddingModelId" binding:"required"`
VLMConfig *types.VLMConfig `json:"vlm_config"`
ASRConfig *types.ASRConfig `json:"asr_config"`
// 文档分块配置
DocumentSplitting struct {
@@ -281,6 +287,19 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
kb.VLMConfig.ModelID = ""
}
// 处理ASR语音识别配置
kb.ASRConfig = types.ASRConfig{}
if req.ASRConfig != nil && req.ASRConfig.Enabled && req.ASRConfig.ModelID != "" {
asrModel, err := h.modelService.GetModelByID(ctx, req.ASRConfig.ModelID)
if err != nil || asrModel == nil {
logger.Warn(ctx, "ASR model not found")
} else {
kb.ASRConfig.Enabled = true
kb.ASRConfig.ModelID = req.ASRConfig.ModelID
kb.ASRConfig.Language = req.ASRConfig.Language
}
}
// 更新文档分块配置
if req.DocumentSplitting.ChunkSize > 0 {
kb.ChunkingConfig.ChunkSize = req.DocumentSplitting.ChunkSize
@@ -1768,6 +1787,106 @@ func (h *InitializationHandler) CheckRerankModel(c *gin.Context) {
})
}
// CheckASRModel godoc
// @Summary 检查ASR模型
// @Description 检查ASR(语音识别)模型连接是否正常,通过发送一段静默音频测试 /v1/audio/transcriptions 端点
// @Tags 初始化
// @Accept json
// @Produce json
// @Param request body object true "ASR检查请求"
// @Success 200 {object} map[string]interface{} "检查结果"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /initialization/models/asr/check [post]
func (h *InitializationHandler) CheckASRModel(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Checking ASR model connection")
var req struct {
ModelName string `json:"modelName" binding:"required"`
BaseURL string `json:"baseUrl" binding:"required"`
APIKey string `json:"apiKey"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse ASR model check request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
if req.ModelName == "" || req.BaseURL == "" {
logger.Error(ctx, "Model name and base URL are required for ASR check")
c.Error(errors.NewBadRequestError("模型名称和Base URL不能为空"))
return
}
// SSRF validation
if err := utils.ValidateURLForSSRF(req.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for ASR BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
// 使用 ASR 模块测试连接:发送一段极短的静默 WAV 音频
asrInstance, err := asr.NewASR(&asr.Config{
BaseURL: req.BaseURL,
ModelName: req.ModelName,
APIKey: req.APIKey,
Source: "remote",
})
if err != nil {
logger.Errorf(ctx, "Failed to create ASR instance for check: %v", err)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
"available": false,
"message": fmt.Sprintf("创建ASR实例失败: %v", err),
},
})
return
}
text, err := asrInstance.Transcribe(ctx, asrTestWAV, "asr_test.wav")
available := true
message := "ASR连接成功"
if err != nil {
errMsg := err.Error()
switch {
case strings.Contains(errMsg, "401") || strings.Contains(errMsg, "Unauthorized") || strings.Contains(errMsg, "authentication"):
available = false
message = "认证失败,请检查API Key"
case strings.Contains(errMsg, "404") || strings.Contains(errMsg, "Not Found"):
available = false
message = "API端点不存在,请检查Base URL"
case strings.Contains(errMsg, "connection refused") || strings.Contains(errMsg, "no such host") || strings.Contains(errMsg, "dial tcp"):
available = false
message = "无法连接到服务器,请检查Base URL"
case strings.Contains(errMsg, "model") && strings.Contains(errMsg, "not found"):
available = false
message = "模型不存在,请检查模型名称"
default:
logger.Infof(ctx, "ASR check got non-fatal error (endpoint reachable): %v", err)
available = true
message = fmt.Sprintf("ASR端点可达(非致命错误: %s)", errMsg)
}
} else if text != "" {
message = fmt.Sprintf("ASR连接成功,转写结果: %s", text)
}
logger.Infof(ctx, "ASR model check completed, available: %v, message: %s", available, message)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
"available": available,
"message": message,
},
})
}
// 使用结构体解析表单数据
type testMultimodalForm struct {
VLMModel string `form:"vlm_model"`
+4
View File
@@ -402,6 +402,8 @@ func modelTypeToFrontend(mt types.ModelType) string {
return "rerank"
case types.ModelTypeVLLM:
return "vllm"
case types.ModelTypeASR:
return "asr"
default:
return string(mt)
}
@@ -437,6 +439,8 @@ func (h *ModelHandler) ListModelProviders(c *gin.Context) {
backendModelType = types.ModelTypeRerank
case "vllm":
backendModelType = types.ModelTypeVLLM
case "asr":
backendModelType = types.ModelTypeASR
default:
backendModelType = types.ModelType(modelType)
}
Binary file not shown.
+8 -20
View File
@@ -2,8 +2,6 @@ package asr
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -19,26 +17,16 @@ type ASR interface {
// Config holds the configuration needed to create an ASR instance.
type Config struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
InterfaceType string // "openai" (default)
Language string // optional: specify language for transcription
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
Language string // optional: specify language for transcription
}
// NewASR creates an ASR instance based on the provided configuration.
// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API.
func NewASR(config *Config) (ASR, error) {
ifType := strings.ToLower(config.InterfaceType)
if ifType == "" {
ifType = "openai"
}
switch ifType {
case "openai":
return NewOpenAIASR(config)
default:
return nil, fmt.Errorf("unsupported ASR interface type: %s", ifType)
}
return NewOpenAIASR(config)
}
-2
View File
@@ -32,14 +32,12 @@ func (p *AliyunProvider) Info() ProviderInfo {
types.ModelTypeEmbedding: AliyunChatBaseURL,
types.ModelTypeRerank: AliyunRerankBaseURL,
types.ModelTypeVLLM: AliyunChatBaseURL,
types.ModelTypeASR: AliyunChatBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RequiresAuth: true,
}
+1
View File
@@ -425,6 +425,7 @@ func RegisterInitializationRoutes(r *gin.RouterGroup, handler *handler.Initializ
r.POST("/initialization/remote/check", handler.CheckRemoteModel)
r.POST("/initialization/embedding/test", handler.TestEmbeddingModel)
r.POST("/initialization/rerank/check", handler.CheckRerankModel)
r.POST("/initialization/asr/check", handler.CheckASRModel)
r.POST("/initialization/multimodal/test", handler.TestMultimodalFunction)
r.POST("/initialization/extract/text-relation", handler.ExtractTextRelations)