feat(video): update ASR terminology and file type labels for improved clarity

- Changed 'Audio Processing' to 'Audio & Video' in English, Korean, Russian, and Chinese translations to reflect expanded functionality.
- Updated ASR configuration titles and descriptions to include video processing capabilities, enhancing user understanding of the feature.
- Renamed file type labels from 'Audio Files' to 'Audio & Video' across multiple languages for consistency and clarity.
- Enhanced document upload handling in the KnowledgeBase to filter unsupported file types, providing user feedback for filtered images, videos, and audio files.

This update improves the internationalization of the application and clarifies the capabilities of the ASR feature.
This commit is contained in:
wizardchen
2026-04-14 22:27:31 +08:00
committed by lyingbug
parent cb97489787
commit 0fcab3a413
23 changed files with 635 additions and 291 deletions
+8 -7
View File
@@ -1516,7 +1516,7 @@ export default {
faq: 'FAQ Settings',
graph: 'Knowledge Graph',
multimodal: 'Image Processing',
asr: 'Audio Processing',
asr: 'Audio & Video',
datasource: 'Data Sources',
share: 'Sharing'
},
@@ -1724,12 +1724,13 @@ export default {
description: 'Configure image content understanding for parsing and retrieving non-text content like images',
},
asr: {
title: 'Audio Processing Configuration',
description: 'Configure speech recognition to upload audio files (mp3, wav, m4a, flac, ogg) and automatically transcribe to text',
label: 'Enable Speech Recognition',
desc: 'When enabled, audio files can be uploaded to the knowledge base and automatically transcribed to text',
title: 'Audio & Video Speech Recognition',
description:
'Configure ASR (speech-to-text). When enabled, you can upload audio files or videos; audio is transcribed directly, and video audio tracks are extracted and transcribed (e.g. mp3, wav, m4a, flac, ogg, mp4, mov, …).',
label: 'Enable audio & video speech recognition',
desc: 'When enabled, audio or video can be uploaded to the knowledge base; speech is transcribed to text for parsing and retrieval.',
modelLabel: 'ASR Model',
modelDescription: 'Speech-to-text model for audio transcription (e.g. OpenAI Whisper)',
modelDescription: 'Speech-to-text model for audio and video soundtracks (e.g. OpenAI Whisper)',
modelPlaceholder: 'Select an ASR model',
},
advanced: {
@@ -3179,7 +3180,7 @@ export default {
fileTypeText: 'Plain Text',
fileTypeJson: 'JSON Files',
fileTypeImage: 'Images',
fileTypeAudio: 'Audio Files',
fileTypeAudiovisual: 'Audio & Video',
engines: {
builtin: {
name: 'Built-in',
+8 -7
View File
@@ -2037,7 +2037,7 @@ export default {
faq: "FAQ 설정",
graph: "지식 그래프",
multimodal: "이미지 처리",
asr: "오디오 처리",
asr: "오디오·영상",
share: "공유관리",
storage: "스토리지 엔진",
datasource: "데이터 소스",
@@ -2256,12 +2256,13 @@ export default {
description: "이미지 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다",
},
asr: {
title: "오디오 처리 설정",
description: "음성 인식 기능을 설정하여 오디오 파일(mp3, wav, m4a, flac, ogg)을 업로드하고 자동으로 텍스트로 전사합니다",
label: "음성 인식 활성화",
desc: "활성화하면 오디오 파일을 지식베이스에 업로드하고 자동으로 텍스트로 전사할 수 있습니다",
title: "오디오·영상 음성 인식",
description:
"ASR(음성-텍스트)을 설정합니다. 활성화 시 오디오 또는 영상을 업로드할 수 있으며, 오디오는 그대로 전사하고 영상은 음성 트랙을 추출해 텍스트로 전사합니다(mp3, wav, m4a, flac, ogg, mp4, mov 등).",
label: "오디오·영상 음성 인식 사용",
desc: "활성화하면 오디오 또는 영상을 지식베이스에 업로드할 수 있으며, 음성이 텍스트로 전사되어 파싱·검색에 사용됩니다",
modelLabel: "ASR 모델",
modelDescription: "오디오 전사를 위한 음성 인식 모델 (예: OpenAI Whisper)",
modelDescription: "오디오·영상의 음성을 텍스트로 전사하는 모델 (예: OpenAI Whisper)",
modelPlaceholder: "ASR 모델을 선택해주세요",
},
advanced: {
@@ -3234,7 +3235,7 @@ export default {
fileTypeText: '일반 텍스트',
fileTypeJson: 'JSON 파일',
fileTypeImage: '이미지',
fileTypeAudio: '오디오 파일',
fileTypeAudiovisual: '오디오·영상',
engines: {
builtin: {
name: '내장',
+8 -7
View File
@@ -1717,7 +1717,7 @@ export default {
faq: 'FAQ настройки',
graph: 'Граф знаний',
multimodal: 'Обработка изображений',
asr: 'Обработка аудио',
asr: 'Аудио и видео',
storage: 'Storage Engine',
datasource: 'Источники данных',
share: 'Sharing'
@@ -1916,12 +1916,13 @@ export default {
description: 'Настройте понимание изображений для парсинга и поиска нетекстового контента',
},
asr: {
title: 'Обработка аудио',
description: 'Настройте распознавание речи для загрузки аудиофайлов (mp3, wav, m4a, flac, ogg) и автоматической транскрибации в текст',
label: 'Включить распознавание речи',
desc: 'При включении аудиофайлы могут быть загружены в базу знаний и автоматически транскрибированы в текст',
title: 'Распознавание речи для аудио и видео',
description:
'Настройте ASR (речь → текст). При включении можно загружать аудио или видео: аудио транскрибируется целиком, из видео извлекается дорожка и транскрибируется (mp3, wav, m4a, flac, ogg, mp4, mov и др.).',
label: 'Включить распознавание для аудио и видео',
desc: 'При включении в базу можно загружать аудио или видео; речь транскрибируется в текст для парсинга и поиска',
modelLabel: 'Модель ASR',
modelDescription: 'Модель распознавания речи для транскрибации аудио (например, OpenAI Whisper)',
modelDescription: 'Модель распознавания речи для транскрибации аудио и звуковых дорожек видео (например, OpenAI Whisper)',
modelPlaceholder: 'Выберите модель ASR',
},
advanced: {
@@ -2849,7 +2850,7 @@ export default {
fileTypeText: 'Текстовые файлы',
fileTypeJson: 'Файлы JSON',
fileTypeImage: 'Изображения',
fileTypeAudio: 'Аудиофайлы',
fileTypeAudiovisual: 'Аудио и видео',
engines: {
builtin: {
name: 'Встроенный',
+7 -7
View File
@@ -2023,7 +2023,7 @@ export default {
faq: "FAQ 设置",
graph: "知识图谱",
multimodal: "图像处理",
asr: "音频处理",
asr: "音视频",
datasource: "数据源",
share: "共享管理",
},
@@ -2228,12 +2228,12 @@ export default {
description: "配置图像内容理解能力,启用后支持图片等非文本内容的解析和检索",
},
asr: {
title: "音频处理配置",
description: "配置语音识别功能,启用后支持上传音频文件(mp3、wav、m4a、flac、ogg)并自动转录为文本",
label: "启用语音识别",
desc: "启用后可以上传音频文件到知识库,系统将自动转录为文本进行处理",
title: "音视频语音识别",
description: "配置语音识别(ASR),启用后支持上传音频及视频文件;音频将整段转写,视频将提取音轨并转写为文本(常见格式:mp3、wav、m4a、flac、ogg、mp4、mov 等)",
label: "启用音视频语音识别",
desc: "启用后可上传音频或视频到知识库,系统自动转写音轨/语音为文本并参与解析与检索",
modelLabel: "ASR 模型",
modelDescription: "用于语音转文本的语音识别模型(如 OpenAI Whisper)",
modelDescription: "用于音频与视频中语音转文本的识别模型(如 OpenAI Whisper)",
modelPlaceholder: "请选择 ASR 模型",
},
advanced: {
@@ -3177,7 +3177,7 @@ export default {
fileTypeText: "纯文本",
fileTypeJson: "JSON 文件",
fileTypeImage: "图片",
fileTypeAudio: "音频文件",
fileTypeAudiovisual: "音视频",
engines: {
builtin: {
name: "内置",
@@ -1030,11 +1030,38 @@ const handleDocumentUpload = async (event: Event) => {
return;
}
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
const asrEnabled = kbInfo.value?.asr_config?.enabled || false;
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
const validFiles: File[] = [];
let skippedCount = 0;
let imageFilteredCount = 0;
let videoFilteredCount = 0;
let audioFilteredCount = 0;
for (let i = 0; i < files.length; i++) {
const file = files[i];
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'];
const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
if (!vlmEnabled) {
if (imageTypes.includes(fileExt)) {
imageFilteredCount++;
continue;
}
if (videoTypes.includes(fileExt)) {
videoFilteredCount++;
continue;
}
}
if (!asrEnabled && audioTypes.includes(fileExt)) {
audioFilteredCount++;
continue;
}
if (!kbFileTypeVerification(file, files.length > 1, dynamicTypes)) {
validFiles.push(file);
} else {
@@ -1042,6 +1069,16 @@ const handleDocumentUpload = async (event: Event) => {
}
}
if (imageFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
}
if (videoFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
}
if (audioFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
}
if (validFiles.length === 0) {
if (skippedCount > 0) {
MessagePlugin.warning(t('knowledgeBase.allFilesSkippedNoEngine'));
@@ -1176,6 +1213,16 @@ const handleFolderUpload = async (event: Event) => {
}
}
if (imageFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
}
if (videoFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
}
if (audioFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
}
if (validFiles.length === 0) {
MessagePlugin.warning(t('knowledgeBase.noValidFilesInFolder', { total: files.length }));
if (input) input.value = '';
@@ -202,7 +202,7 @@
</div>
</div>
<!-- 音频处理(ASR)设置 -->
<!-- 音视频语音识别(ASR)设置 -->
<div v-if="!isFAQ" v-show="currentSection === 'asr'" class="section">
<div v-if="formData" class="kb-multimodal-settings">
<div class="section-header">
@@ -160,6 +160,9 @@ const fileTypeGroups = computed(() => {
const jsonExts = ['json'].filter(e => ft.has(e))
const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e))
const audioExts = ['mp3', 'wav', 'm4a', 'flac', 'ogg'].filter(e => ft.has(e))
const videoExts = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'].filter(e => ft.has(e))
const mediaExtSet = new Set<string>([...audioExts, ...videoExts])
const audiovisualExts = [...mediaExtSet]
if (pdfExts.length) groups.push({ key: 'pdf', label: t('kbSettings.parser.fileTypePdf'), icon: 'file-pdf', extensions: pdfExts })
if (officeExts.length) groups.push({ key: 'office', label: t('kbSettings.parser.fileTypeWord'), icon: 'file-word', extensions: officeExts })
@@ -170,7 +173,14 @@ const fileTypeGroups = computed(() => {
if (txtExts.length) groups.push({ key: 'text', label: t('kbSettings.parser.fileTypeText'), icon: 'file', extensions: txtExts })
if (jsonExts.length) groups.push({ key: 'json', label: t('kbSettings.parser.fileTypeJson'), icon: 'file-code', extensions: jsonExts })
if (imageExts.length) groups.push({ key: 'image', label: t('kbSettings.parser.fileTypeImage'), icon: 'image', extensions: imageExts })
if (audioExts.length) groups.push({ key: 'audio', label: t('kbSettings.parser.fileTypeAudio'), icon: 'sound', extensions: audioExts })
if (audiovisualExts.length) {
groups.push({
key: 'audiovisual',
label: t('kbSettings.parser.fileTypeAudiovisual'),
icon: 'sound',
extensions: audiovisualExts,
})
}
return groups
})
@@ -128,7 +128,9 @@ func (s *agentService) CreateAgentEngine(
// reliably support images in tool role messages across providers).
if config.VLMModelID != "" {
if vlmModel, err := s.modelService.GetVLMModel(ctx, config.VLMModelID); err == nil {
engine.SetImageDescriber(vlmModel.Predict)
engine.SetImageDescriber(func(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
return vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
})
logger.Infof(ctx, "VLM image describer set for MCP tool result analysis (model: %s)", config.VLMModelID)
} else {
logger.Warnf(ctx, "Failed to load VLM model %s for MCP image fallback: %v", config.VLMModelID, err)
@@ -133,7 +133,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
if payload.EnableOCR {
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmOCRPrompt)
if ocrErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
} else {
@@ -147,7 +147,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
if payload.EnableCaption {
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
if capErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
} else if caption != "" {
+27 -29
View File
@@ -1490,6 +1490,25 @@ type ProcessChunksOptions struct {
ParentChunks []types.ParsedParentChunk
}
// buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase.
func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig {
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
Separators: kb.ChunkingConfig.Separators,
}
if chunkCfg.ChunkSize <= 0 {
chunkCfg.ChunkSize = 512
}
if chunkCfg.ChunkOverlap <= 0 {
chunkCfg.ChunkOverlap = 50
}
if len(chunkCfg.Separators) == 0 {
chunkCfg.Separators = []string{"\n\n", "\n", "。"}
}
return chunkCfg
}
// buildParentChildConfigs derives parent and child SplitterConfig from ChunkingConfig.
// The base config (already validated with defaults) is used for separators.
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
@@ -7089,20 +7108,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
}
// Manual content is markdown - chunk directly with Go chunker
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
Separators: kb.ChunkingConfig.Separators,
}
if chunkCfg.ChunkSize <= 0 {
chunkCfg.ChunkSize = 512
}
if chunkCfg.ChunkOverlap <= 0 {
chunkCfg.ChunkOverlap = 50
}
if len(chunkCfg.Separators) == 0 {
chunkCfg.Separators = []string{"\n\n", "\n", "。"}
}
chunkCfg := buildSplitterConfig(kb)
var parsed []types.ParsedChunk
opts := ProcessChunksOptions{
@@ -7847,7 +7853,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
transcribedText, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
if err != nil {
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
if isLastRetry {
@@ -7859,6 +7865,11 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return fmt.Errorf("audio transcription failed: %w", err)
}
var transcribedText string
if transcriptionResult != nil {
transcribedText = transcriptionResult.Text
}
if transcribedText == "" {
logger.Warn(ctx, "[ASR] Transcription returned empty text")
transcribedText = "[No speech detected in audio file]"
@@ -7902,20 +7913,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
// Step 3: Split into chunks using Go chunker
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
Separators: kb.ChunkingConfig.Separators,
}
if chunkCfg.ChunkSize <= 0 {
chunkCfg.ChunkSize = 512
}
if chunkCfg.ChunkOverlap <= 0 {
chunkCfg.ChunkOverlap = 50
}
if len(chunkCfg.Separators) == 0 {
chunkCfg.Separators = []string{"\n\n", "\n", "。"}
}
chunkCfg := buildSplitterConfig(kb)
processOpts := ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration,
+369 -175
View File
@@ -6,13 +6,18 @@ import (
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/video"
"github.com/Tencent/WeKnora/internal/models/vlm"
@@ -24,17 +29,42 @@ import (
)
const (
vlmFramePrompt = "Analyze this video frame and provide a detailed description in Chinese. Requirements:\n" +
"1. Describe visible objects, people, and their actions\n" +
"2. Describe the scene and environment\n" +
"3. Extract and describe any text on the screen\n" +
"4. Note any notable events or changes\n" +
"5. Be concise but comprehensive"
vlmSummaryPrompt = "Summarize the following video frame descriptions into a coherent video summary in Chinese. Requirements:\n" +
"1. The descriptions are ordered chronologically by timestamp\n" +
"2. Focus on the main storyline and key events\n" +
"3. Maintain the temporal flow of the video\n" +
"4. Be concise but comprehensive"
vlmBatchPromptTemplate = `Analyze this sequence of {{count}} video frames and provide a detailed description in {{language}}.
{{previous_batch_instruction}}
<Requirements>
1. The images are provided in chronological order.
2. Describe visible objects, people, and their actions.
3. Describe the scene and environment.
4. Extract and describe any text on the screen.
5. Note any notable events or changes across these frames.
6. Be concise but comprehensive, keeping the word count under 150 words.
7. The output must be in {{language}}.
</Requirements>`
vlmFirstBatchInstruction = "<PreviousContext>\nNone. These are the first frames of the video. Describe them completely.\n</PreviousContext>"
vlmNextBatchInstruction = "<PreviousContext>\n{{previous_description}}\n</PreviousContext>\n\nThese are the subsequent frames. Focus ONLY on what has changed or what is new compared to the previous context description provided above. If there are no significant changes, just state 'No significant changes'."
vlmSummarySystemPromptTemplate = `Please summarize the provided video frame descriptions into a coherent video summary in {{language}}.
<Requirements>
1. The descriptions are ordered chronologically by timestamp. Please maintain this temporal flow.
2. Focus on the main storyline, key events, and the interactions between people and objects.
3. Identify and include any important text or dialogue that appears on screen.
4. Provide a clear, structured summary that captures the essence of the video.
5. Be concise but comprehensive, avoiding redundant descriptions of the same scene.
6. Keep the summary under 1000 words.
7. The output must be in {{language}}.
</Requirements>`
vlmSummaryUserPromptTemplate = `Here are the video frame descriptions:
<Input>
{{descriptions}}
</Input>
Please provide the summary based on the requirements and input.`
)
// VideoMultimodalService handles video:multimodal asynq tasks.
@@ -100,93 +130,268 @@ func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
return nil
}
var frames []*video.Frame
var extractor video.Extractor
if payload.EnableVLM {
var extErr error
var extErr error
if payload.EnableVLM || payload.EnableASR {
extractor, extErr = video.NewExtractor(nil)
if extErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled: %v", extErr)
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
frames = []*video.Frame{}
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
}
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable: %v", extErr)
}
} else {
logger.Infof(ctx, "[VideoMultimodal] VLM disabled, skipping frame extraction")
frames = []*video.Frame{}
}
frameDescriptions := make([]string, 0, len(frames))
var wg sync.WaitGroup
type FrameAnalysis struct {
StartTimestamp float64
EndTimestamp float64
Description string
}
var frameAnalyses []FrameAnalysis
var asrSegments []asr.Segment
var mu sync.Mutex // Mutex to protect concurrent access if needed
lang := payload.Language
if lang == "" {
lang = "zh-CN"
}
batchPrompt := strings.ReplaceAll(vlmBatchPromptTemplate, "{{language}}", lang)
if payload.EnableVLM {
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
for i, frame := range frames {
desc, descErr := vlmMdl.Predict(ctx, frame.ImageData, vlmFramePrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frame %d at %.2fs: %v",
i, frame.Timestamp, descErr)
continue
wg.Add(1)
go func() {
defer wg.Done()
var frames []*video.Frame
if extractor == nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled")
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
} else {
logger.Debugf(ctx, "[VideoMultimodal] Successfully extracted %d frames from video", len(frames))
}
frameDesc := fmt.Sprintf("[%.2fs] %s", frame.Timestamp, desc)
frameDescriptions = append(frameDescriptions, frameDesc)
logger.Infof(ctx, "[VideoMultimodal] Frame %d analyzed at %.2fs, desc len=%d",
i, frame.Timestamp, len(desc))
}
}
if len(frames) == 0 {
return
}
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
// Initialize the slice within the goroutine before appending
localAnalyses := make([]FrameAnalysis, 0)
var previousDescription string
// Batch size configuration (number of frames per API call)
batchSize := 3
for i := 0; i < len(frames); i += batchSize {
end := i + batchSize
if end > len(frames) {
end = len(frames)
}
batchFrames := frames[i:end]
var batchImages [][]byte
for _, f := range batchFrames {
batchImages = append(batchImages, f.ImageData)
}
startTs := batchFrames[0].Timestamp
endTs := batchFrames[len(batchFrames)-1].Timestamp
logger.Debugf(ctx, "[VideoMultimodal] Starting VLM analysis for frames %d-%d/%d (%.2fs - %.2fs)", i+1, end, len(frames), startTs, endTs)
currentPrompt := strings.ReplaceAll(batchPrompt, "{{count}}", fmt.Sprintf("%d", len(batchFrames)))
if i == 0 {
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", vlmFirstBatchInstruction)
} else {
nextInstruction := strings.ReplaceAll(vlmNextBatchInstruction, "{{previous_description}}", previousDescription)
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", nextInstruction)
}
desc, descErr := vlmMdl.Predict(ctx, batchImages, currentPrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frames %d-%d at %.2fs - %.2fs: %v",
i+1, end, startTs, endTs, descErr)
continue
}
// Update previous description for the next iteration if there are changes
if !strings.Contains(strings.ToLower(desc), "no significant changes") {
previousDescription = desc
}
localAnalyses = append(localAnalyses, FrameAnalysis{
StartTimestamp: startTs,
EndTimestamp: endTs,
Description: desc,
})
logger.Infof(ctx, "[VideoMultimodal] Frames %d-%d/%d analyzed (%.2fs - %.2fs), desc len=%d",
i+1, end, len(frames), startTs, endTs, len(desc))
}
mu.Lock()
frameAnalyses = localAnalyses
mu.Unlock()
}
}()
}
var asrText string
if payload.EnableASR {
asrText, err = s.transcribeAudio(ctx, videoBytes, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: %v", err)
} else if asrText != "" {
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed, len=%d", len(asrText))
}
wg.Add(1)
go func() {
defer wg.Done()
audioBytes := videoBytes
fileName := "video.mp4"
if extractor != nil {
extractedAudio, err := extractor.ExtractAudio(ctx, videoBytes)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Audio extraction failed, falling back to original video: %v", err)
} else if len(extractedAudio) == 0 {
logger.Infof(ctx, "[VideoMultimodal] No audio stream found in video, skipping ASR transcription")
return
} else {
audioBytes = extractedAudio
fileName = "audio.mp3"
logger.Debugf(ctx, "[VideoMultimodal] Extracted audio track for ASR, size: %d bytes", len(audioBytes))
}
}
asrRes, asrErr := s.transcribeAudio(ctx, audioBytes, payload.KnowledgeBaseID, fileName)
if asrErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: kb=%s file=%s audioBytes=%d err=%v",
payload.KnowledgeBaseID, fileName, len(audioBytes), asrErr)
} else if asrRes == nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR returned nil result without error: kb=%s file=%s audioBytes=%d",
payload.KnowledgeBaseID, fileName, len(audioBytes))
} else {
asrSegments = asrRes.Segments
nSeg := len(asrRes.Segments)
nRunes := utf8.RuneCountInString(asrRes.Text)
var spanStart, spanEnd float64
if nSeg > 0 {
spanStart = asrRes.Segments[0].Start
spanEnd = asrRes.Segments[0].End
for _, seg := range asrRes.Segments[1:] {
if seg.Start < spanStart {
spanStart = seg.Start
}
if seg.End > spanEnd {
spanEnd = seg.End
}
}
}
const previewRunes = 200
preview := strings.TrimSpace(asrRes.Text)
if utf8.RuneCountInString(preview) > previewRunes {
preview = string([]rune(preview)[:previewRunes]) + "..."
}
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed: kb=%s file=%s audioBytes=%d textRunes=%d segments=%d timeline=%.2fs-%.2fs preview=%q",
payload.KnowledgeBaseID, fileName, len(audioBytes), nRunes, nSeg, spanStart, spanEnd, preview)
}
}()
}
logger.Debugf(ctx, "[VideoMultimodal] Waiting for VLM and ASR tasks to complete...")
wg.Wait()
logger.Debugf(ctx, "[VideoMultimodal] VLM and ASR tasks completed")
var videoSummary string
if len(frameDescriptions) > 0 && payload.EnableVLM {
vlmMdl, _ := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmMdl != nil {
combinedInput := strings.Join(frameDescriptions, "\n")
summary, sumErr := vlmMdl.Predict(ctx, []byte(combinedInput), vlmSummaryPrompt)
if sumErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
var combinedInput string
var kb *types.KnowledgeBase
if (len(frameAnalyses) > 0 || len(asrSegments) > 0) && (payload.EnableVLM || payload.EnableASR) {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
type TimelineEvent struct {
Start float64
End float64
Text string
Type string
}
var events []TimelineEvent
for _, f := range frameAnalyses {
events = append(events, TimelineEvent{Start: f.StartTimestamp, End: f.EndTimestamp, Text: f.Description, Type: "frame"})
}
for _, seg := range asrSegments {
events = append(events, TimelineEvent{Start: seg.Start, End: seg.End, Text: seg.Text, Type: "audio"})
}
sort.Slice(events, func(i, j int) bool {
return events[i].Start < events[j].Start
})
var combinedBuilder strings.Builder
for _, ev := range events {
if ev.Type == "frame" {
if ev.Start == ev.End {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs] Visual: %s\n", ev.Start, ev.Text))
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Visual: %s\n", ev.Start, ev.End, ev.Text))
}
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Audio: %s\n", ev.Start, ev.End, ev.Text))
}
}
combinedInput = combinedBuilder.String()
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for summary: %v", err)
videoSummary = combinedInput
} else {
chatMdl, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil || chatMdl == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get chat model for summary (modelID=%s): %v", kb.SummaryModelID, err)
videoSummary = combinedInput
} else {
videoSummary = summary
// Limit combinedInput to avoid context length overflow.
// Using ~32000 runes to be safe for typical models.
if utf8.RuneCountInString(combinedInput) > 32000 {
runes := []rune(combinedInput)
combinedInput = string(runes[:32000]) + "\n...[truncated due to length]..."
}
systemPrompt := strings.ReplaceAll(vlmSummarySystemPromptTemplate, "{{language}}", lang)
userPrompt := strings.ReplaceAll(vlmSummaryUserPromptTemplate, "{{descriptions}}", combinedInput)
logger.Debugf(ctx, "[VideoMultimodal] Generating video summary with model %s, input length: %d chars", kb.SummaryModelID, len(combinedInput))
resp, sumErr := chatMdl.Chat(ctx, []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
}, nil)
if sumErr != nil || resp == nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
videoSummary = combinedInput
} else {
videoSummary = resp.Content
}
}
} else {
videoSummary = strings.Join(frameDescriptions, "\n")
}
}
if kb == nil {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for video chunking: %v", err)
}
}
videoInfo := types.VideoInfo{
URL: payload.VideoURL,
FrameCount: len(frameDescriptions),
HasVLMAnalysis: len(frameDescriptions) > 0,
HasASR: asrText != "",
VideoSummary: videoSummary,
ASRText: asrText,
FrameDescriptions: frameDescriptions,
URL: payload.VideoURL,
}
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
newChunks := s.buildVideoChunks(payload, frameDescriptions, videoSummary, asrText, string(videoInfoJSON))
newChunks := s.buildVideoChunks(payload, combinedInput, videoSummary, string(videoInfoJSON), kb)
if len(newChunks) == 0 {
// Even if VLM/ASR both failed, mark knowledge as completed
@@ -205,9 +410,6 @@ func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's VideoInfo (mirrors old docreader behaviour)
s.updateParentChunkVideoInfo(ctx, payload, videoInfo)
// For standalone video files, use summary as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
@@ -267,78 +469,115 @@ func downloadVideoFromURL(videoURL string) ([]byte, error) {
return secutils.DownloadBytes(videoURL)
}
// transcribeAudio extracts and transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, videoBytes []byte, kbID string) (string, error) {
// transcribeAudio transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, audioBytes []byte, kbID string, fileName string) (*asr.TranscriptionResult, error) {
asrMdl, err := s.resolveASR(ctx, kbID)
if err != nil {
return "", fmt.Errorf("resolve ASR: %w", err)
return nil, fmt.Errorf("resolve ASR: %w", err)
}
return asrMdl.Transcribe(ctx, videoBytes, "video.mp4")
return asrMdl.Transcribe(ctx, audioBytes, fileName)
}
// buildVideoChunks creates child chunks for video multimodal analysis results:
// - Frame descriptions (one chunk per frame)
// - Video summary (single chunk)
// - ASR transcription (single chunk)
// - Video summary (split into multiple chunks if too long)
// - Timeline combined input (split into multiple chunks if too long)
func (s *VideoMultimodalService) buildVideoChunks(
payload types.VideoMultimodalPayload,
frameDescriptions []string,
combinedInput string,
videoSummary string,
asrText string,
videoInfoJSON string,
kb *types.KnowledgeBase,
) []*types.Chunk {
var newChunks []*types.Chunk
for _, desc := range frameDescriptions {
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: desc,
ChunkType: types.ChunkTypeVideoFrame,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
var parentContentBuilder strings.Builder
if videoSummary != "" {
parentContentBuilder.WriteString("视频摘要:\n")
parentContentBuilder.WriteString(videoSummary)
parentContentBuilder.WriteString("\n\n")
}
if combinedInput != "" {
parentContentBuilder.WriteString("详细时间轴解析:\n")
parentContentBuilder.WriteString(combinedInput)
}
parentContent := strings.TrimSpace(parentContentBuilder.String())
if parentContent == "" {
parentContent = "该视频未解析出有效内容"
}
// Construct a Parent Chunk representing the entire video
parentChunkID := uuid.New().String()
newChunks = append(newChunks, &types.Chunk{
ID: parentChunkID,
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: parentContent,
ChunkType: types.ChunkTypeParentText,
ParentChunkID: payload.ChunkID, // Link back to the original text chunk if extracted from a document
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
var chunkCfg chunker.SplitterConfig
if kb != nil {
baseCfg := buildSplitterConfig(kb)
if kb.ChunkingConfig.EnableParentChild {
// If parent-child is globally enabled, these chunks act as child chunks
// for the newly created video parent chunk.
_, childCfg := buildParentChildConfigs(kb.ChunkingConfig, baseCfg)
chunkCfg = childCfg
} else {
chunkCfg = baseCfg
}
} else {
chunkCfg = chunker.SplitterConfig{
ChunkSize: 512,
ChunkOverlap: 64,
Separators: []string{"\n\n", "\n", "。"},
}
}
if combinedInput != "" {
inputChunks := chunker.SplitText(combinedInput, chunkCfg)
for _, sc := range inputChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
ParentChunkID: parentChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
newChunks = append(newChunks, chunk)
}
if videoSummary != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: videoSummary,
ChunkType: types.ChunkTypeVideoCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if asrText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: asrText,
ChunkType: types.ChunkTypeVideoASR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
summaryChunks := chunker.SplitText(videoSummary, chunkCfg)
for _, sc := range summaryChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
}
return newChunks
@@ -435,51 +674,6 @@ func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
}
// updateParentChunkVideoInfo updates the parent text chunk's VideoInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full video metadata (URL, frame descriptions, summary, ASR).
func (s *VideoMultimodalService) updateParentChunkVideoInfo(
ctx context.Context,
payload types.VideoMultimodalPayload,
videoInfo types.VideoInfo,
) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.VideoInfo
if chunk.VideoInfo != "" {
_ = json.Unmarshal([]byte(chunk.VideoInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == videoInfo.URL {
existingInfos[i] = videoInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, videoInfo)
}
videoInfoJSON, _ := json.Marshal(existingInfos)
chunk.VideoInfo = string(videoInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update parent chunk %s VideoInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Updated parent chunk %s VideoInfo for video", chunk.ID)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
+5 -1
View File
@@ -1893,7 +1893,11 @@ func (h *InitializationHandler) CheckASRModel(c *gin.Context) {
return
}
text, err := asrInstance.Transcribe(ctx, assets.ASRTestWAV, "asr_test.wav")
res, err := asrInstance.Transcribe(ctx, assets.ASRTestWAV, "asr_test.wav")
var text string
if res != nil {
text = res.Text
}
available := true
message := "ASR连接成功"
@@ -209,10 +209,11 @@ func (p *AttachmentProcessor) processAudioFile(
}
logger.Infof(ctx, "starting audio transcription: fileName=%s, size=%d", fileName, len(data))
transcript, err := asrInstance.Transcribe(ctx, data, fileName)
res, err := asrInstance.Transcribe(ctx, data, fileName)
if err != nil {
return fmt.Errorf("audio transcription failed: %w", err)
}
transcript := res.Text
p.applyLineTruncation(ctx, transcript, attachment)
logger.Infof(ctx, "audio transcription done: textLen=%d", len(transcript))
+1 -1
View File
@@ -83,7 +83,7 @@ func (h *Handler) analyzeImageAttachments(ctx context.Context, images []ImageAtt
continue
}
prompt := buildImageAnalysisPrompt(userQuery)
analysis, analysisErr := vlmModel.Predict(ctx, imgBytes, prompt)
analysis, analysisErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
if analysisErr != nil {
logger.Warnf(ctx, "VLM analysis failed for image %d: %v", i, analysisErr)
} else {
+1 -1
View File
@@ -38,7 +38,7 @@ type SplitterConfig struct {
func DefaultConfig() SplitterConfig {
return SplitterConfig{
ChunkSize: 512,
ChunkOverlap: 128,
ChunkOverlap: 64,
Separators: []string{"\n\n", "\n", "。"},
}
}
+15 -2
View File
@@ -6,10 +6,23 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// Segment represents a transcribed segment with timestamps.
type Segment struct {
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
}
// TranscriptionResult holds the full text and its segments.
type TranscriptionResult struct {
Text string `json:"text"`
Segments []Segment `json:"segments,omitempty"`
}
// ASR defines the interface for Automatic Speech Recognition model operations.
type ASR interface {
// Transcribe sends audio bytes to the ASR model and returns the transcribed text.
Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error)
// Transcribe sends audio bytes to the ASR model and returns the transcribed text and segments.
Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error)
GetModelName() string
GetModelID() string
+25 -5
View File
@@ -3,6 +3,7 @@ package asr
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"path/filepath"
@@ -44,9 +45,9 @@ func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
}
// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (string, error) {
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) {
if len(audioBytes) == 0 {
return "", fmt.Errorf("audio bytes are empty")
return nil, fmt.Errorf("audio bytes are empty")
}
// Ensure fileName has an extension for proper MIME type detection
@@ -61,7 +62,7 @@ func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName
Model: s.modelName,
FilePath: fileName,
Reader: bytes.NewReader(audioBytes),
Format: openai.AudioResponseFormatJSON,
Format: openai.AudioResponseFormatVerboseJSON,
}
if s.language != "" {
@@ -70,12 +71,31 @@ func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName
resp, err := s.client.CreateTranscription(ctx, req)
if err != nil {
return "", fmt.Errorf("ASR transcription request failed: %w", err)
return nil, fmt.Errorf("ASR transcription request failed: %w", err)
}
jsonData, err := json.Marshal(resp)
if err != nil {
return nil, fmt.Errorf("marshal transcription response: %w", err)
}
logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData))
text := strings.TrimSpace(resp.Text)
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
return text, nil
var segments []Segment
for _, s := range resp.Segments {
segments = append(segments, Segment{
Start: s.Start,
End: s.End,
Text: strings.TrimSpace(s.Text),
})
}
return &TranscriptionResult{
Text: text,
Segments: segments,
}, nil
}
func (s *OpenAIASR) GetModelName() string { return s.modelName }
+3
View File
@@ -15,6 +15,9 @@ type Extractor interface {
// for each frame's image data. This is useful for streaming processing.
ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error)
// ExtractAudio extracts the audio track from a video file and returns it as bytes.
ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error)
// GetVideoInfo retrieves basic information about a video file.
// It returns a map containing duration, width, height, etc.
GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error)
+44
View File
@@ -247,6 +247,50 @@ func (e *FFMpegExtractor) readExtractedFrames(ctx context.Context, frameDir stri
return frames, nil
}
// ExtractAudio extracts the audio track from a video using FFmpeg.
func (e *FFMpegExtractor) ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error) {
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
tempAudioFile, err := os.CreateTemp(e.tempDir, "audio-output-*.mp3")
if err != nil {
return nil, fmt.Errorf("failed to create temp audio file: %w", err)
}
tempAudioPath := tempAudioFile.Name()
tempAudioFile.Close()
defer os.Remove(tempAudioPath)
cmd := exec.CommandContext(ctx, e.ffmpegPath,
"-i", tempVideoPath,
"-vn",
"-c:a", "libmp3lame",
"-q:a", "2",
"-y",
tempAudioPath,
)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
if strings.Contains(stderr.String(), "Output file does not contain any stream") {
logger.Infof(ctx, "[VideoFFMpeg] Video does not contain an audio stream")
return nil, nil
}
return nil, fmt.Errorf("ffmpeg audio extraction failed: %w, stderr: %s", err, stderr.String())
}
audioBytes, err := os.ReadFile(tempAudioPath)
if err != nil {
return nil, fmt.Errorf("failed to read extracted audio: %w", err)
}
logger.Infof(ctx, "[VideoFFMpeg] Extracted audio, size: %d bytes", len(audioBytes))
return audioBytes, nil
}
// GetVideoInfo returns basic information about a video.
func (e *FFMpegExtractor) GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) {
tempVideoPath, err := e.writeTempVideo(videoBytes)
+14 -3
View File
@@ -29,22 +29,33 @@ func NewOllamaVLM(config *Config, ollamaService *ollama.OllamaService) (*OllamaV
}
// Predict sends an image with a text prompt to the Ollama vision model.
func (v *OllamaVLM) Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
func (v *OllamaVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
streamFlag := false
var images []ollamaapi.ImageData
for _, imgBytes := range imgBytesList {
if len(imgBytes) > 0 {
images = append(images, imgBytes)
}
}
chatReq := &ollamaapi.ChatRequest{
Model: v.modelName,
Messages: []ollamaapi.Message{
{
Role: "user",
Content: prompt,
Images: []ollamaapi.ImageData{imgBytes},
Images: images,
},
},
Stream: &streamFlag,
Options: map[string]interface{}{"temperature": 0.1},
}
logger.Infof(ctx, "[VLM] Calling Ollama API, model=%s, imageSize=%d", v.modelName, len(imgBytes))
totalImageSize := 0
for _, img := range imgBytesList {
totalImageSize += len(img)
}
logger.Infof(ctx, "[VLM] Calling Ollama API, model=%s, numImages=%d, totalImageSize=%d", v.modelName, len(images), totalImageSize)
var result string
err := v.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
+32 -20
View File
@@ -64,37 +64,49 @@ func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
}
// Predict sends an image with a text prompt to the OpenAI-compatible API.
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
mimeType := detectImageMIME(imgBytes)
b64 := base64.StdEncoding.EncodeToString(imgBytes)
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
var parts []openai.ChatMessagePart
// Add text prompt first
parts = append(parts, openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeText,
Text: prompt,
})
// Add images
for _, imgBytes := range imgBytesList {
if len(imgBytes) > 0 {
mimeType := detectImageMIME(imgBytes)
b64 := base64.StdEncoding.EncodeToString(imgBytes)
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
parts = append(parts, openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: dataURI,
Detail: openai.ImageURLDetailAuto,
},
})
}
}
req := openai.ChatCompletionRequest{
Model: v.modelName,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleUser,
MultiContent: []openai.ChatMessagePart{
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: dataURI,
Detail: openai.ImageURLDetailAuto,
},
},
{
Type: openai.ChatMessagePartTypeText,
Text: prompt,
},
},
Role: openai.ChatMessageRoleUser,
MultiContent: parts,
},
},
MaxTokens: defaultMaxToks,
Temperature: defaultTemp,
}
logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, imageSize=%d",
v.modelName, v.baseURL, len(imgBytes))
totalImageSize := 0
for _, img := range imgBytesList {
totalImageSize += len(img)
}
logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, numImages=%d, totalImageSize=%d",
v.modelName, v.baseURL, len(imgBytesList), totalImageSize)
resp, err := v.client.CreateChatCompletion(ctx, req)
if err != nil {
+2 -2
View File
@@ -11,8 +11,8 @@ import (
// VLM defines the interface for Vision Language Model operations.
type VLM interface {
// Predict sends an image with a text prompt to the VLM and returns the generated text.
Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error)
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
GetModelName() string
GetModelID() string
-18
View File
@@ -20,12 +20,6 @@ const (
ChunkTypeImageOCR ChunkType = "image_ocr"
// ChunkTypeImageCaption 表示图片描述的 Chunk
ChunkTypeImageCaption ChunkType = "image_caption"
// ChunkTypeVideoFrame 表示视频帧分析内容的 Chunk
ChunkTypeVideoFrame ChunkType = "video_frame"
// ChunkTypeVideoCaption 表示视频整体描述的 Chunk
ChunkTypeVideoCaption ChunkType = "video_caption"
// ChunkTypeVideoASR 表示视频音频转录的 Chunk
ChunkTypeVideoASR ChunkType = "video_asr"
// ChunkTypeSummary 表示摘要类型的 Chunk
ChunkTypeSummary = "summary"
// ChunkTypeEntity 表示实体类型的 Chunk
@@ -105,18 +99,6 @@ type ImageInfo struct {
type VideoInfo struct {
// 视频URL
URL string `json:"url" gorm:"type:text"`
// 帧数量
FrameCount int `json:"frame_count"`
// 是否有VLM分析
HasVLMAnalysis bool `json:"has_vlm_analysis"`
// 是否有ASR转录
HasASR bool `json:"has_asr"`
// 视频摘要
VideoSummary string `json:"video_summary"`
// ASR转录文本
ASRText string `json:"asr_text"`
// 帧描述列表
FrameDescriptions []string `json:"frame_descriptions"`
}
// Chunk represents a document chunk