From 86d26fa9bfa83ebf9f84472dff2220a3ed1580ef Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 14 Apr 2026 22:57:58 +0800 Subject: [PATCH] feat(video): remove video upload support and update related terminology - Removed support for video file uploads across the application, including updates to the knowledge base and document processing services. - Updated internationalization files to reflect the removal of video-related terminology, changing labels and descriptions to focus solely on audio processing. - Adjusted file type handling to exclude video formats, ensuring users receive accurate feedback regarding supported file types. - Enhanced error handling to provide clear messages when users attempt to upload unsupported video files. This update streamlines the application by focusing on audio processing capabilities and improving user experience through clearer messaging. --- frontend/src/i18n/locales/en-US.ts | 20 +- frontend/src/i18n/locales/ko-KR.ts | 20 +- frontend/src/i18n/locales/ru-RU.ts | 20 +- frontend/src/i18n/locales/zh-CN.ts | 20 +- frontend/src/utils/index.ts | 2 +- .../src/views/knowledge/KnowledgeBase.vue | 31 +- .../knowledge/settings/KBParserSettings.vue | 4 +- internal/application/service/knowledge.go | 81 +- .../application/service/video_multimodal.go | 790 ------------------ internal/container/container.go | 1 - .../docparser/builtin_converter.go | 30 - .../docparser/engine_registry.go | 4 +- internal/models/video/extractor.go | 98 --- internal/models/video/ffmpeg_extractor.go | 342 -------- internal/router/task.go | 4 - internal/types/docparser.go | 2 - internal/types/extract_graph.go | 15 - 17 files changed, 78 insertions(+), 1406 deletions(-) delete mode 100644 internal/application/service/video_multimodal.go delete mode 100644 internal/models/video/extractor.go delete mode 100644 internal/models/video/ffmpeg_extractor.go diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 7534742f..ca6c83b5 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -103,7 +103,7 @@ export default { noValidFilesSelected: 'All selected files are unsupported', hiddenFilesFiltered: 'Filtered {count} hidden files', imagesFilteredNoVLM: 'Filtered {count} image files (VLM not enabled)', - videosFilteredNoVLM: 'Filtered {count} video files (VLM not enabled)', + videosFilteredNoVLM: 'Skipped {count} video file(s) (video upload is not supported)', audiosFilteredNoASR: 'Filtered {count} audio files (ASR not enabled)', invalidFilesFiltered: 'Filtered {count} unsupported files', unsupportedFileType: 'Unsupported file type', @@ -113,7 +113,7 @@ export default { andMoreFiles: '...and {count} more files', duplicateFilesSkipped: '{count} duplicate files skipped', uploadFile: 'Upload File', - uploadFileDesc: 'Supports PDF, Word, TXT, images, videos, audio, etc.', + uploadFileDesc: 'Supports PDF, Word, TXT, images, audio, etc.', importURL: 'Import from URL', addDocument: 'Add Document', importURLDesc: 'Import via URL link', @@ -1516,7 +1516,7 @@ export default { faq: 'FAQ Settings', graph: 'Knowledge Graph', multimodal: 'Image Processing', - asr: 'Audio & Video', + asr: 'Audio', datasource: 'Data Sources', share: 'Sharing' }, @@ -1724,13 +1724,13 @@ export default { description: 'Configure image content understanding for parsing and retrieving non-text content like images', }, asr: { - title: 'Audio & Video Speech Recognition', + title: 'Audio Speech Recognition', description: - 'Configure ASR (speech-to-text). When enabled, you can upload audio files or videos; audio is transcribed directly, and video audio tracks are extracted and transcribed (e.g. mp3, wav, m4a, flac, ogg, mp4, mov, …).', - label: 'Enable audio & video speech recognition', - desc: 'When enabled, audio or video can be uploaded to the knowledge base; speech is transcribed to text for parsing and retrieval.', + 'Configure ASR (speech-to-text). When enabled, you can upload audio files and transcribe them to text (e.g. mp3, wav, m4a, flac, ogg). Video upload is not supported.', + label: 'Enable audio speech recognition', + desc: 'When enabled, audio can be uploaded to the knowledge base; speech is transcribed to text for parsing and retrieval.', modelLabel: 'ASR Model', - modelDescription: 'Speech-to-text model for audio and video soundtracks (e.g. OpenAI Whisper)', + modelDescription: 'Speech-to-text model for audio (e.g. OpenAI Whisper)', modelPlaceholder: 'Select an ASR model', }, advanced: { @@ -1744,7 +1744,7 @@ export default { }, multimodal: { label: 'Multimodal Feature', - description: 'Enable understanding of multimodal content such as images and videos', + description: 'Enable understanding of multimodal content such as images', vllmLabel: 'VLLM Vision Model', vllmDescription: 'Vision-language model required for multimodal understanding', vllmPlaceholder: 'Select a VLLM model (required)', @@ -3180,7 +3180,7 @@ export default { fileTypeText: 'Plain Text', fileTypeJson: 'JSON Files', fileTypeImage: 'Images', - fileTypeAudiovisual: 'Audio & Video', + fileTypeAudiovisual: 'Audio', engines: { builtin: { name: 'Built-in', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 39833a21..20c57c31 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -102,7 +102,7 @@ export default { noValidFilesSelected: "선택한 파일 모두 지원되지 않음", hiddenFilesFiltered: "{count}개 숨김 파일 필터링됨", imagesFilteredNoVLM: "{count}개 이미지 파일 필터링됨(VLM 미활성화)", - videosFilteredNoVLM: "{count}개 동영상 파일 필터링됨(VLM 미활성화)", + videosFilteredNoVLM: "{count}개의 동영상 파일을 건너됨 (동영상 업로드 미지원)", audiosFilteredNoASR: "{count}개 오디오 파일 필터링됨(ASR 미활성화)", invalidFilesFiltered: "{count}개 지원되지 않는 파일 필터링됨", unsupportedFileType: "지원되지 않는 파일 형식", @@ -112,7 +112,7 @@ export default { andMoreFiles: "...외 {count}개 파일", duplicateFilesSkipped: "{count}개 중복 파일 무시됨", uploadFile: "파일 업로드", - uploadFileDesc: "PDF, Word, TXT, 이미지, 동영상, 오디오 등 지원", + uploadFileDesc: "PDF, Word, TXT, 이미지, 오디오 등 지원", importURL: "웹페이지 가져오기", addDocument: "문서 추가", importURLDesc: "URL 링크로 가져오기", @@ -2037,7 +2037,7 @@ export default { faq: "FAQ 설정", graph: "지식 그래프", multimodal: "이미지 처리", - asr: "오디오·영상", + asr: "오디오", share: "공유관리", storage: "스토리지 엔진", datasource: "데이터 소스", @@ -2256,13 +2256,13 @@ export default { description: "이미지 콘텐츠 이해 기능을 설정하여 이미지 등 비텍스트 콘텐츠의 파싱 및 검색을 지원합니다", }, asr: { - title: "오디오·영상 음성 인식", + title: "오디오 음성 인식", description: - "ASR(음성-텍스트)을 설정합니다. 활성화 시 오디오 또는 영상을 업로드할 수 있으며, 오디오는 그대로 전사하고 영상은 음성 트랙을 추출해 텍스트로 전사합니다(mp3, wav, m4a, flac, ogg, mp4, mov 등).", - label: "오디오·영상 음성 인식 사용", - desc: "활성화하면 오디오 또는 영상을 지식베이스에 업로드할 수 있으며, 음성이 텍스트로 전사되어 파싱·검색에 사용됩니다", + "ASR(음성-텍스트)을 설정합니다. 활성화 시 오디오 파일을 업로드해 텍스트로 전사할 수 있습니다(mp3, wav, m4a, flac, ogg 등). 동영상 업로드는 지원하지 않습니다.", + label: "오디오 음성 인식 사용", + desc: "활성화하면 오디오를 지식베이스에 업로드할 수 있으며, 음성이 텍스트로 전사되어 파싱·검색에 사용됩니다", modelLabel: "ASR 모델", - modelDescription: "오디오·영상의 음성을 텍스트로 전사하는 모델 (예: OpenAI Whisper)", + modelDescription: "오디오의 음성을 텍스트로 전사하는 모델 (예: OpenAI Whisper)", modelPlaceholder: "ASR 모델을 선택해주세요", }, advanced: { @@ -2277,7 +2277,7 @@ export default { }, multimodal: { label: "멀티모달 기능", - description: "이미지, 비디오 등 멀티모달 콘텐츠 이해 기능 활성화", + description: "이미지 등 멀티모달 콘텐츠 이해 기능 활성화", vllmLabel: "VLLM 비전 모델", vllmDescription: "멀티모달 이해를 위한 비전 언어 모델 (필수)", vllmPlaceholder: "VLLM 모델을 선택해주세요 (필수)", @@ -3235,7 +3235,7 @@ export default { fileTypeText: '일반 텍스트', fileTypeJson: 'JSON 파일', fileTypeImage: '이미지', - fileTypeAudiovisual: '오디오·영상', + fileTypeAudiovisual: '오디오', engines: { builtin: { name: '내장', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 45a775ad..ef5aa882 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -85,7 +85,7 @@ export default { noValidFilesSelected: 'Все выбранные файлы не поддерживаются', hiddenFilesFiltered: 'Отфильтровано {count} скрытых файлов', imagesFilteredNoVLM: 'Отфильтровано {count} изображений (VLM не включен)', - videosFilteredNoVLM: 'Отфильтровано {count} видеофайлов (VLM не включен)', + videosFilteredNoVLM: 'Пропущено {count} видеофайлов (загрузка видео не поддерживается)', audiosFilteredNoASR: 'Отфильтровано {count} аудиофайлов (ASR не включен)', invalidFilesFiltered: 'Отфильтровано {count} неподдерживаемых файлов', unsupportedFileType: 'Неподдерживаемый тип файла', @@ -95,7 +95,7 @@ export default { andMoreFiles: '...и ещё {count} файлов', duplicateFilesSkipped: 'Пропущено {count} повторяющихся файлов', uploadFile: 'Загрузить файл', - uploadFileDesc: 'Поддерживает PDF, Word, TXT, изображения, видео, аудио и т.д.', + uploadFileDesc: 'Поддерживает PDF, Word, TXT, изображения, аудио и т.д.', importURL: 'Импорт из URL', importURLDesc: 'Импорт по ссылке URL', importURLTitle: 'Импорт из URL', @@ -1717,7 +1717,7 @@ export default { faq: 'FAQ настройки', graph: 'Граф знаний', multimodal: 'Обработка изображений', - asr: 'Аудио и видео', + asr: 'Аудио', storage: 'Storage Engine', datasource: 'Источники данных', share: 'Sharing' @@ -1916,13 +1916,13 @@ export default { description: 'Настройте понимание изображений для парсинга и поиска нетекстового контента', }, asr: { - title: 'Распознавание речи для аудио и видео', + title: 'Распознавание речи для аудио', description: - 'Настройте ASR (речь → текст). При включении можно загружать аудио или видео: аудио транскрибируется целиком, из видео извлекается дорожка и транскрибируется (mp3, wav, m4a, flac, ogg, mp4, mov и др.).', - label: 'Включить распознавание для аудио и видео', - desc: 'При включении в базу можно загружать аудио или видео; речь транскрибируется в текст для парсинга и поиска', + 'Настройте ASR (речь → текст). При включении можно загружать аудиофайлы и транскрибировать их в текст (mp3, wav, m4a, flac, ogg и др.). Загрузка видео не поддерживается.', + label: 'Включить распознавание речи для аудио', + desc: 'При включении в базу можно загружать аудио; речь транскрибируется в текст для парсинга и поиска', modelLabel: 'Модель ASR', - modelDescription: 'Модель распознавания речи для транскрибации аудио и звуковых дорожек видео (например, OpenAI Whisper)', + modelDescription: 'Модель распознавания речи для транскрибации аудио (например, OpenAI Whisper)', modelPlaceholder: 'Выберите модель ASR', }, advanced: { @@ -1936,7 +1936,7 @@ export default { }, multimodal: { label: 'Мультимодальная функция', - description: 'Включите понимание мультимедийного контента, такого как изображения и видео', + description: 'Включите понимание мультимедийного контента, такого как изображения', vllmLabel: 'VLLM модель для зрения', vllmDescription: 'Визуально-языковая модель, необходимая для мультимодального понимания', vllmPlaceholder: 'Выберите VLLM модель (обязательно)', @@ -2850,7 +2850,7 @@ export default { fileTypeText: 'Текстовые файлы', fileTypeJson: 'Файлы JSON', fileTypeImage: 'Изображения', - fileTypeAudiovisual: 'Аудио и видео', + fileTypeAudiovisual: 'Аудио', engines: { builtin: { name: 'Встроенный', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index bc141af7..e092bca4 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -102,7 +102,7 @@ export default { noValidFilesSelected: "选中的文件均不支持", hiddenFilesFiltered: "已过滤 {count} 个隐藏文件", imagesFilteredNoVLM: "已过滤 {count} 个图片文件(未启用VLM)", - videosFilteredNoVLM: "已过滤 {count} 个视频文件(未启用VLM)", + videosFilteredNoVLM: "已跳过 {count} 个视频文件（暂不支持视频上传）", audiosFilteredNoASR: "已过滤 {count} 个音频文件(未启用ASR)", invalidFilesFiltered: "已过滤 {count} 个不支持的文件", unsupportedFileType: "不支持的文件格式", @@ -112,7 +112,7 @@ export default { andMoreFiles: "...及其他 {count} 个文件", duplicateFilesSkipped: "已忽略 {count} 个重复文件", uploadFile: "上传文件", - uploadFileDesc: "支持 PDF、Word、TXT、图片、视频、音频等", + uploadFileDesc: "支持 PDF、Word、TXT、图片、音频等", importURL: "导入网页", addDocument: "添加文档", importURLDesc: "通过URL链接导入", @@ -2023,7 +2023,7 @@ export default { faq: "FAQ 设置", graph: "知识图谱", multimodal: "图像处理", - asr: "音视频", + asr: "音频处理", datasource: "数据源", share: "共享管理", }, @@ -2228,12 +2228,12 @@ export default { description: "配置图像内容理解能力，启用后支持图片等非文本内容的解析和检索", }, asr: { - title: "音视频语音识别", - description: "配置语音识别（ASR），启用后支持上传音频及视频文件；音频将整段转写，视频将提取音轨并转写为文本（常见格式：mp3、wav、m4a、flac、ogg、mp4、mov 等）", - label: "启用音视频语音识别", - desc: "启用后可上传音频或视频到知识库，系统自动转写音轨/语音为文本并参与解析与检索", + title: "音频语音识别", + description: "配置语音识别（ASR），启用后可上传音频文件并整段转写为文本（常见格式：mp3、wav、m4a、flac、ogg 等）。暂不支持视频上传。", + label: "启用音频语音识别", + desc: "启用后可上传音频到知识库，系统自动将语音转写为文本并参与解析与检索", modelLabel: "ASR 模型", - modelDescription: "用于音频与视频中语音转文本的识别模型（如 OpenAI Whisper）", + modelDescription: "用于音频中语音转文本的识别模型（如 OpenAI Whisper）", modelPlaceholder: "请选择 ASR 模型", }, advanced: { @@ -2247,7 +2247,7 @@ export default { }, multimodal: { label: "多模态功能", - description: "启用图片、视频等多模态内容的理解能力", + description: "启用图片等多模态内容的理解能力", vllmLabel: "VLLM 视觉模型", vllmDescription: "用于多模态理解的视觉语言模型（必选）", vllmPlaceholder: "请选择 VLLM 模型（必选）", @@ -3177,7 +3177,7 @@ export default { fileTypeText: "纯文本", fileTypeJson: "JSON 文件", fileTypeImage: "图片", - fileTypeAudiovisual: "音视频", + fileTypeAudiovisual: "音频", engines: { builtin: { name: "内置", diff --git a/frontend/src/utils/index.ts b/frontend/src/utils/index.ts index ccc3f83c..fe44bc16 100644 --- a/frontend/src/utils/index.ts +++ b/frontend/src/utils/index.ts @@ -40,7 +40,7 @@ export function formatStringDate(date: any) { year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second ); } -const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "jpg", "jpeg", "png", "csv", "xlsx", "xls", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv", "mp3", "wav", "m4a", "flac", "ogg"]); +const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "jpg", "jpeg", "png", "csv", "xlsx", "xls", "mp3", "wav", "m4a", "flac", "ogg"]); /** * Returns true when the file should be **rejected**. diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index c6ed91ee..5e347eba 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -211,13 +211,6 @@ const fileTypeOptions = computed(() => [ { content: 'MD', value: 'md' }, { content: 'URL', value: 'url' }, { content: t('knowledgeBase.typeManual'), value: 'manual' }, - { content: 'MP4', value: 'mp4' }, - { content: 'MOV', value: 'mov' }, - { content: 'AVI', value: 'avi' }, - { content: 'MKV', value: 'mkv' }, - { content: 'WEBM', value: 'webm' }, - { content: 'WMV', value: 'wmv' }, - { content: 'FLV', value: 'flv' }, { content: 'MP3', value: 'mp3' }, { content: 'WAV', value: 'wav' }, { content: 'M4A', value: 'm4a' }, @@ -1046,15 +1039,16 @@ const handleDocumentUpload = async (event: Event) => { const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv']; const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg']; + if (videoTypes.includes(fileExt)) { + videoFilteredCount++; + continue; + } + if (!vlmEnabled) { if (imageTypes.includes(fileExt)) { imageFilteredCount++; continue; } - if (videoTypes.includes(fileExt)) { - videoFilteredCount++; - continue; - } } if (!asrEnabled && audioTypes.includes(fileExt)) { @@ -1191,18 +1185,19 @@ const handleFolderUpload = async (event: Event) => { const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']; const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv']; const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg']; - + + if (videoTypes.includes(fileExt)) { + videoFilteredCount++; + continue; + } + if (!vlmEnabled) { if (imageTypes.includes(fileExt)) { imageFilteredCount++; continue; } - if (videoTypes.includes(fileExt)) { - videoFilteredCount++; - continue; - } } - + if (!asrEnabled && audioTypes.includes(fileExt)) { audioFilteredCount++; continue; @@ -1600,7 +1595,7 @@ async function createNewSession(value: string): Promise { ref="uploadInputRef" type="file" class="document-upload-input" - :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg,.mp4,.mov,.avi,.mkv,.webm,.wmv,.flv'" + :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'" multiple @change="handleDocumentUpload" /> diff --git a/frontend/src/views/knowledge/settings/KBParserSettings.vue b/frontend/src/views/knowledge/settings/KBParserSettings.vue index b32036ab..3d2e5974 100644 --- a/frontend/src/views/knowledge/settings/KBParserSettings.vue +++ b/frontend/src/views/knowledge/settings/KBParserSettings.vue @@ -160,9 +160,7 @@ const fileTypeGroups = computed(() => { const jsonExts = ['json'].filter(e => ft.has(e)) const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e)) const audioExts = ['mp3', 'wav', 'm4a', 'flac', 'ogg'].filter(e => ft.has(e)) - const videoExts = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'].filter(e => ft.has(e)) - const mediaExtSet = new Set([...audioExts, ...videoExts]) - const audiovisualExts = [...mediaExtSet] + const audiovisualExts = [...audioExts] if (pdfExts.length) groups.push({ key: 'pdf', label: t('kbSettings.parser.fileTypePdf'), icon: 'file-pdf', extensions: pdfExts }) if (officeExts.length) groups.push({ key: 'office', label: t('kbSettings.parser.fileTypeWord'), icon: 'file-word', extensions: officeExts }) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index fff037ba..19001586 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -203,6 +203,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName) + if IsVideoType(getFileType(fileName)) { + logger.Error(ctx, "Video file upload is not supported") + return nil, werrors.NewBadRequestError("暂不支持上传视频文件") + } + // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) @@ -1482,8 +1487,6 @@ type ProcessChunksOptions struct { QuestionCount int EnableMultimodel bool StoredImages []docparser.StoredImage - IsVideo bool // true when knowledge is a standalone video file - VideoFilePath string // storage path to video file (provider:// URL) // ParentChunks holds parent chunk data when parent-child chunking is enabled. // When set, the chunks passed to processChunks are child chunks, and each // child's ParentIndex references an entry in this slice. @@ -1889,14 +1892,13 @@ func (s *knowledgeService) processChunks(ctx context.Context, } // Skip summary/question generation for image-type and video-type knowledge — the text chunk - // is just a markdown reference, so LLM summary would be useless. - // The multimodal task will provide a caption/summary as the description instead. + // is just a markdown reference (or placeholder), so LLM summary would be useless. + // For images, the multimodal task provides a caption/summary as the description instead. isImage := IsImageType(knowledge.FileType) isVideo := IsVideoType(knowledge.FileType) - pendingMultimodal := (isImage && options.EnableMultimodel && len(options.StoredImages) > 0) || - (isVideo && options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "") + pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0 - // For image/video files with pending multimodal processing, keep "processing" status + // For image files with pending multimodal processing, keep "processing" status // so the frontend waits until the description is ready before showing "completed". if pendingMultimodal { knowledge.ParseStatus = types.ParseStatusProcessing @@ -1942,11 +1944,6 @@ func (s *knowledgeService) processChunks(ctx context.Context, s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks) } - // Enqueue multimodal tasks for videos (async, non-blocking) - if options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "" { - s.enqueueVideoMultimodalTasks(ctx, knowledge, kb, options.VideoFilePath, chunks) - } - // Update tenant's storage usage tenantInfo.StorageUsed += totalStorageSize if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil { @@ -3033,8 +3030,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", - "mp3", "wav", "m4a", "flac", "ogg", - "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv": + "mp3", "wav", "m4a", "flac", "ogg": return true default: return false @@ -7706,6 +7702,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } + // 视频文件不再支持入库解析 + if payload.FilePath != "" && IsVideoType(payload.FileType) { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + Errorf("processDocument video not supported") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "暂不支持视频文件" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + return nil + } + // New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks var convertResult *types.ReadResult var chunks []types.ParsedChunk @@ -7920,8 +7927,6 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e QuestionCount: payload.QuestionCount, EnableMultimodel: payload.EnableMultimodel, StoredImages: storedImages, - IsVideo: convertResult != nil && convertResult.IsVideo, - VideoFilePath: payload.FilePath, } if kb.ChunkingConfig.EnableParentChild { @@ -8150,50 +8155,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks( } } -// enqueueVideoMultimodalTasks enqueues asynq tasks for multimodal video processing. -func (s *knowledgeService) enqueueVideoMultimodalTasks( - ctx context.Context, - knowledge *types.Knowledge, - kb *types.KnowledgeBase, - videoFilePath string, - chunks []types.ParsedChunk, -) { - if s.task == nil || videoFilePath == "" { - return - } - - // Get the first chunk's ID as the parent chunk - chunkID := "" - if len(chunks) > 0 { - chunkID = chunks[0].ChunkID - } - - lang, _ := types.LanguageFromContext(ctx) - payload := types.VideoMultimodalPayload{ - TenantID: knowledge.TenantID, - KnowledgeID: knowledge.ID, - KnowledgeBaseID: kb.ID, - ChunkID: chunkID, - VideoURL: videoFilePath, - EnableVLM: true, - EnableASR: true, - Language: lang, - } - - payloadBytes, err := json.Marshal(payload) - if err != nil { - logger.Warnf(ctx, "Failed to marshal video multimodal payload: %v", err) - return - } - - task := asynq.NewTask(types.TypeVideoMultimodal, payloadBytes) - if _, err := s.task.Enqueue(task); err != nil { - logger.Warnf(ctx, "Failed to enqueue video multimodal task for %s: %v", videoFilePath, err) - } else { - logger.Infof(ctx, "Enqueued video:multimodal task for %s", videoFilePath) - } -} - // ProcessFAQImport handles Asynq FAQ import tasks (including dry run mode) func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error { var payload types.FAQImportPayload diff --git a/internal/application/service/video_multimodal.go b/internal/application/service/video_multimodal.go deleted file mode 100644 index 264e2120..00000000 --- a/internal/application/service/video_multimodal.go +++ /dev/null @@ -1,790 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "sort" - "strings" - "sync" - "time" - "unicode/utf8" - - filesvc "github.com/Tencent/WeKnora/internal/application/service/file" - "github.com/Tencent/WeKnora/internal/application/service/retriever" - "github.com/Tencent/WeKnora/internal/infrastructure/chunker" - "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/models/asr" - "github.com/Tencent/WeKnora/internal/models/chat" - "github.com/Tencent/WeKnora/internal/models/utils/ollama" - "github.com/Tencent/WeKnora/internal/models/video" - "github.com/Tencent/WeKnora/internal/models/vlm" - "github.com/Tencent/WeKnora/internal/types" - "github.com/Tencent/WeKnora/internal/types/interfaces" - secutils "github.com/Tencent/WeKnora/internal/utils" - "github.com/google/uuid" - "github.com/hibiken/asynq" -) - -const ( - vlmBatchPromptTemplate = `Analyze this sequence of {{count}} video frames and provide a detailed description in {{language}}. - -{{previous_batch_instruction}} - - -1. The images are provided in chronological order. -2. Describe visible objects, people, and their actions. -3. Describe the scene and environment. -4. Extract and describe any text on the screen. -5. Note any notable events or changes across these frames. -6. Be concise but comprehensive, keeping the word count under 150 words. -7. The output must be in {{language}}. -` - - vlmFirstBatchInstruction = "\nNone. These are the first frames of the video. Describe them completely.\n" - vlmNextBatchInstruction = "\n{{previous_description}}\n\n\nThese are the subsequent frames. Focus ONLY on what has changed or what is new compared to the previous context description provided above. If there are no significant changes, just state 'No significant changes'." - - vlmSummarySystemPromptTemplate = `Please summarize the provided video frame descriptions into a coherent video summary in {{language}}. - - -1. The descriptions are ordered chronologically by timestamp. Please maintain this temporal flow. -2. Focus on the main storyline, key events, and the interactions between people and objects. -3. Identify and include any important text or dialogue that appears on screen. -4. Provide a clear, structured summary that captures the essence of the video. -5. Be concise but comprehensive, avoiding redundant descriptions of the same scene. -6. Keep the summary under 1000 words. -7. The output must be in {{language}}. -` - - vlmSummaryUserPromptTemplate = `Here are the video frame descriptions: - - -{{descriptions}} - - -Please provide the summary based on the requirements and input.` -) - -// VideoMultimodalService handles video:multimodal asynq tasks. -// It reads videos from storage (via FileService for provider:// URLs), -// extracts frames using FFmpeg, performs VLM analysis and ASR transcription, -// and creates child chunks. -type VideoMultimodalService struct { - chunkService interfaces.ChunkService - modelService interfaces.ModelService - kbService interfaces.KnowledgeBaseService - knowledgeRepo interfaces.KnowledgeRepository - tenantRepo interfaces.TenantRepository - retrieveEngine interfaces.RetrieveEngineRegistry - ollamaService *ollama.OllamaService - taskEnqueuer interfaces.TaskEnqueuer -} - -func NewVideoMultimodalService( - chunkService interfaces.ChunkService, - modelService interfaces.ModelService, - kbService interfaces.KnowledgeBaseService, - knowledgeRepo interfaces.KnowledgeRepository, - tenantRepo interfaces.TenantRepository, - retrieveEngine interfaces.RetrieveEngineRegistry, - ollamaService *ollama.OllamaService, - taskEnqueuer interfaces.TaskEnqueuer, -) interfaces.TaskHandler { - return &VideoMultimodalService{ - chunkService: chunkService, - modelService: modelService, - kbService: kbService, - knowledgeRepo: knowledgeRepo, - tenantRepo: tenantRepo, - retrieveEngine: retrieveEngine, - ollamaService: ollamaService, - taskEnqueuer: taskEnqueuer, - } -} - -// Handle implements asynq handler for TypeVideoMultimodal. -func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error { - var payload types.VideoMultimodalPayload - if err := json.Unmarshal(task.Payload(), &payload); err != nil { - return fmt.Errorf("unmarshal video multimodal payload: %w", err) - } - - logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v", - payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR) - - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) - if payload.Language != "" { - ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) - } - - var videoBytes []byte - videoBytes, err := s.readVideoBytes(ctx, payload) - if err != nil { - return fmt.Errorf("read video bytes: %w", err) - } - - if !payload.EnableVLM && !payload.EnableASR { - logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do") - return nil - } - - var extractor video.Extractor - var extErr error - if payload.EnableVLM || payload.EnableASR { - extractor, extErr = video.NewExtractor(nil) - if extErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable: %v", extErr) - } - } - - var wg sync.WaitGroup - type FrameAnalysis struct { - StartTimestamp float64 - EndTimestamp float64 - Description string - } - var frameAnalyses []FrameAnalysis - var asrSegments []asr.Segment - var mu sync.Mutex // Mutex to protect concurrent access if needed - - lang := payload.Language - if lang == "" { - lang = "zh-CN" - } - batchPrompt := strings.ReplaceAll(vlmBatchPromptTemplate, "{{language}}", lang) - - if payload.EnableVLM { - wg.Add(1) - go func() { - defer wg.Done() - - var frames []*video.Frame - if extractor == nil { - logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled") - logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis") - } else { - options := video.DefaultExtractOptions() - if payload.MaxFrames > 0 { - options.MaxFrames = payload.MaxFrames - } - var extractErr error - frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options) - if extractErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr) - frames = []*video.Frame{} - } else { - logger.Debugf(ctx, "[VideoMultimodal] Successfully extracted %d frames from video", len(frames)) - } - } - - if len(frames) == 0 { - return - } - - vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID) - if vlmErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr) - } else { - // Initialize the slice within the goroutine before appending - localAnalyses := make([]FrameAnalysis, 0) - var previousDescription string - - // Batch size configuration (number of frames per API call) - batchSize := 3 - for i := 0; i < len(frames); i += batchSize { - end := i + batchSize - if end > len(frames) { - end = len(frames) - } - batchFrames := frames[i:end] - - var batchImages [][]byte - for _, f := range batchFrames { - batchImages = append(batchImages, f.ImageData) - } - - startTs := batchFrames[0].Timestamp - endTs := batchFrames[len(batchFrames)-1].Timestamp - - logger.Debugf(ctx, "[VideoMultimodal] Starting VLM analysis for frames %d-%d/%d (%.2fs - %.2fs)", i+1, end, len(frames), startTs, endTs) - - currentPrompt := strings.ReplaceAll(batchPrompt, "{{count}}", fmt.Sprintf("%d", len(batchFrames))) - if i == 0 { - currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", vlmFirstBatchInstruction) - } else { - nextInstruction := strings.ReplaceAll(vlmNextBatchInstruction, "{{previous_description}}", previousDescription) - currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", nextInstruction) - } - - desc, descErr := vlmMdl.Predict(ctx, batchImages, currentPrompt) - if descErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frames %d-%d at %.2fs - %.2fs: %v", - i+1, end, startTs, endTs, descErr) - continue - } - - // Update previous description for the next iteration if there are changes - if !strings.Contains(strings.ToLower(desc), "no significant changes") { - previousDescription = desc - } - - localAnalyses = append(localAnalyses, FrameAnalysis{ - StartTimestamp: startTs, - EndTimestamp: endTs, - Description: desc, - }) - logger.Infof(ctx, "[VideoMultimodal] Frames %d-%d/%d analyzed (%.2fs - %.2fs), desc len=%d", - i+1, end, len(frames), startTs, endTs, len(desc)) - } - mu.Lock() - frameAnalyses = localAnalyses - mu.Unlock() - } - }() - } - - if payload.EnableASR { - wg.Add(1) - go func() { - defer wg.Done() - - audioBytes := videoBytes - fileName := "video.mp4" - if extractor != nil { - extractedAudio, err := extractor.ExtractAudio(ctx, videoBytes) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Audio extraction failed, falling back to original video: %v", err) - } else if len(extractedAudio) == 0 { - logger.Infof(ctx, "[VideoMultimodal] No audio stream found in video, skipping ASR transcription") - return - } else { - audioBytes = extractedAudio - fileName = "audio.mp3" - logger.Debugf(ctx, "[VideoMultimodal] Extracted audio track for ASR, size: %d bytes", len(audioBytes)) - } - } - - asrRes, asrErr := s.transcribeAudio(ctx, audioBytes, payload.KnowledgeBaseID, fileName) - if asrErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: kb=%s file=%s audioBytes=%d err=%v", - payload.KnowledgeBaseID, fileName, len(audioBytes), asrErr) - } else if asrRes == nil { - logger.Warnf(ctx, "[VideoMultimodal] ASR returned nil result without error: kb=%s file=%s audioBytes=%d", - payload.KnowledgeBaseID, fileName, len(audioBytes)) - } else { - asrSegments = asrRes.Segments - nSeg := len(asrRes.Segments) - nRunes := utf8.RuneCountInString(asrRes.Text) - var spanStart, spanEnd float64 - if nSeg > 0 { - spanStart = asrRes.Segments[0].Start - spanEnd = asrRes.Segments[0].End - for _, seg := range asrRes.Segments[1:] { - if seg.Start < spanStart { - spanStart = seg.Start - } - if seg.End > spanEnd { - spanEnd = seg.End - } - } - } - const previewRunes = 200 - preview := strings.TrimSpace(asrRes.Text) - if utf8.RuneCountInString(preview) > previewRunes { - preview = string([]rune(preview)[:previewRunes]) + "..." - } - logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed: kb=%s file=%s audioBytes=%d textRunes=%d segments=%d timeline=%.2fs-%.2fs preview=%q", - payload.KnowledgeBaseID, fileName, len(audioBytes), nRunes, nSeg, spanStart, spanEnd, preview) - } - }() - } - - logger.Debugf(ctx, "[VideoMultimodal] Waiting for VLM and ASR tasks to complete...") - wg.Wait() - logger.Debugf(ctx, "[VideoMultimodal] VLM and ASR tasks completed") - - var videoSummary string - var combinedInput string - var kb *types.KnowledgeBase - if (len(frameAnalyses) > 0 || len(asrSegments) > 0) && (payload.EnableVLM || payload.EnableASR) { - var err error - kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - - type TimelineEvent struct { - Start float64 - End float64 - Text string - Type string - } - var events []TimelineEvent - for _, f := range frameAnalyses { - events = append(events, TimelineEvent{Start: f.StartTimestamp, End: f.EndTimestamp, Text: f.Description, Type: "frame"}) - } - for _, seg := range asrSegments { - events = append(events, TimelineEvent{Start: seg.Start, End: seg.End, Text: seg.Text, Type: "audio"}) - } - sort.Slice(events, func(i, j int) bool { - return events[i].Start < events[j].Start - }) - - var combinedBuilder strings.Builder - for _, ev := range events { - if ev.Type == "frame" { - if ev.Start == ev.End { - combinedBuilder.WriteString(fmt.Sprintf("[%.2fs] Visual: %s\n", ev.Start, ev.Text)) - } else { - combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Visual: %s\n", ev.Start, ev.End, ev.Text)) - } - } else { - combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Audio: %s\n", ev.Start, ev.End, ev.Text)) - } - } - combinedInput = combinedBuilder.String() - - if err != nil || kb == nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for summary: %v", err) - videoSummary = combinedInput - } else { - chatMdl, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID) - if err != nil || chatMdl == nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get chat model for summary (modelID=%s): %v", kb.SummaryModelID, err) - videoSummary = combinedInput - } else { - // Limit combinedInput to avoid context length overflow. - // Using ~32000 runes to be safe for typical models. - if utf8.RuneCountInString(combinedInput) > 32000 { - runes := []rune(combinedInput) - combinedInput = string(runes[:32000]) + "\n...[truncated due to length]..." - } - - systemPrompt := strings.ReplaceAll(vlmSummarySystemPromptTemplate, "{{language}}", lang) - userPrompt := strings.ReplaceAll(vlmSummaryUserPromptTemplate, "{{descriptions}}", combinedInput) - - logger.Debugf(ctx, "[VideoMultimodal] Generating video summary with model %s, input length: %d chars", kb.SummaryModelID, len(combinedInput)) - resp, sumErr := chatMdl.Chat(ctx, []chat.Message{ - {Role: "system", Content: systemPrompt}, - {Role: "user", Content: userPrompt}, - }, nil) - - if sumErr != nil || resp == nil { - logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr) - videoSummary = combinedInput - } else { - videoSummary = resp.Content - } - } - } - } - - if kb == nil { - var err error - kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for video chunking: %v", err) - } - } - - videoInfo := types.VideoInfo{ - URL: payload.VideoURL, - } - videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo}) - - newChunks := s.buildVideoChunks(payload, combinedInput, videoSummary, string(videoInfoJSON), kb) - - if len(newChunks) == 0 { - // Even if VLM/ASR both failed, mark knowledge as completed - s.finalizeVideoKnowledge(ctx, payload, "") - return nil - } - - if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil { - return fmt.Errorf("create video multimodal chunks: %w", err) - } - for _, c := range newChunks { - logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d", - c.ChunkType, c.ID, payload.VideoURL, len(c.Content)) - } - - // Index chunks so they can be retrieved - s.indexChunks(ctx, payload, newChunks) - - // For standalone video files, use summary as the knowledge description - // and mark the knowledge as completed (it was kept in "processing" until now). - s.finalizeVideoKnowledge(ctx, payload, videoSummary) - - // Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled. - // During initial processChunks, question generation is skipped for video-type - // knowledge because the text chunk is just a markdown reference. Now that we - // have real textual content (frame descriptions, summary, ASR), we can generate questions. - s.enqueueQuestionGenerationIfEnabled(ctx, payload) - - return nil -} - -func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) { - var err error - var videoBytes []byte - if types.ParseProviderScheme(payload.VideoURL) != "" { - fileSvc := s.resolveFileServiceForPayload(ctx, payload) - if fileSvc == nil { - logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s", - payload.TenantID, payload.KnowledgeBaseID) - } else { - // provider:// scheme — read via FileService - reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL) - if getErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr) - } else { - videoBytes, err = io.ReadAll(reader) - reader.Close() - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err) - videoBytes = nil - } - } - } - } - if videoBytes == nil && payload.VideoLocalPath != "" { - videoBytes, err = os.ReadFile(payload.VideoLocalPath) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err) - videoBytes = nil - } - } - if videoBytes == nil { - videoBytes, err = downloadVideoFromURL(payload.VideoURL) - if err != nil { - logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err) - return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err) - } - logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes)) - } - return videoBytes, nil -} - -// downloadVideoFromURL downloads video bytes from an HTTP(S) URL. -func downloadVideoFromURL(videoURL string) ([]byte, error) { - return secutils.DownloadBytes(videoURL) -} - -// transcribeAudio transcribes audio from a video using ASR. -func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, audioBytes []byte, kbID string, fileName string) (*asr.TranscriptionResult, error) { - asrMdl, err := s.resolveASR(ctx, kbID) - if err != nil { - return nil, fmt.Errorf("resolve ASR: %w", err) - } - return asrMdl.Transcribe(ctx, audioBytes, fileName) -} - -// buildVideoChunks creates child chunks for video multimodal analysis results: -// - Video summary (split into multiple chunks if too long) -// - Timeline combined input (split into multiple chunks if too long) -func (s *VideoMultimodalService) buildVideoChunks( - payload types.VideoMultimodalPayload, - combinedInput string, - videoSummary string, - videoInfoJSON string, - kb *types.KnowledgeBase, -) []*types.Chunk { - var newChunks []*types.Chunk - - var parentContentBuilder strings.Builder - if videoSummary != "" { - parentContentBuilder.WriteString("视频摘要：\n") - parentContentBuilder.WriteString(videoSummary) - parentContentBuilder.WriteString("\n\n") - } - if combinedInput != "" { - parentContentBuilder.WriteString("详细时间轴解析：\n") - parentContentBuilder.WriteString(combinedInput) - } - parentContent := strings.TrimSpace(parentContentBuilder.String()) - if parentContent == "" { - parentContent = "该视频未解析出有效内容" - } - - // Construct a Parent Chunk representing the entire video - parentChunkID := uuid.New().String() - newChunks = append(newChunks, &types.Chunk{ - ID: parentChunkID, - TenantID: payload.TenantID, - KnowledgeID: payload.KnowledgeID, - KnowledgeBaseID: payload.KnowledgeBaseID, - Content: parentContent, - ChunkType: types.ChunkTypeParentText, - ParentChunkID: payload.ChunkID, // Link back to the original text chunk if extracted from a document - IsEnabled: true, - Flags: types.ChunkFlagRecommended, - VideoInfo: videoInfoJSON, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }) - - var chunkCfg chunker.SplitterConfig - if kb != nil { - baseCfg := buildSplitterConfig(kb) - if kb.ChunkingConfig.EnableParentChild { - // If parent-child is globally enabled, these chunks act as child chunks - // for the newly created video parent chunk. - _, childCfg := buildParentChildConfigs(kb.ChunkingConfig, baseCfg) - chunkCfg = childCfg - } else { - chunkCfg = baseCfg - } - } else { - chunkCfg = chunker.SplitterConfig{ - ChunkSize: 512, - ChunkOverlap: 64, - Separators: []string{"\n\n", "\n", "。"}, - } - } - - if combinedInput != "" { - inputChunks := chunker.SplitText(combinedInput, chunkCfg) - for _, sc := range inputChunks { - newChunks = append(newChunks, &types.Chunk{ - ID: uuid.New().String(), - TenantID: payload.TenantID, - KnowledgeID: payload.KnowledgeID, - KnowledgeBaseID: payload.KnowledgeBaseID, - Content: sc.Content, - ChunkType: types.ChunkTypeText, - ParentChunkID: parentChunkID, - IsEnabled: true, - Flags: types.ChunkFlagRecommended, - VideoInfo: videoInfoJSON, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }) - } - } - - if videoSummary != "" { - summaryChunks := chunker.SplitText(videoSummary, chunkCfg) - for _, sc := range summaryChunks { - newChunks = append(newChunks, &types.Chunk{ - ID: uuid.New().String(), - TenantID: payload.TenantID, - KnowledgeID: payload.KnowledgeID, - KnowledgeBaseID: payload.KnowledgeBaseID, - Content: sc.Content, - ChunkType: types.ChunkTypeText, - IsEnabled: true, - Flags: types.ChunkFlagRecommended, - VideoInfo: videoInfoJSON, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }) - } - } - - return newChunks -} - -// finalizeVideoKnowledge updates the knowledge after multimodal processing: -// - For standalone video files: sets Description from summary and marks ParseStatus as completed. -// - For videos extracted from documents: no-op (description comes from summary generation). -func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) { - knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err) - return - } - if knowledge == nil { - return - } - if !IsVideoType(knowledge.FileType) { - return - } - - if summary != "" { - knowledge.Description = summary - } - knowledge.ParseStatus = types.ParseStatusCompleted - knowledge.UpdatedAt = time.Now() - if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err) - } else { - logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)", - payload.KnowledgeID, len(knowledge.Description)) - } -} - -// indexChunks indexes the newly created video chunks into the retrieval engine -// so they can participate in semantic search. -func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) { - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if err != nil || kb == nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err) - return - } - - embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err) - return - } - - tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err) - return - } - - engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err) - return - } - - indexInfoList := make([]*types.IndexInfo, 0, len(chunks)) - for _, chunk := range chunks { - indexInfoList = append(indexInfoList, &types.IndexInfo{ - Content: chunk.Content, - SourceID: chunk.ID, - SourceType: types.ChunkSourceType, - ChunkID: chunk.ID, - KnowledgeID: chunk.KnowledgeID, - KnowledgeBaseID: chunk.KnowledgeBaseID, - }) - } - - if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil { - logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err) - return - } - - // Mark chunks as indexed. - // Must re-fetch from DB because the in-memory objects lack auto-generated fields - // (e.g. seq_id), and GORM Save would overwrite them with zero values. - for _, chunk := range chunks { - dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err) - continue - } - dbChunk.Status = int(types.ChunkStatusIndexed) - if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err) - } - } - - logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL) -} - -// resolveVLM creates a vlm.VLM instance for the given knowledge base, -// supporting both new-style (ModelID) and legacy (inline BaseURL) configs. -func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) { - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) - if err != nil { - return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err) - } - if kb == nil { - return nil, fmt.Errorf("knowledge base %s not found", kbID) - } - - vlmCfg := kb.VLMConfig - if !vlmCfg.IsEnabled() { - return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID) - } - - // New-style: resolve model through ModelService - if vlmCfg.ModelID != "" { - return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID) - } - - // Legacy: create VLM from inline config - return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService) -} - -// resolveASR creates an asr.ASR instance for the given knowledge base. -func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) { - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) - if err != nil { - return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err) - } - if kb == nil { - return nil, fmt.Errorf("knowledge base %s not found", kbID) - } - - asrCfg := kb.ASRConfig - if !asrCfg.IsASREnabled() { - return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID) - } - - return s.modelService.GetASRModel(ctx, asrCfg.ModelID) -} - -// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs. -func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService { - tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) - if err != nil || tenant == nil { - logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err) - return nil - } - - provider := types.ParseProviderScheme(payload.VideoURL) - if provider == "" { - kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if kbErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr) - } else if kb != nil { - provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider())) - } - } - - baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) - fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir) - if svcErr != nil { - logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr) - return nil - } - return fileSvc -} - -// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question -// generation enabled and, if so, enqueues a task for the video knowledge. -func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) { - if s.taskEnqueuer == nil { - return - } - - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if err != nil || kb == nil { - return - } - if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled { - return - } - - questionCount := kb.QuestionGenerationConfig.QuestionCount - if questionCount <= 0 { - questionCount = 3 - } - if questionCount > 10 { - questionCount = 10 - } - - taskPayload := types.QuestionGenerationPayload{ - TenantID: payload.TenantID, - KnowledgeBaseID: payload.KnowledgeBaseID, - KnowledgeID: payload.KnowledgeID, - QuestionCount: questionCount, - Language: payload.Language, - } - payloadBytes, err := json.Marshal(taskPayload) - if err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err) - return - } - - task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) - if _, err := s.taskEnqueuer.Enqueue(task); err != nil { - logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err) - } else { - logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)", - payload.KnowledgeID, questionCount) - } -} diff --git a/internal/container/container.go b/internal/container/container.go index 0bc5a218..d11266ab 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -175,7 +175,6 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor"))) must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary"))) must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal"))) - must(container.Provide(service.NewVideoMultimodalService, dig.Name("videoMultimodal"))) must(container.Provide(service.NewMessageService)) must(container.Provide(service.NewMCPServiceService)) diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go index ec6ac077..3779cfa7 100644 --- a/internal/infrastructure/docparser/builtin_converter.go +++ b/internal/infrastructure/docparser/builtin_converter.go @@ -28,10 +28,6 @@ var audioFormats = map[string]bool{ "mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true, } -var videoFormats = map[string]bool{ - "mp4": true, "mov": true, "avi": true, "mkv": true, "webm": true, "wmv": true, "flv": true, -} - func init() { for k := range imageFormats { simpleFormats[k] = true @@ -39,9 +35,6 @@ func init() { for k := range audioFormats { simpleFormats[k] = true } - for k := range videoFormats { - simpleFormats[k] = true - } } // IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader. @@ -81,8 +74,6 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t return imageToResult(req.FileName, req.FileContent), nil case audioFormats[ft]: return audioToResult(req.FileName, req.FileContent), nil - case videoFormats[ft]: - return videoToResult(req.FileName, req.FileContent), nil default: return nil, fmt.Errorf("unsupported simple format: %s", ft) } @@ -122,11 +113,6 @@ func IsAudioFormat(fileType string) bool { return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))] } -// IsVideoFormat returns true if the file type is a recognized video format. -func IsVideoFormat(fileType string) bool { - return videoFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))] -} - // audioToResult wraps a standalone audio file. The actual transcription is // handled by the ASR model in the knowledge service pipeline. Here we just // return a placeholder markdown with the raw bytes preserved for upstream @@ -144,22 +130,6 @@ func audioToResult(fileName string, data []byte) *types.ReadResult { } } -// videoToResult wraps a standalone video file. The actual frame extraction, -// VLM analysis and ASR transcription is handled by the VideoMultimodalService -// in the knowledge service pipeline. Here we just return a placeholder markdown -// with the raw bytes preserved for upstream processing. -func videoToResult(fileName string, data []byte) *types.ReadResult { - if fileName == "" { - fileName = "video.mp4" - } - // Return a placeholder; the knowledge service will handle the actual processing. - return &types.ReadResult{ - MarkdownContent: fmt.Sprintf("[Video file: %s]", fileName), - IsVideo: true, - VideoData: data, - } -} - // ensureOriginalImageRef checks whether the input file is an image and, if the // returned markdown does not already contain a markdown image reference for it, // prepends one and appends the raw bytes to imageRefs. This guarantees that diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 9c244d18..2cd1983e 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -43,7 +43,7 @@ func (e *builtinEngine) Description() string { return "DocReader built-in parser engine" } func (e *builtinEngine) FileTypes(_ bool) []string { - return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"} + return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} } func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) { if docreaderConnected { @@ -71,7 +71,7 @@ func (e *simpleEngine) Description() string { return "Simple format & image parsing (no external service required)" } func (e *simpleEngine) FileTypes(_ bool) []string { - return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"} + return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} } func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { return true, "" diff --git a/internal/models/video/extractor.go b/internal/models/video/extractor.go deleted file mode 100644 index eed9ef2a..00000000 --- a/internal/models/video/extractor.go +++ /dev/null @@ -1,98 +0,0 @@ -package video - -import ( - "context" - "io" -) - -// Extractor defines the interface for video frame extraction operations. -type Extractor interface { - // ExtractFrames extracts frames from a video file. - // It returns a slice of Frame objects representing the extracted frames. - ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error) - - // ExtractFramesToReader extracts frames from a video and returns an io.Reader - // for each frame's image data. This is useful for streaming processing. - ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error) - - // ExtractAudio extracts the audio track from a video file and returns it as bytes. - ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error) - - // GetVideoInfo retrieves basic information about a video file. - // It returns a map containing duration, width, height, etc. - GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) -} - -// Frame represents a single extracted frame from a video. -type Frame struct { - // Index is the sequential index of the frame (0-based). - Index int - - // Timestamp is the time in seconds from the start of the video. - Timestamp float64 - - // ImageData contains the raw image bytes of the frame. - ImageData []byte - - // Format specifies the image format (e.g., "jpeg", "png"). - Format string - - // Width is the width of the frame in pixels. - Width int - - // Height is the height of the frame in pixels. - Height int -} - -// ExtractOptions configures how frames are extracted from a video. -type ExtractOptions struct { - // FrameInterval specifies how often to extract frames (in seconds). - // For example, 1.0 means extract one frame per second. - FrameInterval float64 - - // MaxFrames specifies the maximum number of frames to extract. - // If 0, extract all frames according to FrameInterval. - MaxFrames int - - // Format specifies the output image format for extracted frames. - // Supported formats: "jpeg" (default), "png". - Format string - - // Quality specifies the image quality for JPEG output (1-100, default 80). - Quality int - - // Width specifies the target width for resizing frames (0 = no resize). - Width int - - // Height specifies the target height for resizing frames (0 = no resize). - // If only one dimension is specified, the aspect ratio is preserved. - Height int -} - -// DefaultExtractOptions returns the default frame extraction options. -func DefaultExtractOptions() *ExtractOptions { - return &ExtractOptions{ - FrameInterval: 1.0, - MaxFrames: 0, - Format: "jpeg", - Quality: 80, - Width: 0, - Height: 0, - } -} - -// Config holds the configuration needed to create a video extractor. -type Config struct { - // FFMpegPath is the path to the ffmpeg executable. - // If empty, it will try to find ffmpeg in the system PATH. - FFMpegPath string - - // TempDir is the directory to use for temporary files. - // If empty, the system default temporary directory is used. - TempDir string -} - -// NewExtractor creates a new video frame extractor based on the provided configuration. -func NewExtractor(config *Config) (Extractor, error) { - return NewFFMpegExtractor(config) -} diff --git a/internal/models/video/ffmpeg_extractor.go b/internal/models/video/ffmpeg_extractor.go deleted file mode 100644 index d3019b8f..00000000 --- a/internal/models/video/ffmpeg_extractor.go +++ /dev/null @@ -1,342 +0,0 @@ -package video - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/Tencent/WeKnora/internal/logger" -) - -// FFMpegExtractor implements the Extractor interface using FFmpeg. -type FFMpegExtractor struct { - ffmpegPath string - tempDir string -} - -// NewFFMpegExtractor creates a new FFmpeg-based video extractor. -func NewFFMpegExtractor(config *Config) (Extractor, error) { - ffmpegPath := "ffmpeg" - tempDir := os.TempDir() - - if config != nil { - if config.FFMpegPath != "" { - ffmpegPath = config.FFMpegPath - } - if config.TempDir != "" { - tempDir = config.TempDir - } - } - - extractor := &FFMpegExtractor{ffmpegPath: ffmpegPath, tempDir: tempDir} - if err := extractor.checkFFmpeg(); err != nil { - return nil, err - } - return extractor, nil -} - -// checkFFmpeg verifies that FFmpeg is available. -func (e *FFMpegExtractor) checkFFmpeg() error { - cmd := exec.Command(e.ffmpegPath, "-version") - _, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("ffmpeg not found or not executable: %w", err) - } - return nil -} - -// ExtractFrames extracts frames from a video using FFmpeg. -func (e *FFMpegExtractor) ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error) { - if options == nil { - options = DefaultExtractOptions() - } - - tempVideoPath, err := e.writeTempVideo(videoBytes) - if err != nil { - return nil, err - } - defer os.Remove(tempVideoPath) - - tempFrameDir, err := os.MkdirTemp(e.tempDir, "video-frames-") - if err != nil { - return nil, fmt.Errorf("failed to create temp frame directory: %w", err) - } - defer os.RemoveAll(tempFrameDir) - - if _, err := e.getVideoDuration(ctx, tempVideoPath); err != nil { - logger.Warnf(ctx, "[VideoFFMpeg] Failed to get video duration: %v", err) - } - - if err := e.extractFramesWithFFmpeg(ctx, tempVideoPath, tempFrameDir, options); err != nil { - return nil, err - } - - return e.readExtractedFrames(ctx, tempFrameDir, options) -} - -// ExtractFramesToReaders extracts frames and returns them as io.Readers. -func (e *FFMpegExtractor) ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error) { - frames, err := e.ExtractFrames(ctx, videoBytes, options) - if err != nil { - return nil, err - } - - readers := make([]io.Reader, 0, len(frames)) - for _, frame := range frames { - readers = append(readers, bytes.NewReader(frame.ImageData)) - } - - return readers, nil -} - -// writeTempVideo writes the video bytes to a temporary file. -func (e *FFMpegExtractor) writeTempVideo(videoBytes []byte) (string, error) { - tempFile, err := os.CreateTemp(e.tempDir, "video-input-*.mp4") - if err != nil { - return "", fmt.Errorf("failed to create temp video file: %w", err) - } - defer tempFile.Close() - - if _, err := tempFile.Write(videoBytes); err != nil { - os.Remove(tempFile.Name()) - return "", fmt.Errorf("failed to write video bytes: %w", err) - } - - return tempFile.Name(), nil -} - -// getVideoDuration gets the duration of a video using FFmpeg. -func (e *FFMpegExtractor) getVideoDuration(ctx context.Context, videoPath string) (float64, error) { - cmd := exec.CommandContext(ctx, e.ffmpegPath, - "-i", videoPath, - "-f", "null", - "-") - - output, err := cmd.CombinedOutput() - if err != nil { - return 0, err - } - - outputStr := string(output) - durationStr := "" - lines := strings.Split(outputStr, "\n") - for _, line := range lines { - if strings.Contains(line, "Duration:") { - parts := strings.Split(line, "Duration:") - if len(parts) > 1 { - durationPart := strings.TrimSpace(parts[1]) - durationStr = strings.Split(durationPart, ",")[0] - break - } - } - } - - if durationStr == "" { - return 0, fmt.Errorf("could not parse duration") - } - - timeParts := strings.Split(durationStr, ":") - if len(timeParts) != 3 { - return 0, fmt.Errorf("invalid duration format: %s", durationStr) - } - - hours, _ := strconv.Atoi(timeParts[0]) - minutes, _ := strconv.Atoi(timeParts[1]) - seconds, _ := strconv.ParseFloat(timeParts[2], 64) - - totalSeconds := float64(hours*3600) + float64(minutes*60) + seconds - return totalSeconds, nil -} - -// extractFramesWithFFmpeg uses FFmpeg to extract frames from a video. -func (e *FFMpegExtractor) extractFramesWithFFmpeg(ctx context.Context, videoPath, outputDir string, options *ExtractOptions) error { - filters := []string{fmt.Sprintf("fps=1/%f", options.FrameInterval)} - - if options.Width > 0 || options.Height > 0 { - scaleFilter := "scale=" - if options.Width > 0 { - scaleFilter += strconv.Itoa(options.Width) - } else { - scaleFilter += "-1" - } - scaleFilter += ":" - if options.Height > 0 { - scaleFilter += strconv.Itoa(options.Height) - } else { - scaleFilter += "-1" - } - filters = append(filters, scaleFilter) - } - - args := []string{ - "-i", videoPath, - "-vf", strings.Join(filters, ","), - } - - quality := options.Quality - if quality < 1 { - quality = 1 - } else if quality > 100 { - quality = 100 - } - if options.Format == "jpeg" { - args = append(args, "-q:v", strconv.Itoa(quality)) - } - - outputPattern := filepath.Join(outputDir, "frame-%05d."+options.Format) - args = append(args, outputPattern) - - cmd := exec.CommandContext(ctx, e.ffmpegPath, args...) - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("ffmpeg extraction failed: %w, stderr: %s", err, stderr.String()) - } - - logger.Infof(ctx, "[VideoFFMpeg] Extracted frames to: %s", outputDir) - return nil -} - -// readExtractedFrames reads the extracted frames from the temporary directory. -func (e *FFMpegExtractor) readExtractedFrames(ctx context.Context, frameDir string, options *ExtractOptions) ([]*Frame, error) { - files, err := filepath.Glob(filepath.Join(frameDir, "frame-*."+options.Format)) - if err != nil { - return nil, fmt.Errorf("failed to list frame files: %w", err) - } - - if len(files) == 0 { - return nil, fmt.Errorf("no frames were extracted") - } - - frames := make([]*Frame, 0, len(files)) - for i, filePath := range files { - imgData, err := os.ReadFile(filePath) - if err != nil { - logger.Warnf(ctx, "[VideoFFMpeg] Failed to read frame %d: %v", i, err) - continue - } - - frame := &Frame{ - Index: i, - Timestamp: float64(i) * options.FrameInterval, - ImageData: imgData, - Format: options.Format, - Width: options.Width, - Height: options.Height, - } - - frames = append(frames, frame) - - if options.MaxFrames > 0 && len(frames) >= options.MaxFrames { - break - } - } - - if len(frames) == 0 { - return nil, fmt.Errorf("no frames could be read") - } - - logger.Infof(ctx, "[VideoFFMpeg] Successfully read %d frames", len(frames)) - return frames, nil -} - -// ExtractAudio extracts the audio track from a video using FFmpeg. -func (e *FFMpegExtractor) ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error) { - tempVideoPath, err := e.writeTempVideo(videoBytes) - if err != nil { - return nil, err - } - defer os.Remove(tempVideoPath) - - tempAudioFile, err := os.CreateTemp(e.tempDir, "audio-output-*.mp3") - if err != nil { - return nil, fmt.Errorf("failed to create temp audio file: %w", err) - } - tempAudioPath := tempAudioFile.Name() - tempAudioFile.Close() - defer os.Remove(tempAudioPath) - - cmd := exec.CommandContext(ctx, e.ffmpegPath, - "-i", tempVideoPath, - "-vn", - "-c:a", "libmp3lame", - "-q:a", "2", - "-y", - tempAudioPath, - ) - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - if strings.Contains(stderr.String(), "Output file does not contain any stream") { - logger.Infof(ctx, "[VideoFFMpeg] Video does not contain an audio stream") - return nil, nil - } - return nil, fmt.Errorf("ffmpeg audio extraction failed: %w, stderr: %s", err, stderr.String()) - } - - audioBytes, err := os.ReadFile(tempAudioPath) - if err != nil { - return nil, fmt.Errorf("failed to read extracted audio: %w", err) - } - - logger.Infof(ctx, "[VideoFFMpeg] Extracted audio, size: %d bytes", len(audioBytes)) - return audioBytes, nil -} - -// GetVideoInfo returns basic information about a video. -func (e *FFMpegExtractor) GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) { - tempVideoPath, err := e.writeTempVideo(videoBytes) - if err != nil { - return nil, err - } - defer os.Remove(tempVideoPath) - - duration, err := e.getVideoDuration(ctx, tempVideoPath) - if err != nil { - return nil, err - } - - cmd := exec.CommandContext(ctx, e.ffmpegPath, "-i", tempVideoPath, "-f", "null", "-") - output, _ := cmd.CombinedOutput() - outputStr := string(output) - - width, height := 0, 0 - lines := strings.Split(outputStr, "\n") - for _, line := range lines { - if strings.Contains(line, "Video:") { - parts := strings.Split(line, ",") - for _, part := range parts { - part = strings.TrimSpace(part) - if strings.Contains(part, "x") && !strings.Contains(part, "SAR") { - resParts := strings.Split(part, "x") - if len(resParts) == 2 { - width, _ = strconv.Atoi(strings.TrimSpace(resParts[0])) - heightStr := strings.TrimSpace(resParts[1]) - heightParts := strings.Split(heightStr, " ") - if len(heightParts) > 0 { - height, _ = strconv.Atoi(heightParts[0]) - } - } - break - } - } - break - } - } - - videoInfo := map[string]interface{}{ - "duration": duration, - "width": width, - "height": height, - } - - return videoInfo, nil -} diff --git a/internal/router/task.go b/internal/router/task.go index b804028c..ff4e99fe 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -23,7 +23,6 @@ type AsynqTaskParams struct { ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"` DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"` ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"` - VideoMultimodal interfaces.TaskHandler `name:"videoMultimodal"` } func getAsynqRedisClientOpt() *asynq.RedisClientOpt { @@ -110,9 +109,6 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register image multimodal handler mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle) - // Register video multimodal handler - mux.HandleFunc(types.TypeVideoMultimodal, params.VideoMultimodal.Handle) - // Register data source sync handler mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync) diff --git a/internal/types/docparser.go b/internal/types/docparser.go index 5ef46947..861a8af4 100644 --- a/internal/types/docparser.go +++ b/internal/types/docparser.go @@ -22,8 +22,6 @@ type ReadResult struct { Error string IsAudio bool // true when the result contains raw audio data needing ASR transcription AudioData []byte // raw audio bytes for ASR processing - IsVideo bool // true when the result contains raw video data needing multimodal processing - VideoData []byte // raw video bytes for video multimodal processing } // ImageRef represents an image reference extracted from the document. diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index 47da6a05..cc589a3f 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -13,7 +13,6 @@ const ( TypeKnowledgeMove = "knowledge:move" // 知识移动任务 TypeDataTableSummary = "datatable:summary" // 表格摘要任务 TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务（OCR + VLM Caption） - TypeVideoMultimodal = "video:multimodal" // 视频多模态处理任务（抽帧 + VLM + ASR） TypeManualProcess = "manual:process" // 手工知识更新任务（cleanup + 重新索引） TypeDataSourceSync = "datasource:sync" // 数据源同步任务 ) @@ -156,20 +155,6 @@ type ImageMultimodalPayload struct { Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates } -// VideoMultimodalPayload represents the video multimodal processing task payload. -type VideoMultimodalPayload struct { - TenantID uint64 `json:"tenant_id"` - KnowledgeID string `json:"knowledge_id"` - KnowledgeBaseID string `json:"knowledge_base_id"` - ChunkID string `json:"chunk_id"` // parent text chunk - VideoURL string `json:"video_url"` // provider:// URL (e.g. local://..., minio://...) - VideoLocalPath string `json:"video_local_path"` // deprecated: kept for backward compat with in-flight tasks - EnableVLM bool `json:"enable_vlm"` // enable VLM frame analysis - EnableASR bool `json:"enable_asr"` // enable ASR audio transcription - Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates - MaxFrames int `json:"max_frames,omitempty"` // optional: max frames to extract, 0 = no limit -} - // KBCloneTaskStatus represents the status of a knowledge base clone task type KBCloneTaskStatus string