feat(video): remove video upload support and update related terminology

- Removed support for video file uploads across the application, including updates to the knowledge base and document processing services.
- Updated internationalization files to reflect the removal of video-related terminology, changing labels and descriptions to focus solely on audio processing.
- Adjusted file type handling to exclude video formats, ensuring users receive accurate feedback regarding supported file types.
- Enhanced error handling to provide clear messages when users attempt to upload unsupported video files.

This update streamlines the application by focusing on audio processing capabilities and improving user experience through clearer messaging.
This commit is contained in:
wizardchen
2026-04-14 22:57:58 +08:00
committed by lyingbug
parent 0fcab3a413
commit 86d26fa9bf
17 changed files with 78 additions and 1406 deletions
+21 -60
View File
@@ -203,6 +203,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName)
if IsVideoType(getFileType(fileName)) {
logger.Error(ctx, "Video file upload is not supported")
return nil, werrors.NewBadRequestError("暂不支持上传视频文件")
}
// Get knowledge base configuration
logger.Info(ctx, "Getting knowledge base configuration")
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
@@ -1482,8 +1487,6 @@ type ProcessChunksOptions struct {
QuestionCount int
EnableMultimodel bool
StoredImages []docparser.StoredImage
IsVideo bool // true when knowledge is a standalone video file
VideoFilePath string // storage path to video file (provider:// URL)
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
@@ -1889,14 +1892,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
// Skip summary/question generation for image-type and video-type knowledge — the text chunk
// is just a markdown reference, so LLM summary would be useless.
// The multimodal task will provide a caption/summary as the description instead.
// is just a markdown reference (or placeholder), so LLM summary would be useless.
// For images, the multimodal task provides a caption/summary as the description instead.
isImage := IsImageType(knowledge.FileType)
isVideo := IsVideoType(knowledge.FileType)
pendingMultimodal := (isImage && options.EnableMultimodel && len(options.StoredImages) > 0) ||
(isVideo && options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "")
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
// For image/video files with pending multimodal processing, keep "processing" status
// For image files with pending multimodal processing, keep "processing" status
// so the frontend waits until the description is ready before showing "completed".
if pendingMultimodal {
knowledge.ParseStatus = types.ParseStatusProcessing
@@ -1942,11 +1944,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks)
}
// Enqueue multimodal tasks for videos (async, non-blocking)
if options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "" {
s.enqueueVideoMultimodalTasks(ctx, knowledge, kb, options.VideoFilePath, chunks)
}
// Update tenant's storage usage
tenantInfo.StorageUsed += totalStorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
@@ -3033,8 +3030,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
"mp3", "wav", "m4a", "flac", "ogg",
"mp4", "mov", "avi", "mkv", "webm", "wmv", "flv":
"mp3", "wav", "m4a", "flac", "ogg":
return true
default:
return false
@@ -7706,6 +7702,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
// 视频文件不再支持入库解析
if payload.FilePath != "" && IsVideoType(payload.FileType) {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument video not supported")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "暂不支持视频文件"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
var convertResult *types.ReadResult
var chunks []types.ParsedChunk
@@ -7920,8 +7927,6 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
QuestionCount: payload.QuestionCount,
EnableMultimodel: payload.EnableMultimodel,
StoredImages: storedImages,
IsVideo: convertResult != nil && convertResult.IsVideo,
VideoFilePath: payload.FilePath,
}
if kb.ChunkingConfig.EnableParentChild {
@@ -8150,50 +8155,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
}
}
// enqueueVideoMultimodalTasks enqueues asynq tasks for multimodal video processing.
func (s *knowledgeService) enqueueVideoMultimodalTasks(
ctx context.Context,
knowledge *types.Knowledge,
kb *types.KnowledgeBase,
videoFilePath string,
chunks []types.ParsedChunk,
) {
if s.task == nil || videoFilePath == "" {
return
}
// Get the first chunk's ID as the parent chunk
chunkID := ""
if len(chunks) > 0 {
chunkID = chunks[0].ChunkID
}
lang, _ := types.LanguageFromContext(ctx)
payload := types.VideoMultimodalPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kb.ID,
ChunkID: chunkID,
VideoURL: videoFilePath,
EnableVLM: true,
EnableASR: true,
Language: lang,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Warnf(ctx, "Failed to marshal video multimodal payload: %v", err)
return
}
task := asynq.NewTask(types.TypeVideoMultimodal, payloadBytes)
if _, err := s.task.Enqueue(task); err != nil {
logger.Warnf(ctx, "Failed to enqueue video multimodal task for %s: %v", videoFilePath, err)
} else {
logger.Infof(ctx, "Enqueued video:multimodal task for %s", videoFilePath)
}
}
// ProcessFAQImport handles Asynq FAQ import tasks (including dry run mode)
func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error {
var payload types.FAQImportPayload
@@ -1,790 +0,0 @@
package service
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/video"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmBatchPromptTemplate = `Analyze this sequence of {{count}} video frames and provide a detailed description in {{language}}.
{{previous_batch_instruction}}
<Requirements>
1. The images are provided in chronological order.
2. Describe visible objects, people, and their actions.
3. Describe the scene and environment.
4. Extract and describe any text on the screen.
5. Note any notable events or changes across these frames.
6. Be concise but comprehensive, keeping the word count under 150 words.
7. The output must be in {{language}}.
</Requirements>`
vlmFirstBatchInstruction = "<PreviousContext>\nNone. These are the first frames of the video. Describe them completely.\n</PreviousContext>"
vlmNextBatchInstruction = "<PreviousContext>\n{{previous_description}}\n</PreviousContext>\n\nThese are the subsequent frames. Focus ONLY on what has changed or what is new compared to the previous context description provided above. If there are no significant changes, just state 'No significant changes'."
vlmSummarySystemPromptTemplate = `Please summarize the provided video frame descriptions into a coherent video summary in {{language}}.
<Requirements>
1. The descriptions are ordered chronologically by timestamp. Please maintain this temporal flow.
2. Focus on the main storyline, key events, and the interactions between people and objects.
3. Identify and include any important text or dialogue that appears on screen.
4. Provide a clear, structured summary that captures the essence of the video.
5. Be concise but comprehensive, avoiding redundant descriptions of the same scene.
6. Keep the summary under 1000 words.
7. The output must be in {{language}}.
</Requirements>`
vlmSummaryUserPromptTemplate = `Here are the video frame descriptions:
<Input>
{{descriptions}}
</Input>
Please provide the summary based on the requirements and input.`
)
// VideoMultimodalService handles video:multimodal asynq tasks.
// It reads videos from storage (via FileService for provider:// URLs),
// extracts frames using FFmpeg, performs VLM analysis and ASR transcription,
// and creates child chunks.
type VideoMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewVideoMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &VideoMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
// Handle implements asynq handler for TypeVideoMultimodal.
func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.VideoMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal video multimodal payload: %w", err)
}
logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v",
payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
var videoBytes []byte
videoBytes, err := s.readVideoBytes(ctx, payload)
if err != nil {
return fmt.Errorf("read video bytes: %w", err)
}
if !payload.EnableVLM && !payload.EnableASR {
logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do")
return nil
}
var extractor video.Extractor
var extErr error
if payload.EnableVLM || payload.EnableASR {
extractor, extErr = video.NewExtractor(nil)
if extErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable: %v", extErr)
}
}
var wg sync.WaitGroup
type FrameAnalysis struct {
StartTimestamp float64
EndTimestamp float64
Description string
}
var frameAnalyses []FrameAnalysis
var asrSegments []asr.Segment
var mu sync.Mutex // Mutex to protect concurrent access if needed
lang := payload.Language
if lang == "" {
lang = "zh-CN"
}
batchPrompt := strings.ReplaceAll(vlmBatchPromptTemplate, "{{language}}", lang)
if payload.EnableVLM {
wg.Add(1)
go func() {
defer wg.Done()
var frames []*video.Frame
if extractor == nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled")
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
} else {
logger.Debugf(ctx, "[VideoMultimodal] Successfully extracted %d frames from video", len(frames))
}
}
if len(frames) == 0 {
return
}
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
// Initialize the slice within the goroutine before appending
localAnalyses := make([]FrameAnalysis, 0)
var previousDescription string
// Batch size configuration (number of frames per API call)
batchSize := 3
for i := 0; i < len(frames); i += batchSize {
end := i + batchSize
if end > len(frames) {
end = len(frames)
}
batchFrames := frames[i:end]
var batchImages [][]byte
for _, f := range batchFrames {
batchImages = append(batchImages, f.ImageData)
}
startTs := batchFrames[0].Timestamp
endTs := batchFrames[len(batchFrames)-1].Timestamp
logger.Debugf(ctx, "[VideoMultimodal] Starting VLM analysis for frames %d-%d/%d (%.2fs - %.2fs)", i+1, end, len(frames), startTs, endTs)
currentPrompt := strings.ReplaceAll(batchPrompt, "{{count}}", fmt.Sprintf("%d", len(batchFrames)))
if i == 0 {
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", vlmFirstBatchInstruction)
} else {
nextInstruction := strings.ReplaceAll(vlmNextBatchInstruction, "{{previous_description}}", previousDescription)
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", nextInstruction)
}
desc, descErr := vlmMdl.Predict(ctx, batchImages, currentPrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frames %d-%d at %.2fs - %.2fs: %v",
i+1, end, startTs, endTs, descErr)
continue
}
// Update previous description for the next iteration if there are changes
if !strings.Contains(strings.ToLower(desc), "no significant changes") {
previousDescription = desc
}
localAnalyses = append(localAnalyses, FrameAnalysis{
StartTimestamp: startTs,
EndTimestamp: endTs,
Description: desc,
})
logger.Infof(ctx, "[VideoMultimodal] Frames %d-%d/%d analyzed (%.2fs - %.2fs), desc len=%d",
i+1, end, len(frames), startTs, endTs, len(desc))
}
mu.Lock()
frameAnalyses = localAnalyses
mu.Unlock()
}
}()
}
if payload.EnableASR {
wg.Add(1)
go func() {
defer wg.Done()
audioBytes := videoBytes
fileName := "video.mp4"
if extractor != nil {
extractedAudio, err := extractor.ExtractAudio(ctx, videoBytes)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Audio extraction failed, falling back to original video: %v", err)
} else if len(extractedAudio) == 0 {
logger.Infof(ctx, "[VideoMultimodal] No audio stream found in video, skipping ASR transcription")
return
} else {
audioBytes = extractedAudio
fileName = "audio.mp3"
logger.Debugf(ctx, "[VideoMultimodal] Extracted audio track for ASR, size: %d bytes", len(audioBytes))
}
}
asrRes, asrErr := s.transcribeAudio(ctx, audioBytes, payload.KnowledgeBaseID, fileName)
if asrErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: kb=%s file=%s audioBytes=%d err=%v",
payload.KnowledgeBaseID, fileName, len(audioBytes), asrErr)
} else if asrRes == nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR returned nil result without error: kb=%s file=%s audioBytes=%d",
payload.KnowledgeBaseID, fileName, len(audioBytes))
} else {
asrSegments = asrRes.Segments
nSeg := len(asrRes.Segments)
nRunes := utf8.RuneCountInString(asrRes.Text)
var spanStart, spanEnd float64
if nSeg > 0 {
spanStart = asrRes.Segments[0].Start
spanEnd = asrRes.Segments[0].End
for _, seg := range asrRes.Segments[1:] {
if seg.Start < spanStart {
spanStart = seg.Start
}
if seg.End > spanEnd {
spanEnd = seg.End
}
}
}
const previewRunes = 200
preview := strings.TrimSpace(asrRes.Text)
if utf8.RuneCountInString(preview) > previewRunes {
preview = string([]rune(preview)[:previewRunes]) + "..."
}
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed: kb=%s file=%s audioBytes=%d textRunes=%d segments=%d timeline=%.2fs-%.2fs preview=%q",
payload.KnowledgeBaseID, fileName, len(audioBytes), nRunes, nSeg, spanStart, spanEnd, preview)
}
}()
}
logger.Debugf(ctx, "[VideoMultimodal] Waiting for VLM and ASR tasks to complete...")
wg.Wait()
logger.Debugf(ctx, "[VideoMultimodal] VLM and ASR tasks completed")
var videoSummary string
var combinedInput string
var kb *types.KnowledgeBase
if (len(frameAnalyses) > 0 || len(asrSegments) > 0) && (payload.EnableVLM || payload.EnableASR) {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
type TimelineEvent struct {
Start float64
End float64
Text string
Type string
}
var events []TimelineEvent
for _, f := range frameAnalyses {
events = append(events, TimelineEvent{Start: f.StartTimestamp, End: f.EndTimestamp, Text: f.Description, Type: "frame"})
}
for _, seg := range asrSegments {
events = append(events, TimelineEvent{Start: seg.Start, End: seg.End, Text: seg.Text, Type: "audio"})
}
sort.Slice(events, func(i, j int) bool {
return events[i].Start < events[j].Start
})
var combinedBuilder strings.Builder
for _, ev := range events {
if ev.Type == "frame" {
if ev.Start == ev.End {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs] Visual: %s\n", ev.Start, ev.Text))
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Visual: %s\n", ev.Start, ev.End, ev.Text))
}
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Audio: %s\n", ev.Start, ev.End, ev.Text))
}
}
combinedInput = combinedBuilder.String()
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for summary: %v", err)
videoSummary = combinedInput
} else {
chatMdl, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil || chatMdl == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get chat model for summary (modelID=%s): %v", kb.SummaryModelID, err)
videoSummary = combinedInput
} else {
// Limit combinedInput to avoid context length overflow.
// Using ~32000 runes to be safe for typical models.
if utf8.RuneCountInString(combinedInput) > 32000 {
runes := []rune(combinedInput)
combinedInput = string(runes[:32000]) + "\n...[truncated due to length]..."
}
systemPrompt := strings.ReplaceAll(vlmSummarySystemPromptTemplate, "{{language}}", lang)
userPrompt := strings.ReplaceAll(vlmSummaryUserPromptTemplate, "{{descriptions}}", combinedInput)
logger.Debugf(ctx, "[VideoMultimodal] Generating video summary with model %s, input length: %d chars", kb.SummaryModelID, len(combinedInput))
resp, sumErr := chatMdl.Chat(ctx, []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
}, nil)
if sumErr != nil || resp == nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
videoSummary = combinedInput
} else {
videoSummary = resp.Content
}
}
}
}
if kb == nil {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for video chunking: %v", err)
}
}
videoInfo := types.VideoInfo{
URL: payload.VideoURL,
}
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
newChunks := s.buildVideoChunks(payload, combinedInput, videoSummary, string(videoInfoJSON), kb)
if len(newChunks) == 0 {
// Even if VLM/ASR both failed, mark knowledge as completed
s.finalizeVideoKnowledge(ctx, payload, "")
return nil
}
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create video multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d",
c.ChunkType, c.ID, payload.VideoURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// For standalone video files, use summary as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
// Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled.
// During initial processChunks, question generation is skipped for video-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (frame descriptions, summary, ASR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) {
var err error
var videoBytes []byte
if types.ParseProviderScheme(payload.VideoURL) != "" {
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
if fileSvc == nil {
logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
payload.TenantID, payload.KnowledgeBaseID)
} else {
// provider:// scheme — read via FileService
reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL)
if getErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr)
} else {
videoBytes, err = io.ReadAll(reader)
reader.Close()
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err)
videoBytes = nil
}
}
}
}
if videoBytes == nil && payload.VideoLocalPath != "" {
videoBytes, err = os.ReadFile(payload.VideoLocalPath)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err)
videoBytes = nil
}
}
if videoBytes == nil {
videoBytes, err = downloadVideoFromURL(payload.VideoURL)
if err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err)
return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err)
}
logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes))
}
return videoBytes, nil
}
// downloadVideoFromURL downloads video bytes from an HTTP(S) URL.
func downloadVideoFromURL(videoURL string) ([]byte, error) {
return secutils.DownloadBytes(videoURL)
}
// transcribeAudio transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, audioBytes []byte, kbID string, fileName string) (*asr.TranscriptionResult, error) {
asrMdl, err := s.resolveASR(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("resolve ASR: %w", err)
}
return asrMdl.Transcribe(ctx, audioBytes, fileName)
}
// buildVideoChunks creates child chunks for video multimodal analysis results:
// - Video summary (split into multiple chunks if too long)
// - Timeline combined input (split into multiple chunks if too long)
func (s *VideoMultimodalService) buildVideoChunks(
payload types.VideoMultimodalPayload,
combinedInput string,
videoSummary string,
videoInfoJSON string,
kb *types.KnowledgeBase,
) []*types.Chunk {
var newChunks []*types.Chunk
var parentContentBuilder strings.Builder
if videoSummary != "" {
parentContentBuilder.WriteString("视频摘要:\n")
parentContentBuilder.WriteString(videoSummary)
parentContentBuilder.WriteString("\n\n")
}
if combinedInput != "" {
parentContentBuilder.WriteString("详细时间轴解析:\n")
parentContentBuilder.WriteString(combinedInput)
}
parentContent := strings.TrimSpace(parentContentBuilder.String())
if parentContent == "" {
parentContent = "该视频未解析出有效内容"
}
// Construct a Parent Chunk representing the entire video
parentChunkID := uuid.New().String()
newChunks = append(newChunks, &types.Chunk{
ID: parentChunkID,
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: parentContent,
ChunkType: types.ChunkTypeParentText,
ParentChunkID: payload.ChunkID, // Link back to the original text chunk if extracted from a document
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
var chunkCfg chunker.SplitterConfig
if kb != nil {
baseCfg := buildSplitterConfig(kb)
if kb.ChunkingConfig.EnableParentChild {
// If parent-child is globally enabled, these chunks act as child chunks
// for the newly created video parent chunk.
_, childCfg := buildParentChildConfigs(kb.ChunkingConfig, baseCfg)
chunkCfg = childCfg
} else {
chunkCfg = baseCfg
}
} else {
chunkCfg = chunker.SplitterConfig{
ChunkSize: 512,
ChunkOverlap: 64,
Separators: []string{"\n\n", "\n", "。"},
}
}
if combinedInput != "" {
inputChunks := chunker.SplitText(combinedInput, chunkCfg)
for _, sc := range inputChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
ParentChunkID: parentChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
}
if videoSummary != "" {
summaryChunks := chunker.SplitText(videoSummary, chunkCfg)
for _, sc := range summaryChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
}
return newChunks
}
// finalizeVideoKnowledge updates the knowledge after multimodal processing:
// - For standalone video files: sets Description from summary and marks ParseStatus as completed.
// - For videos extracted from documents: no-op (description comes from summary generation).
func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsVideoType(knowledge.FileType) {
return
}
if summary != "" {
knowledge.Description = summary
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created video chunks into the retrieval engine
// so they can participate in semantic search.
func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// resolveASR creates an asr.ASR instance for the given knowledge base.
func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
asrCfg := kb.ASRConfig
if !asrCfg.IsASREnabled() {
return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID)
}
return s.modelService.GetASRModel(ctx, asrCfg.ModelID)
}
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService {
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil || tenant == nil {
logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
return nil
}
provider := types.ParseProviderScheme(payload.VideoURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if kbErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
}
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
return nil
}
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the video knowledge.
func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}
-1
View File
@@ -175,7 +175,6 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
must(container.Provide(service.NewVideoMultimodalService, dig.Name("videoMultimodal")))
must(container.Provide(service.NewMessageService))
must(container.Provide(service.NewMCPServiceService))
@@ -28,10 +28,6 @@ var audioFormats = map[string]bool{
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
}
var videoFormats = map[string]bool{
"mp4": true, "mov": true, "avi": true, "mkv": true, "webm": true, "wmv": true, "flv": true,
}
func init() {
for k := range imageFormats {
simpleFormats[k] = true
@@ -39,9 +35,6 @@ func init() {
for k := range audioFormats {
simpleFormats[k] = true
}
for k := range videoFormats {
simpleFormats[k] = true
}
}
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
@@ -81,8 +74,6 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
return imageToResult(req.FileName, req.FileContent), nil
case audioFormats[ft]:
return audioToResult(req.FileName, req.FileContent), nil
case videoFormats[ft]:
return videoToResult(req.FileName, req.FileContent), nil
default:
return nil, fmt.Errorf("unsupported simple format: %s", ft)
}
@@ -122,11 +113,6 @@ func IsAudioFormat(fileType string) bool {
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// IsVideoFormat returns true if the file type is a recognized video format.
func IsVideoFormat(fileType string) bool {
return videoFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// audioToResult wraps a standalone audio file. The actual transcription is
// handled by the ASR model in the knowledge service pipeline. Here we just
// return a placeholder markdown with the raw bytes preserved for upstream
@@ -144,22 +130,6 @@ func audioToResult(fileName string, data []byte) *types.ReadResult {
}
}
// videoToResult wraps a standalone video file. The actual frame extraction,
// VLM analysis and ASR transcription is handled by the VideoMultimodalService
// in the knowledge service pipeline. Here we just return a placeholder markdown
// with the raw bytes preserved for upstream processing.
func videoToResult(fileName string, data []byte) *types.ReadResult {
if fileName == "" {
fileName = "video.mp4"
}
// Return a placeholder; the knowledge service will handle the actual processing.
return &types.ReadResult{
MarkdownContent: fmt.Sprintf("[Video file: %s]", fileName),
IsVideo: true,
VideoData: data,
}
}
// ensureOriginalImageRef checks whether the input file is an image and, if the
// returned markdown does not already contain a markdown image reference for it,
// prepends one and appends the raw bytes to imageRefs. This guarantees that
@@ -43,7 +43,7 @@ func (e *builtinEngine) Description() string {
return "DocReader built-in parser engine"
}
func (e *builtinEngine) FileTypes(_ bool) []string {
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
}
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
if docreaderConnected {
@@ -71,7 +71,7 @@ func (e *simpleEngine) Description() string {
return "Simple format & image parsing (no external service required)"
}
func (e *simpleEngine) FileTypes(_ bool) []string {
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
}
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
return true, ""
-98
View File
@@ -1,98 +0,0 @@
package video
import (
"context"
"io"
)
// Extractor defines the interface for video frame extraction operations.
type Extractor interface {
// ExtractFrames extracts frames from a video file.
// It returns a slice of Frame objects representing the extracted frames.
ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error)
// ExtractFramesToReader extracts frames from a video and returns an io.Reader
// for each frame's image data. This is useful for streaming processing.
ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error)
// ExtractAudio extracts the audio track from a video file and returns it as bytes.
ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error)
// GetVideoInfo retrieves basic information about a video file.
// It returns a map containing duration, width, height, etc.
GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error)
}
// Frame represents a single extracted frame from a video.
type Frame struct {
// Index is the sequential index of the frame (0-based).
Index int
// Timestamp is the time in seconds from the start of the video.
Timestamp float64
// ImageData contains the raw image bytes of the frame.
ImageData []byte
// Format specifies the image format (e.g., "jpeg", "png").
Format string
// Width is the width of the frame in pixels.
Width int
// Height is the height of the frame in pixels.
Height int
}
// ExtractOptions configures how frames are extracted from a video.
type ExtractOptions struct {
// FrameInterval specifies how often to extract frames (in seconds).
// For example, 1.0 means extract one frame per second.
FrameInterval float64
// MaxFrames specifies the maximum number of frames to extract.
// If 0, extract all frames according to FrameInterval.
MaxFrames int
// Format specifies the output image format for extracted frames.
// Supported formats: "jpeg" (default), "png".
Format string
// Quality specifies the image quality for JPEG output (1-100, default 80).
Quality int
// Width specifies the target width for resizing frames (0 = no resize).
Width int
// Height specifies the target height for resizing frames (0 = no resize).
// If only one dimension is specified, the aspect ratio is preserved.
Height int
}
// DefaultExtractOptions returns the default frame extraction options.
func DefaultExtractOptions() *ExtractOptions {
return &ExtractOptions{
FrameInterval: 1.0,
MaxFrames: 0,
Format: "jpeg",
Quality: 80,
Width: 0,
Height: 0,
}
}
// Config holds the configuration needed to create a video extractor.
type Config struct {
// FFMpegPath is the path to the ffmpeg executable.
// If empty, it will try to find ffmpeg in the system PATH.
FFMpegPath string
// TempDir is the directory to use for temporary files.
// If empty, the system default temporary directory is used.
TempDir string
}
// NewExtractor creates a new video frame extractor based on the provided configuration.
func NewExtractor(config *Config) (Extractor, error) {
return NewFFMpegExtractor(config)
}
-342
View File
@@ -1,342 +0,0 @@
package video
import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
)
// FFMpegExtractor implements the Extractor interface using FFmpeg.
type FFMpegExtractor struct {
ffmpegPath string
tempDir string
}
// NewFFMpegExtractor creates a new FFmpeg-based video extractor.
func NewFFMpegExtractor(config *Config) (Extractor, error) {
ffmpegPath := "ffmpeg"
tempDir := os.TempDir()
if config != nil {
if config.FFMpegPath != "" {
ffmpegPath = config.FFMpegPath
}
if config.TempDir != "" {
tempDir = config.TempDir
}
}
extractor := &FFMpegExtractor{ffmpegPath: ffmpegPath, tempDir: tempDir}
if err := extractor.checkFFmpeg(); err != nil {
return nil, err
}
return extractor, nil
}
// checkFFmpeg verifies that FFmpeg is available.
func (e *FFMpegExtractor) checkFFmpeg() error {
cmd := exec.Command(e.ffmpegPath, "-version")
_, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("ffmpeg not found or not executable: %w", err)
}
return nil
}
// ExtractFrames extracts frames from a video using FFmpeg.
func (e *FFMpegExtractor) ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error) {
if options == nil {
options = DefaultExtractOptions()
}
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
tempFrameDir, err := os.MkdirTemp(e.tempDir, "video-frames-")
if err != nil {
return nil, fmt.Errorf("failed to create temp frame directory: %w", err)
}
defer os.RemoveAll(tempFrameDir)
if _, err := e.getVideoDuration(ctx, tempVideoPath); err != nil {
logger.Warnf(ctx, "[VideoFFMpeg] Failed to get video duration: %v", err)
}
if err := e.extractFramesWithFFmpeg(ctx, tempVideoPath, tempFrameDir, options); err != nil {
return nil, err
}
return e.readExtractedFrames(ctx, tempFrameDir, options)
}
// ExtractFramesToReaders extracts frames and returns them as io.Readers.
func (e *FFMpegExtractor) ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error) {
frames, err := e.ExtractFrames(ctx, videoBytes, options)
if err != nil {
return nil, err
}
readers := make([]io.Reader, 0, len(frames))
for _, frame := range frames {
readers = append(readers, bytes.NewReader(frame.ImageData))
}
return readers, nil
}
// writeTempVideo writes the video bytes to a temporary file.
func (e *FFMpegExtractor) writeTempVideo(videoBytes []byte) (string, error) {
tempFile, err := os.CreateTemp(e.tempDir, "video-input-*.mp4")
if err != nil {
return "", fmt.Errorf("failed to create temp video file: %w", err)
}
defer tempFile.Close()
if _, err := tempFile.Write(videoBytes); err != nil {
os.Remove(tempFile.Name())
return "", fmt.Errorf("failed to write video bytes: %w", err)
}
return tempFile.Name(), nil
}
// getVideoDuration gets the duration of a video using FFmpeg.
func (e *FFMpegExtractor) getVideoDuration(ctx context.Context, videoPath string) (float64, error) {
cmd := exec.CommandContext(ctx, e.ffmpegPath,
"-i", videoPath,
"-f", "null",
"-")
output, err := cmd.CombinedOutput()
if err != nil {
return 0, err
}
outputStr := string(output)
durationStr := ""
lines := strings.Split(outputStr, "\n")
for _, line := range lines {
if strings.Contains(line, "Duration:") {
parts := strings.Split(line, "Duration:")
if len(parts) > 1 {
durationPart := strings.TrimSpace(parts[1])
durationStr = strings.Split(durationPart, ",")[0]
break
}
}
}
if durationStr == "" {
return 0, fmt.Errorf("could not parse duration")
}
timeParts := strings.Split(durationStr, ":")
if len(timeParts) != 3 {
return 0, fmt.Errorf("invalid duration format: %s", durationStr)
}
hours, _ := strconv.Atoi(timeParts[0])
minutes, _ := strconv.Atoi(timeParts[1])
seconds, _ := strconv.ParseFloat(timeParts[2], 64)
totalSeconds := float64(hours*3600) + float64(minutes*60) + seconds
return totalSeconds, nil
}
// extractFramesWithFFmpeg uses FFmpeg to extract frames from a video.
func (e *FFMpegExtractor) extractFramesWithFFmpeg(ctx context.Context, videoPath, outputDir string, options *ExtractOptions) error {
filters := []string{fmt.Sprintf("fps=1/%f", options.FrameInterval)}
if options.Width > 0 || options.Height > 0 {
scaleFilter := "scale="
if options.Width > 0 {
scaleFilter += strconv.Itoa(options.Width)
} else {
scaleFilter += "-1"
}
scaleFilter += ":"
if options.Height > 0 {
scaleFilter += strconv.Itoa(options.Height)
} else {
scaleFilter += "-1"
}
filters = append(filters, scaleFilter)
}
args := []string{
"-i", videoPath,
"-vf", strings.Join(filters, ","),
}
quality := options.Quality
if quality < 1 {
quality = 1
} else if quality > 100 {
quality = 100
}
if options.Format == "jpeg" {
args = append(args, "-q:v", strconv.Itoa(quality))
}
outputPattern := filepath.Join(outputDir, "frame-%05d."+options.Format)
args = append(args, outputPattern)
cmd := exec.CommandContext(ctx, e.ffmpegPath, args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("ffmpeg extraction failed: %w, stderr: %s", err, stderr.String())
}
logger.Infof(ctx, "[VideoFFMpeg] Extracted frames to: %s", outputDir)
return nil
}
// readExtractedFrames reads the extracted frames from the temporary directory.
func (e *FFMpegExtractor) readExtractedFrames(ctx context.Context, frameDir string, options *ExtractOptions) ([]*Frame, error) {
files, err := filepath.Glob(filepath.Join(frameDir, "frame-*."+options.Format))
if err != nil {
return nil, fmt.Errorf("failed to list frame files: %w", err)
}
if len(files) == 0 {
return nil, fmt.Errorf("no frames were extracted")
}
frames := make([]*Frame, 0, len(files))
for i, filePath := range files {
imgData, err := os.ReadFile(filePath)
if err != nil {
logger.Warnf(ctx, "[VideoFFMpeg] Failed to read frame %d: %v", i, err)
continue
}
frame := &Frame{
Index: i,
Timestamp: float64(i) * options.FrameInterval,
ImageData: imgData,
Format: options.Format,
Width: options.Width,
Height: options.Height,
}
frames = append(frames, frame)
if options.MaxFrames > 0 && len(frames) >= options.MaxFrames {
break
}
}
if len(frames) == 0 {
return nil, fmt.Errorf("no frames could be read")
}
logger.Infof(ctx, "[VideoFFMpeg] Successfully read %d frames", len(frames))
return frames, nil
}
// ExtractAudio extracts the audio track from a video using FFmpeg.
func (e *FFMpegExtractor) ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error) {
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
tempAudioFile, err := os.CreateTemp(e.tempDir, "audio-output-*.mp3")
if err != nil {
return nil, fmt.Errorf("failed to create temp audio file: %w", err)
}
tempAudioPath := tempAudioFile.Name()
tempAudioFile.Close()
defer os.Remove(tempAudioPath)
cmd := exec.CommandContext(ctx, e.ffmpegPath,
"-i", tempVideoPath,
"-vn",
"-c:a", "libmp3lame",
"-q:a", "2",
"-y",
tempAudioPath,
)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
if strings.Contains(stderr.String(), "Output file does not contain any stream") {
logger.Infof(ctx, "[VideoFFMpeg] Video does not contain an audio stream")
return nil, nil
}
return nil, fmt.Errorf("ffmpeg audio extraction failed: %w, stderr: %s", err, stderr.String())
}
audioBytes, err := os.ReadFile(tempAudioPath)
if err != nil {
return nil, fmt.Errorf("failed to read extracted audio: %w", err)
}
logger.Infof(ctx, "[VideoFFMpeg] Extracted audio, size: %d bytes", len(audioBytes))
return audioBytes, nil
}
// GetVideoInfo returns basic information about a video.
func (e *FFMpegExtractor) GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) {
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
duration, err := e.getVideoDuration(ctx, tempVideoPath)
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, e.ffmpegPath, "-i", tempVideoPath, "-f", "null", "-")
output, _ := cmd.CombinedOutput()
outputStr := string(output)
width, height := 0, 0
lines := strings.Split(outputStr, "\n")
for _, line := range lines {
if strings.Contains(line, "Video:") {
parts := strings.Split(line, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
if strings.Contains(part, "x") && !strings.Contains(part, "SAR") {
resParts := strings.Split(part, "x")
if len(resParts) == 2 {
width, _ = strconv.Atoi(strings.TrimSpace(resParts[0]))
heightStr := strings.TrimSpace(resParts[1])
heightParts := strings.Split(heightStr, " ")
if len(heightParts) > 0 {
height, _ = strconv.Atoi(heightParts[0])
}
}
break
}
}
break
}
}
videoInfo := map[string]interface{}{
"duration": duration,
"width": width,
"height": height,
}
return videoInfo, nil
}
-4
View File
@@ -23,7 +23,6 @@ type AsynqTaskParams struct {
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
VideoMultimodal interfaces.TaskHandler `name:"videoMultimodal"`
}
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
@@ -110,9 +109,6 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register image multimodal handler
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
// Register video multimodal handler
mux.HandleFunc(types.TypeVideoMultimodal, params.VideoMultimodal.Handle)
// Register data source sync handler
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
-2
View File
@@ -22,8 +22,6 @@ type ReadResult struct {
Error string
IsAudio bool // true when the result contains raw audio data needing ASR transcription
AudioData []byte // raw audio bytes for ASR processing
IsVideo bool // true when the result contains raw video data needing multimodal processing
VideoData []byte // raw video bytes for video multimodal processing
}
// ImageRef represents an image reference extracted from the document.
-15
View File
@@ -13,7 +13,6 @@ const (
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
TypeVideoMultimodal = "video:multimodal" // 视频多模态处理任务(抽帧 + VLM + ASR)
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
)
@@ -156,20 +155,6 @@ type ImageMultimodalPayload struct {
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// VideoMultimodalPayload represents the video multimodal processing task payload.
type VideoMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
VideoURL string `json:"video_url"` // provider:// URL (e.g. local://..., minio://...)
VideoLocalPath string `json:"video_local_path"` // deprecated: kept for backward compat with in-flight tasks
EnableVLM bool `json:"enable_vlm"` // enable VLM frame analysis
EnableASR bool `json:"enable_asr"` // enable ASR audio transcription
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
MaxFrames int `json:"max_frames,omitempty"` // optional: max frames to extract, 0 = no limit
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string