package service import ( "context" "encoding/json" "fmt" "io" "os" "strings" "time" filesvc "github.com/Tencent/WeKnora/internal/application/service/file" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/models/video" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" "github.com/hibiken/asynq" ) const ( vlmFramePrompt = "Analyze this video frame and provide a detailed description in Chinese. Requirements:\n" + "1. Describe visible objects, people, and their actions\n" + "2. Describe the scene and environment\n" + "3. Extract and describe any text on the screen\n" + "4. Note any notable events or changes\n" + "5. Be concise but comprehensive" vlmSummaryPrompt = "Summarize the following video frame descriptions into a coherent video summary in Chinese. Requirements:\n" + "1. The descriptions are ordered chronologically by timestamp\n" + "2. Focus on the main storyline and key events\n" + "3. Maintain the temporal flow of the video\n" + "4. Be concise but comprehensive" ) // VideoMultimodalService handles video:multimodal asynq tasks. // It reads videos from storage (via FileService for provider:// URLs), // extracts frames using FFmpeg, performs VLM analysis and ASR transcription, // and creates child chunks. type VideoMultimodalService struct { chunkService interfaces.ChunkService modelService interfaces.ModelService kbService interfaces.KnowledgeBaseService knowledgeRepo interfaces.KnowledgeRepository tenantRepo interfaces.TenantRepository retrieveEngine interfaces.RetrieveEngineRegistry ollamaService *ollama.OllamaService taskEnqueuer interfaces.TaskEnqueuer } func NewVideoMultimodalService( chunkService interfaces.ChunkService, modelService interfaces.ModelService, kbService interfaces.KnowledgeBaseService, knowledgeRepo interfaces.KnowledgeRepository, tenantRepo interfaces.TenantRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ollamaService *ollama.OllamaService, taskEnqueuer interfaces.TaskEnqueuer, ) interfaces.TaskHandler { return &VideoMultimodalService{ chunkService: chunkService, modelService: modelService, kbService: kbService, knowledgeRepo: knowledgeRepo, tenantRepo: tenantRepo, retrieveEngine: retrieveEngine, ollamaService: ollamaService, taskEnqueuer: taskEnqueuer, } } // Handle implements asynq handler for TypeVideoMultimodal. func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error { var payload types.VideoMultimodalPayload if err := json.Unmarshal(task.Payload(), &payload); err != nil { return fmt.Errorf("unmarshal video multimodal payload: %w", err) } logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v", payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } var videoBytes []byte videoBytes, err := s.readVideoBytes(ctx, payload) if err != nil { return fmt.Errorf("read video bytes: %w", err) } if !payload.EnableVLM && !payload.EnableASR { logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do") return nil } var frames []*video.Frame var extractor video.Extractor if payload.EnableVLM { var extErr error extractor, extErr = video.NewExtractor(nil) if extErr != nil { logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled: %v", extErr) logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis") frames = []*video.Frame{} } else { options := video.DefaultExtractOptions() if payload.MaxFrames > 0 { options.MaxFrames = payload.MaxFrames } var extractErr error frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options) if extractErr != nil { logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr) frames = []*video.Frame{} } } } else { logger.Infof(ctx, "[VideoMultimodal] VLM disabled, skipping frame extraction") frames = []*video.Frame{} } frameDescriptions := make([]string, 0, len(frames)) if payload.EnableVLM { vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID) if vlmErr != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr) } else { for i, frame := range frames { desc, descErr := vlmMdl.Predict(ctx, frame.ImageData, vlmFramePrompt) if descErr != nil { logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frame %d at %.2fs: %v", i, frame.Timestamp, descErr) continue } frameDesc := fmt.Sprintf("[%.2fs] %s", frame.Timestamp, desc) frameDescriptions = append(frameDescriptions, frameDesc) logger.Infof(ctx, "[VideoMultimodal] Frame %d analyzed at %.2fs, desc len=%d", i, frame.Timestamp, len(desc)) } } } var asrText string if payload.EnableASR { asrText, err = s.transcribeAudio(ctx, videoBytes, payload.KnowledgeBaseID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: %v", err) } else if asrText != "" { logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed, len=%d", len(asrText)) } } var videoSummary string if len(frameDescriptions) > 0 && payload.EnableVLM { vlmMdl, _ := s.resolveVLM(ctx, payload.KnowledgeBaseID) if vlmMdl != nil { combinedInput := strings.Join(frameDescriptions, "\n") summary, sumErr := vlmMdl.Predict(ctx, []byte(combinedInput), vlmSummaryPrompt) if sumErr != nil { logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr) videoSummary = combinedInput } else { videoSummary = summary } } else { videoSummary = strings.Join(frameDescriptions, "\n") } } videoInfo := types.VideoInfo{ URL: payload.VideoURL, FrameCount: len(frameDescriptions), HasVLMAnalysis: len(frameDescriptions) > 0, HasASR: asrText != "", VideoSummary: videoSummary, ASRText: asrText, FrameDescriptions: frameDescriptions, } videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo}) newChunks := s.buildVideoChunks(payload, frameDescriptions, videoSummary, asrText, string(videoInfoJSON)) if len(newChunks) == 0 { // Even if VLM/ASR both failed, mark knowledge as completed s.finalizeVideoKnowledge(ctx, payload, "") return nil } if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil { return fmt.Errorf("create video multimodal chunks: %w", err) } for _, c := range newChunks { logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d", c.ChunkType, c.ID, payload.VideoURL, len(c.Content)) } // Index chunks so they can be retrieved s.indexChunks(ctx, payload, newChunks) // Update the parent text chunk's VideoInfo (mirrors old docreader behaviour) s.updateParentChunkVideoInfo(ctx, payload, videoInfo) // For standalone video files, use summary as the knowledge description // and mark the knowledge as completed (it was kept in "processing" until now). s.finalizeVideoKnowledge(ctx, payload, videoSummary) // Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled. // During initial processChunks, question generation is skipped for video-type // knowledge because the text chunk is just a markdown reference. Now that we // have real textual content (frame descriptions, summary, ASR), we can generate questions. s.enqueueQuestionGenerationIfEnabled(ctx, payload) return nil } func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) { var err error var videoBytes []byte if types.ParseProviderScheme(payload.VideoURL) != "" { fileSvc := s.resolveFileServiceForPayload(ctx, payload) if fileSvc == nil { logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s", payload.TenantID, payload.KnowledgeBaseID) } else { // provider:// scheme — read via FileService reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL) if getErr != nil { logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr) } else { videoBytes, err = io.ReadAll(reader) reader.Close() if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err) videoBytes = nil } } } } if videoBytes == nil && payload.VideoLocalPath != "" { videoBytes, err = os.ReadFile(payload.VideoLocalPath) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err) videoBytes = nil } } if videoBytes == nil { videoBytes, err = downloadVideoFromURL(payload.VideoURL) if err != nil { logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err) return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err) } logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes)) } return videoBytes, nil } // downloadVideoFromURL downloads video bytes from an HTTP(S) URL. func downloadVideoFromURL(videoURL string) ([]byte, error) { return secutils.DownloadBytes(videoURL) } // transcribeAudio extracts and transcribes audio from a video using ASR. func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, videoBytes []byte, kbID string) (string, error) { asrMdl, err := s.resolveASR(ctx, kbID) if err != nil { return "", fmt.Errorf("resolve ASR: %w", err) } return asrMdl.Transcribe(ctx, videoBytes, "video.mp4") } // buildVideoChunks creates child chunks for video multimodal analysis results: // - Frame descriptions (one chunk per frame) // - Video summary (single chunk) // - ASR transcription (single chunk) func (s *VideoMultimodalService) buildVideoChunks( payload types.VideoMultimodalPayload, frameDescriptions []string, videoSummary string, asrText string, videoInfoJSON string, ) []*types.Chunk { var newChunks []*types.Chunk for _, desc := range frameDescriptions { chunk := &types.Chunk{ ID: uuid.New().String(), TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Content: desc, ChunkType: types.ChunkTypeVideoFrame, ParentChunkID: payload.ChunkID, IsEnabled: true, Flags: types.ChunkFlagRecommended, VideoInfo: videoInfoJSON, CreatedAt: time.Now(), UpdatedAt: time.Now(), } newChunks = append(newChunks, chunk) } if videoSummary != "" { newChunks = append(newChunks, &types.Chunk{ ID: uuid.New().String(), TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Content: videoSummary, ChunkType: types.ChunkTypeVideoCaption, ParentChunkID: payload.ChunkID, IsEnabled: true, Flags: types.ChunkFlagRecommended, VideoInfo: videoInfoJSON, CreatedAt: time.Now(), UpdatedAt: time.Now(), }) } if asrText != "" { newChunks = append(newChunks, &types.Chunk{ ID: uuid.New().String(), TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Content: asrText, ChunkType: types.ChunkTypeVideoASR, ParentChunkID: payload.ChunkID, IsEnabled: true, Flags: types.ChunkFlagRecommended, VideoInfo: videoInfoJSON, CreatedAt: time.Now(), UpdatedAt: time.Now(), }) } return newChunks } // finalizeVideoKnowledge updates the knowledge after multimodal processing: // - For standalone video files: sets Description from summary and marks ParseStatus as completed. // - For videos extracted from documents: no-op (description comes from summary generation). func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) { knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err) return } if knowledge == nil { return } if !IsVideoType(knowledge.FileType) { return } if summary != "" { knowledge.Description = summary } knowledge.ParseStatus = types.ParseStatusCompleted knowledge.UpdatedAt = time.Now() if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err) } else { logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)", payload.KnowledgeID, len(knowledge.Description)) } } // indexChunks indexes the newly created video chunks into the retrieval engine // so they can participate in semantic search. func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) { kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil || kb == nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err) return } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err) return } tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err) return } engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err) return } indexInfoList := make([]*types.IndexInfo, 0, len(chunks)) for _, chunk := range chunks { indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) } if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil { logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err) return } // Mark chunks as indexed. // Must re-fetch from DB because the in-memory objects lack auto-generated fields // (e.g. seq_id), and GORM Save would overwrite them with zero values. for _, chunk := range chunks { dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err) continue } dbChunk.Status = int(types.ChunkStatusIndexed) if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err) } } logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL) } // updateParentChunkVideoInfo updates the parent text chunk's VideoInfo field, // replicating the behaviour of the old docreader flow where the parent chunk // carried the full video metadata (URL, frame descriptions, summary, ASR). func (s *VideoMultimodalService) updateParentChunkVideoInfo( ctx context.Context, payload types.VideoMultimodalPayload, videoInfo types.VideoInfo, ) { if payload.ChunkID == "" { return } chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err) return } var existingInfos []types.VideoInfo if chunk.VideoInfo != "" { _ = json.Unmarshal([]byte(chunk.VideoInfo), &existingInfos) } found := false for i, info := range existingInfos { if info.URL == videoInfo.URL { existingInfos[i] = videoInfo found = true break } } if !found { existingInfos = append(existingInfos, videoInfo) } videoInfoJSON, _ := json.Marshal(existingInfos) chunk.VideoInfo = string(videoInfoJSON) chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to update parent chunk %s VideoInfo: %v", chunk.ID, err) } else { logger.Infof(ctx, "[VideoMultimodal] Updated parent chunk %s VideoInfo for video", chunk.ID) } } // resolveVLM creates a vlm.VLM instance for the given knowledge base, // supporting both new-style (ModelID) and legacy (inline BaseURL) configs. func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) { kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) if err != nil { return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err) } if kb == nil { return nil, fmt.Errorf("knowledge base %s not found", kbID) } vlmCfg := kb.VLMConfig if !vlmCfg.IsEnabled() { return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID) } // New-style: resolve model through ModelService if vlmCfg.ModelID != "" { return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID) } // Legacy: create VLM from inline config return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService) } // resolveASR creates an asr.ASR instance for the given knowledge base. func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) { kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) if err != nil { return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err) } if kb == nil { return nil, fmt.Errorf("knowledge base %s not found", kbID) } asrCfg := kb.ASRConfig if !asrCfg.IsASREnabled() { return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID) } return s.modelService.GetASRModel(ctx, asrCfg.ModelID) } // resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs. func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService { tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil || tenant == nil { logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err) return nil } provider := types.ParseProviderScheme(payload.VideoURL) if provider == "" { kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if kbErr != nil { logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr) } else if kb != nil { provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider())) } } baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir) if svcErr != nil { logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr) return nil } return fileSvc } // enqueueQuestionGenerationIfEnabled checks if the knowledge base has question // generation enabled and, if so, enqueues a task for the video knowledge. func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) { if s.taskEnqueuer == nil { return } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil || kb == nil { return } if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled { return } questionCount := kb.QuestionGenerationConfig.QuestionCount if questionCount <= 0 { questionCount = 3 } if questionCount > 10 { questionCount = 10 } taskPayload := types.QuestionGenerationPayload{ TenantID: payload.TenantID, KnowledgeBaseID: payload.KnowledgeBaseID, KnowledgeID: payload.KnowledgeID, QuestionCount: questionCount, Language: payload.Language, } payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err) return } task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err) } else { logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)", payload.KnowledgeID, questionCount) } }