Files
Enterprise-Knowledge-Base/internal/application/service/video_multimodal.go
T
2026-04-09 19:19:42 +08:00

597 lines
21 KiB
Go

package service
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/video"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmFramePrompt = "Analyze this video frame and provide a detailed description in Chinese. Requirements:\n" +
"1. Describe visible objects, people, and their actions\n" +
"2. Describe the scene and environment\n" +
"3. Extract and describe any text on the screen\n" +
"4. Note any notable events or changes\n" +
"5. Be concise but comprehensive"
vlmSummaryPrompt = "Summarize the following video frame descriptions into a coherent video summary in Chinese. Requirements:\n" +
"1. The descriptions are ordered chronologically by timestamp\n" +
"2. Focus on the main storyline and key events\n" +
"3. Maintain the temporal flow of the video\n" +
"4. Be concise but comprehensive"
)
// VideoMultimodalService handles video:multimodal asynq tasks.
// It reads videos from storage (via FileService for provider:// URLs),
// extracts frames using FFmpeg, performs VLM analysis and ASR transcription,
// and creates child chunks.
type VideoMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewVideoMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &VideoMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
// Handle implements asynq handler for TypeVideoMultimodal.
func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.VideoMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal video multimodal payload: %w", err)
}
logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v",
payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
var videoBytes []byte
videoBytes, err := s.readVideoBytes(ctx, payload)
if err != nil {
return fmt.Errorf("read video bytes: %w", err)
}
if !payload.EnableVLM && !payload.EnableASR {
logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do")
return nil
}
var frames []*video.Frame
var extractor video.Extractor
if payload.EnableVLM {
var extErr error
extractor, extErr = video.NewExtractor(nil)
if extErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled: %v", extErr)
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
frames = []*video.Frame{}
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
}
}
} else {
logger.Infof(ctx, "[VideoMultimodal] VLM disabled, skipping frame extraction")
frames = []*video.Frame{}
}
frameDescriptions := make([]string, 0, len(frames))
if payload.EnableVLM {
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
for i, frame := range frames {
desc, descErr := vlmMdl.Predict(ctx, frame.ImageData, vlmFramePrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frame %d at %.2fs: %v",
i, frame.Timestamp, descErr)
continue
}
frameDesc := fmt.Sprintf("[%.2fs] %s", frame.Timestamp, desc)
frameDescriptions = append(frameDescriptions, frameDesc)
logger.Infof(ctx, "[VideoMultimodal] Frame %d analyzed at %.2fs, desc len=%d",
i, frame.Timestamp, len(desc))
}
}
}
var asrText string
if payload.EnableASR {
asrText, err = s.transcribeAudio(ctx, videoBytes, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: %v", err)
} else if asrText != "" {
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed, len=%d", len(asrText))
}
}
var videoSummary string
if len(frameDescriptions) > 0 && payload.EnableVLM {
vlmMdl, _ := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmMdl != nil {
combinedInput := strings.Join(frameDescriptions, "\n")
summary, sumErr := vlmMdl.Predict(ctx, []byte(combinedInput), vlmSummaryPrompt)
if sumErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
videoSummary = combinedInput
} else {
videoSummary = summary
}
} else {
videoSummary = strings.Join(frameDescriptions, "\n")
}
}
videoInfo := types.VideoInfo{
URL: payload.VideoURL,
FrameCount: len(frameDescriptions),
HasVLMAnalysis: len(frameDescriptions) > 0,
HasASR: asrText != "",
VideoSummary: videoSummary,
ASRText: asrText,
FrameDescriptions: frameDescriptions,
}
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
newChunks := s.buildVideoChunks(payload, frameDescriptions, videoSummary, asrText, string(videoInfoJSON))
if len(newChunks) == 0 {
// Even if VLM/ASR both failed, mark knowledge as completed
s.finalizeVideoKnowledge(ctx, payload, "")
return nil
}
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create video multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d",
c.ChunkType, c.ID, payload.VideoURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's VideoInfo (mirrors old docreader behaviour)
s.updateParentChunkVideoInfo(ctx, payload, videoInfo)
// For standalone video files, use summary as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
// Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled.
// During initial processChunks, question generation is skipped for video-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (frame descriptions, summary, ASR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) {
var err error
var videoBytes []byte
if types.ParseProviderScheme(payload.VideoURL) != "" {
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
if fileSvc == nil {
logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
payload.TenantID, payload.KnowledgeBaseID)
} else {
// provider:// scheme — read via FileService
reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL)
if getErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr)
} else {
videoBytes, err = io.ReadAll(reader)
reader.Close()
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err)
videoBytes = nil
}
}
}
}
if videoBytes == nil && payload.VideoLocalPath != "" {
videoBytes, err = os.ReadFile(payload.VideoLocalPath)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err)
videoBytes = nil
}
}
if videoBytes == nil {
videoBytes, err = downloadVideoFromURL(payload.VideoURL)
if err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err)
return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err)
}
logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes))
}
return videoBytes, nil
}
// downloadVideoFromURL downloads video bytes from an HTTP(S) URL.
func downloadVideoFromURL(videoURL string) ([]byte, error) {
return secutils.DownloadBytes(videoURL)
}
// transcribeAudio extracts and transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, videoBytes []byte, kbID string) (string, error) {
asrMdl, err := s.resolveASR(ctx, kbID)
if err != nil {
return "", fmt.Errorf("resolve ASR: %w", err)
}
return asrMdl.Transcribe(ctx, videoBytes, "video.mp4")
}
// buildVideoChunks creates child chunks for video multimodal analysis results:
// - Frame descriptions (one chunk per frame)
// - Video summary (single chunk)
// - ASR transcription (single chunk)
func (s *VideoMultimodalService) buildVideoChunks(
payload types.VideoMultimodalPayload,
frameDescriptions []string,
videoSummary string,
asrText string,
videoInfoJSON string,
) []*types.Chunk {
var newChunks []*types.Chunk
for _, desc := range frameDescriptions {
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: desc,
ChunkType: types.ChunkTypeVideoFrame,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
newChunks = append(newChunks, chunk)
}
if videoSummary != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: videoSummary,
ChunkType: types.ChunkTypeVideoCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if asrText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: asrText,
ChunkType: types.ChunkTypeVideoASR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
return newChunks
}
// finalizeVideoKnowledge updates the knowledge after multimodal processing:
// - For standalone video files: sets Description from summary and marks ParseStatus as completed.
// - For videos extracted from documents: no-op (description comes from summary generation).
func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsVideoType(knowledge.FileType) {
return
}
if summary != "" {
knowledge.Description = summary
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created video chunks into the retrieval engine
// so they can participate in semantic search.
func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
}
// updateParentChunkVideoInfo updates the parent text chunk's VideoInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full video metadata (URL, frame descriptions, summary, ASR).
func (s *VideoMultimodalService) updateParentChunkVideoInfo(
ctx context.Context,
payload types.VideoMultimodalPayload,
videoInfo types.VideoInfo,
) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.VideoInfo
if chunk.VideoInfo != "" {
_ = json.Unmarshal([]byte(chunk.VideoInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == videoInfo.URL {
existingInfos[i] = videoInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, videoInfo)
}
videoInfoJSON, _ := json.Marshal(existingInfos)
chunk.VideoInfo = string(videoInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update parent chunk %s VideoInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Updated parent chunk %s VideoInfo for video", chunk.ID)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// resolveASR creates an asr.ASR instance for the given knowledge base.
func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
asrCfg := kb.ASRConfig
if !asrCfg.IsASREnabled() {
return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID)
}
return s.modelService.GetASRModel(ctx, asrCfg.ModelID)
}
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService {
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil || tenant == nil {
logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
return nil
}
provider := types.ParseProviderScheme(payload.VideoURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if kbErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
}
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
return nil
}
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the video knowledge.
func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}