feat: add video multimodal functionality

This commit is contained in:
DaWesen
2026-04-08 19:52:28 +08:00
committed by lyingbug
parent 2b10cc54de
commit b80bdb1079
20 changed files with 1213 additions and 28 deletions
+75 -9
View File
@@ -1479,6 +1479,8 @@ type ProcessChunksOptions struct {
QuestionCount int
EnableMultimodel bool
StoredImages []docparser.StoredImage
IsVideo bool // true when knowledge is a standalone video file
VideoFilePath string // storage path to video file (provider:// URL)
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
@@ -1864,13 +1866,15 @@ func (s *knowledgeService) processChunks(ctx context.Context,
return
}
// Skip summary/question generation for image-type knowledge — the text chunk
// is just a markdown image reference, so LLM summary would be useless.
// The multimodal task will provide a caption as the description instead.
// Skip summary/question generation for image-type and video-type knowledge — the text chunk
// is just a markdown reference, so LLM summary would be useless.
// The multimodal task will provide a caption/summary as the description instead.
isImage := IsImageType(knowledge.FileType)
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
isVideo := IsVideoType(knowledge.FileType)
pendingMultimodal := (isImage && options.EnableMultimodel && len(options.StoredImages) > 0) ||
(isVideo && options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "")
// For image files with pending multimodal processing, keep "processing" status
// For image/video files with pending multimodal processing, keep "processing" status
// so the frontend waits until the description is ready before showing "completed".
if pendingMultimodal {
knowledge.ParseStatus = types.ParseStatusProcessing
@@ -1884,7 +1888,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
knowledge.UpdatedAt = now
// Set summary status based on whether summary generation will be triggered
if len(textChunks) > 0 && !isImage {
if len(textChunks) > 0 && !isImage && !isVideo {
knowledge.SummaryStatus = types.SummaryStatusPending
} else {
knowledge.SummaryStatus = types.SummaryStatusNone
@@ -1895,7 +1899,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
// Enqueue question generation task if enabled (async, non-blocking)
if options.EnableQuestionGeneration && len(textChunks) > 0 && !isImage {
if options.EnableQuestionGeneration && len(textChunks) > 0 && !isImage && !isVideo {
questionCount := options.QuestionCount
if questionCount <= 0 {
questionCount = 3
@@ -1907,7 +1911,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
// Enqueue summary generation task (async, non-blocking)
if len(textChunks) > 0 && !isImage {
if len(textChunks) > 0 && !isImage && !isVideo {
s.enqueueSummaryGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
}
@@ -1916,6 +1920,11 @@ func (s *knowledgeService) processChunks(ctx context.Context,
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks)
}
// Enqueue multimodal tasks for videos (async, non-blocking)
if options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "" {
s.enqueueVideoMultimodalTasks(ctx, knowledge, kb, options.VideoFilePath, chunks)
}
// Update tenant's storage usage
tenantInfo.StorageUsed += totalStorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
@@ -3002,7 +3011,8 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
"mp3", "wav", "m4a", "flac", "ogg":
"mp3", "wav", "m4a", "flac", "ogg",
"mp4", "mov", "avi", "mkv", "webm", "wmv", "flv":
return true
default:
return false
@@ -7424,6 +7434,16 @@ func IsAudioType(fileType string) bool {
}
}
// IsVideoType checks if a file type is a video format
func IsVideoType(fileType string) bool {
switch strings.ToLower(fileType) {
case "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv":
return true
default:
return false
}
}
// downloadFileFromURL downloads a remote file to a temp file and returns its binary content.
// payloadFileName and payloadFileType are in/out pointers: if they point to an empty string,
// the function resolves the value from Content-Disposition / URL path and writes it back.
@@ -7899,6 +7919,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
QuestionCount: payload.QuestionCount,
EnableMultimodel: payload.EnableMultimodel,
StoredImages: storedImages,
IsVideo: convertResult != nil && convertResult.IsVideo,
VideoFilePath: payload.FilePath,
}
if kb.ChunkingConfig.EnableParentChild {
@@ -8111,6 +8133,50 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
}
}
// enqueueVideoMultimodalTasks enqueues asynq tasks for multimodal video processing.
func (s *knowledgeService) enqueueVideoMultimodalTasks(
ctx context.Context,
knowledge *types.Knowledge,
kb *types.KnowledgeBase,
videoFilePath string,
chunks []types.ParsedChunk,
) {
if s.task == nil || videoFilePath == "" {
return
}
// Get the first chunk's ID as the parent chunk
chunkID := ""
if len(chunks) > 0 {
chunkID = chunks[0].ChunkID
}
lang, _ := types.LanguageFromContext(ctx)
payload := types.VideoMultimodalPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kb.ID,
ChunkID: chunkID,
VideoURL: videoFilePath,
EnableVLM: true,
EnableASR: true,
Language: lang,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Warnf(ctx, "Failed to marshal video multimodal payload: %v", err)
return
}
task := asynq.NewTask(types.TypeVideoMultimodal, payloadBytes)
if _, err := s.task.Enqueue(task); err != nil {
logger.Warnf(ctx, "Failed to enqueue video multimodal task for %s: %v", videoFilePath, err)
} else {
logger.Infof(ctx, "Enqueued video:multimodal task for %s", videoFilePath)
}
}
// ProcessFAQImport handles Asynq FAQ import tasks (including dry run mode)
func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error {
var payload types.FAQImportPayload
@@ -0,0 +1,596 @@
package service
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/video"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmFramePrompt = "Analyze this video frame and provide a detailed description in Chinese. Requirements:\n" +
"1. Describe visible objects, people, and their actions\n" +
"2. Describe the scene and environment\n" +
"3. Extract and describe any text on the screen\n" +
"4. Note any notable events or changes\n" +
"5. Be concise but comprehensive"
vlmSummaryPrompt = "Summarize the following video frame descriptions into a coherent video summary in Chinese. Requirements:\n" +
"1. The descriptions are ordered chronologically by timestamp\n" +
"2. Focus on the main storyline and key events\n" +
"3. Maintain the temporal flow of the video\n" +
"4. Be concise but comprehensive"
)
// VideoMultimodalService handles video:multimodal asynq tasks.
// It reads videos from storage (via FileService for provider:// URLs),
// extracts frames using FFmpeg, performs VLM analysis and ASR transcription,
// and creates child chunks.
type VideoMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewVideoMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &VideoMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
// Handle implements asynq handler for TypeVideoMultimodal.
func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.VideoMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal video multimodal payload: %w", err)
}
logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v",
payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
var videoBytes []byte
videoBytes, err := s.readVideoBytes(ctx, payload)
if err != nil {
return fmt.Errorf("read video bytes: %w", err)
}
if !payload.EnableVLM && !payload.EnableASR {
logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do")
return nil
}
var frames []*video.Frame
var extractor video.Extractor
if payload.EnableVLM {
var extErr error
extractor, extErr = video.NewExtractor(nil)
if extErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled: %v", extErr)
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
frames = []*video.Frame{}
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
}
}
} else {
logger.Infof(ctx, "[VideoMultimodal] VLM disabled, skipping frame extraction")
frames = []*video.Frame{}
}
frameDescriptions := make([]string, 0, len(frames))
if payload.EnableVLM {
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
for i, frame := range frames {
desc, descErr := vlmMdl.Predict(ctx, frame.ImageData, vlmFramePrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frame %d at %.2fs: %v",
i, frame.Timestamp, descErr)
continue
}
frameDesc := fmt.Sprintf("[%.2fs] %s", frame.Timestamp, desc)
frameDescriptions = append(frameDescriptions, frameDesc)
logger.Infof(ctx, "[VideoMultimodal] Frame %d analyzed at %.2fs, desc len=%d",
i, frame.Timestamp, len(desc))
}
}
}
var asrText string
if payload.EnableASR {
asrText, err = s.transcribeAudio(ctx, videoBytes, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: %v", err)
} else if asrText != "" {
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed, len=%d", len(asrText))
}
}
var videoSummary string
if len(frameDescriptions) > 0 && payload.EnableVLM {
vlmMdl, _ := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmMdl != nil {
combinedInput := strings.Join(frameDescriptions, "\n")
summary, sumErr := vlmMdl.Predict(ctx, []byte(combinedInput), vlmSummaryPrompt)
if sumErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
videoSummary = combinedInput
} else {
videoSummary = summary
}
} else {
videoSummary = strings.Join(frameDescriptions, "\n")
}
}
videoInfo := types.VideoInfo{
URL: payload.VideoURL,
FrameCount: len(frameDescriptions),
HasVLMAnalysis: len(frameDescriptions) > 0,
HasASR: asrText != "",
VideoSummary: videoSummary,
ASRText: asrText,
FrameDescriptions: frameDescriptions,
}
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
newChunks := s.buildVideoChunks(payload, frameDescriptions, videoSummary, asrText, string(videoInfoJSON))
if len(newChunks) == 0 {
// Even if VLM/ASR both failed, mark knowledge as completed
s.finalizeVideoKnowledge(ctx, payload, "")
return nil
}
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create video multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d",
c.ChunkType, c.ID, payload.VideoURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's VideoInfo (mirrors old docreader behaviour)
s.updateParentChunkVideoInfo(ctx, payload, videoInfo)
// For standalone video files, use summary as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
// Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled.
// During initial processChunks, question generation is skipped for video-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (frame descriptions, summary, ASR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) {
var err error
var videoBytes []byte
if types.ParseProviderScheme(payload.VideoURL) != "" {
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
if fileSvc == nil {
logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
payload.TenantID, payload.KnowledgeBaseID)
} else {
// provider:// scheme — read via FileService
reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL)
if getErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr)
} else {
videoBytes, err = io.ReadAll(reader)
reader.Close()
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err)
videoBytes = nil
}
}
}
}
if videoBytes == nil && payload.VideoLocalPath != "" {
videoBytes, err = os.ReadFile(payload.VideoLocalPath)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err)
videoBytes = nil
}
}
if videoBytes == nil {
videoBytes, err = downloadVideoFromURL(payload.VideoURL)
if err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err)
return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err)
}
logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes))
}
return videoBytes, nil
}
// downloadVideoFromURL downloads video bytes from an HTTP(S) URL.
func downloadVideoFromURL(videoURL string) ([]byte, error) {
return secutils.DownloadBytes(videoURL)
}
// transcribeAudio extracts and transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, videoBytes []byte, kbID string) (string, error) {
asrMdl, err := s.resolveASR(ctx, kbID)
if err != nil {
return "", fmt.Errorf("resolve ASR: %w", err)
}
return asrMdl.Transcribe(ctx, videoBytes, "video.mp4")
}
// buildVideoChunks creates child chunks for video multimodal analysis results:
// - Frame descriptions (one chunk per frame)
// - Video summary (single chunk)
// - ASR transcription (single chunk)
func (s *VideoMultimodalService) buildVideoChunks(
payload types.VideoMultimodalPayload,
frameDescriptions []string,
videoSummary string,
asrText string,
videoInfoJSON string,
) []*types.Chunk {
var newChunks []*types.Chunk
for _, desc := range frameDescriptions {
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: desc,
ChunkType: types.ChunkTypeVideoFrame,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
newChunks = append(newChunks, chunk)
}
if videoSummary != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: videoSummary,
ChunkType: types.ChunkTypeVideoCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if asrText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: asrText,
ChunkType: types.ChunkTypeVideoASR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
return newChunks
}
// finalizeVideoKnowledge updates the knowledge after multimodal processing:
// - For standalone video files: sets Description from summary and marks ParseStatus as completed.
// - For videos extracted from documents: no-op (description comes from summary generation).
func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsVideoType(knowledge.FileType) {
return
}
if summary != "" {
knowledge.Description = summary
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created video chunks into the retrieval engine
// so they can participate in semantic search.
func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
}
// updateParentChunkVideoInfo updates the parent text chunk's VideoInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full video metadata (URL, frame descriptions, summary, ASR).
func (s *VideoMultimodalService) updateParentChunkVideoInfo(
ctx context.Context,
payload types.VideoMultimodalPayload,
videoInfo types.VideoInfo,
) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.VideoInfo
if chunk.VideoInfo != "" {
_ = json.Unmarshal([]byte(chunk.VideoInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == videoInfo.URL {
existingInfos[i] = videoInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, videoInfo)
}
videoInfoJSON, _ := json.Marshal(existingInfos)
chunk.VideoInfo = string(videoInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update parent chunk %s VideoInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Updated parent chunk %s VideoInfo for video", chunk.ID)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// resolveASR creates an asr.ASR instance for the given knowledge base.
func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
asrCfg := kb.ASRConfig
if !asrCfg.IsASREnabled() {
return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID)
}
return s.modelService.GetASRModel(ctx, asrCfg.ModelID)
}
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService {
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil || tenant == nil {
logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
return nil
}
provider := types.ParseProviderScheme(payload.VideoURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if kbErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
}
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
return nil
}
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the video knowledge.
func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}