feat(video): remove video upload support and update related terminology

- Removed support for video file uploads across the application, including updates to the knowledge base and document processing services.
- Updated internationalization files to reflect the removal of video-related terminology, changing labels and descriptions to focus solely on audio processing.
- Adjusted file type handling to exclude video formats, ensuring users receive accurate feedback regarding supported file types.
- Enhanced error handling to provide clear messages when users attempt to upload unsupported video files.

This update streamlines the application by focusing on audio processing capabilities and improving user experience through clearer messaging.
This commit is contained in:
wizardchen
2026-04-14 22:57:58 +08:00
committed by lyingbug
parent 0fcab3a413
commit 86d26fa9bf
17 changed files with 78 additions and 1406 deletions
+21 -60
View File
@@ -203,6 +203,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName)
if IsVideoType(getFileType(fileName)) {
logger.Error(ctx, "Video file upload is not supported")
return nil, werrors.NewBadRequestError("暂不支持上传视频文件")
}
// Get knowledge base configuration
logger.Info(ctx, "Getting knowledge base configuration")
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
@@ -1482,8 +1487,6 @@ type ProcessChunksOptions struct {
QuestionCount int
EnableMultimodel bool
StoredImages []docparser.StoredImage
IsVideo bool // true when knowledge is a standalone video file
VideoFilePath string // storage path to video file (provider:// URL)
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
@@ -1889,14 +1892,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
// Skip summary/question generation for image-type and video-type knowledge — the text chunk
// is just a markdown reference, so LLM summary would be useless.
// The multimodal task will provide a caption/summary as the description instead.
// is just a markdown reference (or placeholder), so LLM summary would be useless.
// For images, the multimodal task provides a caption/summary as the description instead.
isImage := IsImageType(knowledge.FileType)
isVideo := IsVideoType(knowledge.FileType)
pendingMultimodal := (isImage && options.EnableMultimodel && len(options.StoredImages) > 0) ||
(isVideo && options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "")
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
// For image/video files with pending multimodal processing, keep "processing" status
// For image files with pending multimodal processing, keep "processing" status
// so the frontend waits until the description is ready before showing "completed".
if pendingMultimodal {
knowledge.ParseStatus = types.ParseStatusProcessing
@@ -1942,11 +1944,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks)
}
// Enqueue multimodal tasks for videos (async, non-blocking)
if options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "" {
s.enqueueVideoMultimodalTasks(ctx, knowledge, kb, options.VideoFilePath, chunks)
}
// Update tenant's storage usage
tenantInfo.StorageUsed += totalStorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
@@ -3033,8 +3030,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
"mp3", "wav", "m4a", "flac", "ogg",
"mp4", "mov", "avi", "mkv", "webm", "wmv", "flv":
"mp3", "wav", "m4a", "flac", "ogg":
return true
default:
return false
@@ -7706,6 +7702,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
// 视频文件不再支持入库解析
if payload.FilePath != "" && IsVideoType(payload.FileType) {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument video not supported")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "暂不支持视频文件"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
var convertResult *types.ReadResult
var chunks []types.ParsedChunk
@@ -7920,8 +7927,6 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
QuestionCount: payload.QuestionCount,
EnableMultimodel: payload.EnableMultimodel,
StoredImages: storedImages,
IsVideo: convertResult != nil && convertResult.IsVideo,
VideoFilePath: payload.FilePath,
}
if kb.ChunkingConfig.EnableParentChild {
@@ -8150,50 +8155,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
}
}
// enqueueVideoMultimodalTasks enqueues asynq tasks for multimodal video processing.
func (s *knowledgeService) enqueueVideoMultimodalTasks(
ctx context.Context,
knowledge *types.Knowledge,
kb *types.KnowledgeBase,
videoFilePath string,
chunks []types.ParsedChunk,
) {
if s.task == nil || videoFilePath == "" {
return
}
// Get the first chunk's ID as the parent chunk
chunkID := ""
if len(chunks) > 0 {
chunkID = chunks[0].ChunkID
}
lang, _ := types.LanguageFromContext(ctx)
payload := types.VideoMultimodalPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kb.ID,
ChunkID: chunkID,
VideoURL: videoFilePath,
EnableVLM: true,
EnableASR: true,
Language: lang,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Warnf(ctx, "Failed to marshal video multimodal payload: %v", err)
return
}
task := asynq.NewTask(types.TypeVideoMultimodal, payloadBytes)
if _, err := s.task.Enqueue(task); err != nil {
logger.Warnf(ctx, "Failed to enqueue video multimodal task for %s: %v", videoFilePath, err)
} else {
logger.Infof(ctx, "Enqueued video:multimodal task for %s", videoFilePath)
}
}
// ProcessFAQImport handles Asynq FAQ import tasks (including dry run mode)
func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error {
var payload types.FAQImportPayload
@@ -1,790 +0,0 @@
package service
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/video"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmBatchPromptTemplate = `Analyze this sequence of {{count}} video frames and provide a detailed description in {{language}}.
{{previous_batch_instruction}}
<Requirements>
1. The images are provided in chronological order.
2. Describe visible objects, people, and their actions.
3. Describe the scene and environment.
4. Extract and describe any text on the screen.
5. Note any notable events or changes across these frames.
6. Be concise but comprehensive, keeping the word count under 150 words.
7. The output must be in {{language}}.
</Requirements>`
vlmFirstBatchInstruction = "<PreviousContext>\nNone. These are the first frames of the video. Describe them completely.\n</PreviousContext>"
vlmNextBatchInstruction = "<PreviousContext>\n{{previous_description}}\n</PreviousContext>\n\nThese are the subsequent frames. Focus ONLY on what has changed or what is new compared to the previous context description provided above. If there are no significant changes, just state 'No significant changes'."
vlmSummarySystemPromptTemplate = `Please summarize the provided video frame descriptions into a coherent video summary in {{language}}.
<Requirements>
1. The descriptions are ordered chronologically by timestamp. Please maintain this temporal flow.
2. Focus on the main storyline, key events, and the interactions between people and objects.
3. Identify and include any important text or dialogue that appears on screen.
4. Provide a clear, structured summary that captures the essence of the video.
5. Be concise but comprehensive, avoiding redundant descriptions of the same scene.
6. Keep the summary under 1000 words.
7. The output must be in {{language}}.
</Requirements>`
vlmSummaryUserPromptTemplate = `Here are the video frame descriptions:
<Input>
{{descriptions}}
</Input>
Please provide the summary based on the requirements and input.`
)
// VideoMultimodalService handles video:multimodal asynq tasks.
// It reads videos from storage (via FileService for provider:// URLs),
// extracts frames using FFmpeg, performs VLM analysis and ASR transcription,
// and creates child chunks.
type VideoMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewVideoMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &VideoMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
// Handle implements asynq handler for TypeVideoMultimodal.
func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.VideoMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal video multimodal payload: %w", err)
}
logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v",
payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
var videoBytes []byte
videoBytes, err := s.readVideoBytes(ctx, payload)
if err != nil {
return fmt.Errorf("read video bytes: %w", err)
}
if !payload.EnableVLM && !payload.EnableASR {
logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do")
return nil
}
var extractor video.Extractor
var extErr error
if payload.EnableVLM || payload.EnableASR {
extractor, extErr = video.NewExtractor(nil)
if extErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable: %v", extErr)
}
}
var wg sync.WaitGroup
type FrameAnalysis struct {
StartTimestamp float64
EndTimestamp float64
Description string
}
var frameAnalyses []FrameAnalysis
var asrSegments []asr.Segment
var mu sync.Mutex // Mutex to protect concurrent access if needed
lang := payload.Language
if lang == "" {
lang = "zh-CN"
}
batchPrompt := strings.ReplaceAll(vlmBatchPromptTemplate, "{{language}}", lang)
if payload.EnableVLM {
wg.Add(1)
go func() {
defer wg.Done()
var frames []*video.Frame
if extractor == nil {
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled")
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
} else {
options := video.DefaultExtractOptions()
if payload.MaxFrames > 0 {
options.MaxFrames = payload.MaxFrames
}
var extractErr error
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
if extractErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
frames = []*video.Frame{}
} else {
logger.Debugf(ctx, "[VideoMultimodal] Successfully extracted %d frames from video", len(frames))
}
}
if len(frames) == 0 {
return
}
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if vlmErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
} else {
// Initialize the slice within the goroutine before appending
localAnalyses := make([]FrameAnalysis, 0)
var previousDescription string
// Batch size configuration (number of frames per API call)
batchSize := 3
for i := 0; i < len(frames); i += batchSize {
end := i + batchSize
if end > len(frames) {
end = len(frames)
}
batchFrames := frames[i:end]
var batchImages [][]byte
for _, f := range batchFrames {
batchImages = append(batchImages, f.ImageData)
}
startTs := batchFrames[0].Timestamp
endTs := batchFrames[len(batchFrames)-1].Timestamp
logger.Debugf(ctx, "[VideoMultimodal] Starting VLM analysis for frames %d-%d/%d (%.2fs - %.2fs)", i+1, end, len(frames), startTs, endTs)
currentPrompt := strings.ReplaceAll(batchPrompt, "{{count}}", fmt.Sprintf("%d", len(batchFrames)))
if i == 0 {
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", vlmFirstBatchInstruction)
} else {
nextInstruction := strings.ReplaceAll(vlmNextBatchInstruction, "{{previous_description}}", previousDescription)
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", nextInstruction)
}
desc, descErr := vlmMdl.Predict(ctx, batchImages, currentPrompt)
if descErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frames %d-%d at %.2fs - %.2fs: %v",
i+1, end, startTs, endTs, descErr)
continue
}
// Update previous description for the next iteration if there are changes
if !strings.Contains(strings.ToLower(desc), "no significant changes") {
previousDescription = desc
}
localAnalyses = append(localAnalyses, FrameAnalysis{
StartTimestamp: startTs,
EndTimestamp: endTs,
Description: desc,
})
logger.Infof(ctx, "[VideoMultimodal] Frames %d-%d/%d analyzed (%.2fs - %.2fs), desc len=%d",
i+1, end, len(frames), startTs, endTs, len(desc))
}
mu.Lock()
frameAnalyses = localAnalyses
mu.Unlock()
}
}()
}
if payload.EnableASR {
wg.Add(1)
go func() {
defer wg.Done()
audioBytes := videoBytes
fileName := "video.mp4"
if extractor != nil {
extractedAudio, err := extractor.ExtractAudio(ctx, videoBytes)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Audio extraction failed, falling back to original video: %v", err)
} else if len(extractedAudio) == 0 {
logger.Infof(ctx, "[VideoMultimodal] No audio stream found in video, skipping ASR transcription")
return
} else {
audioBytes = extractedAudio
fileName = "audio.mp3"
logger.Debugf(ctx, "[VideoMultimodal] Extracted audio track for ASR, size: %d bytes", len(audioBytes))
}
}
asrRes, asrErr := s.transcribeAudio(ctx, audioBytes, payload.KnowledgeBaseID, fileName)
if asrErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: kb=%s file=%s audioBytes=%d err=%v",
payload.KnowledgeBaseID, fileName, len(audioBytes), asrErr)
} else if asrRes == nil {
logger.Warnf(ctx, "[VideoMultimodal] ASR returned nil result without error: kb=%s file=%s audioBytes=%d",
payload.KnowledgeBaseID, fileName, len(audioBytes))
} else {
asrSegments = asrRes.Segments
nSeg := len(asrRes.Segments)
nRunes := utf8.RuneCountInString(asrRes.Text)
var spanStart, spanEnd float64
if nSeg > 0 {
spanStart = asrRes.Segments[0].Start
spanEnd = asrRes.Segments[0].End
for _, seg := range asrRes.Segments[1:] {
if seg.Start < spanStart {
spanStart = seg.Start
}
if seg.End > spanEnd {
spanEnd = seg.End
}
}
}
const previewRunes = 200
preview := strings.TrimSpace(asrRes.Text)
if utf8.RuneCountInString(preview) > previewRunes {
preview = string([]rune(preview)[:previewRunes]) + "..."
}
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed: kb=%s file=%s audioBytes=%d textRunes=%d segments=%d timeline=%.2fs-%.2fs preview=%q",
payload.KnowledgeBaseID, fileName, len(audioBytes), nRunes, nSeg, spanStart, spanEnd, preview)
}
}()
}
logger.Debugf(ctx, "[VideoMultimodal] Waiting for VLM and ASR tasks to complete...")
wg.Wait()
logger.Debugf(ctx, "[VideoMultimodal] VLM and ASR tasks completed")
var videoSummary string
var combinedInput string
var kb *types.KnowledgeBase
if (len(frameAnalyses) > 0 || len(asrSegments) > 0) && (payload.EnableVLM || payload.EnableASR) {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
type TimelineEvent struct {
Start float64
End float64
Text string
Type string
}
var events []TimelineEvent
for _, f := range frameAnalyses {
events = append(events, TimelineEvent{Start: f.StartTimestamp, End: f.EndTimestamp, Text: f.Description, Type: "frame"})
}
for _, seg := range asrSegments {
events = append(events, TimelineEvent{Start: seg.Start, End: seg.End, Text: seg.Text, Type: "audio"})
}
sort.Slice(events, func(i, j int) bool {
return events[i].Start < events[j].Start
})
var combinedBuilder strings.Builder
for _, ev := range events {
if ev.Type == "frame" {
if ev.Start == ev.End {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs] Visual: %s\n", ev.Start, ev.Text))
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Visual: %s\n", ev.Start, ev.End, ev.Text))
}
} else {
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Audio: %s\n", ev.Start, ev.End, ev.Text))
}
}
combinedInput = combinedBuilder.String()
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for summary: %v", err)
videoSummary = combinedInput
} else {
chatMdl, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil || chatMdl == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get chat model for summary (modelID=%s): %v", kb.SummaryModelID, err)
videoSummary = combinedInput
} else {
// Limit combinedInput to avoid context length overflow.
// Using ~32000 runes to be safe for typical models.
if utf8.RuneCountInString(combinedInput) > 32000 {
runes := []rune(combinedInput)
combinedInput = string(runes[:32000]) + "\n...[truncated due to length]..."
}
systemPrompt := strings.ReplaceAll(vlmSummarySystemPromptTemplate, "{{language}}", lang)
userPrompt := strings.ReplaceAll(vlmSummaryUserPromptTemplate, "{{descriptions}}", combinedInput)
logger.Debugf(ctx, "[VideoMultimodal] Generating video summary with model %s, input length: %d chars", kb.SummaryModelID, len(combinedInput))
resp, sumErr := chatMdl.Chat(ctx, []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
}, nil)
if sumErr != nil || resp == nil {
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
videoSummary = combinedInput
} else {
videoSummary = resp.Content
}
}
}
}
if kb == nil {
var err error
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for video chunking: %v", err)
}
}
videoInfo := types.VideoInfo{
URL: payload.VideoURL,
}
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
newChunks := s.buildVideoChunks(payload, combinedInput, videoSummary, string(videoInfoJSON), kb)
if len(newChunks) == 0 {
// Even if VLM/ASR both failed, mark knowledge as completed
s.finalizeVideoKnowledge(ctx, payload, "")
return nil
}
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create video multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d",
c.ChunkType, c.ID, payload.VideoURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// For standalone video files, use summary as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
// Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled.
// During initial processChunks, question generation is skipped for video-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (frame descriptions, summary, ASR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) {
var err error
var videoBytes []byte
if types.ParseProviderScheme(payload.VideoURL) != "" {
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
if fileSvc == nil {
logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
payload.TenantID, payload.KnowledgeBaseID)
} else {
// provider:// scheme — read via FileService
reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL)
if getErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr)
} else {
videoBytes, err = io.ReadAll(reader)
reader.Close()
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err)
videoBytes = nil
}
}
}
}
if videoBytes == nil && payload.VideoLocalPath != "" {
videoBytes, err = os.ReadFile(payload.VideoLocalPath)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err)
videoBytes = nil
}
}
if videoBytes == nil {
videoBytes, err = downloadVideoFromURL(payload.VideoURL)
if err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err)
return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err)
}
logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes))
}
return videoBytes, nil
}
// downloadVideoFromURL downloads video bytes from an HTTP(S) URL.
func downloadVideoFromURL(videoURL string) ([]byte, error) {
return secutils.DownloadBytes(videoURL)
}
// transcribeAudio transcribes audio from a video using ASR.
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, audioBytes []byte, kbID string, fileName string) (*asr.TranscriptionResult, error) {
asrMdl, err := s.resolveASR(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("resolve ASR: %w", err)
}
return asrMdl.Transcribe(ctx, audioBytes, fileName)
}
// buildVideoChunks creates child chunks for video multimodal analysis results:
// - Video summary (split into multiple chunks if too long)
// - Timeline combined input (split into multiple chunks if too long)
func (s *VideoMultimodalService) buildVideoChunks(
payload types.VideoMultimodalPayload,
combinedInput string,
videoSummary string,
videoInfoJSON string,
kb *types.KnowledgeBase,
) []*types.Chunk {
var newChunks []*types.Chunk
var parentContentBuilder strings.Builder
if videoSummary != "" {
parentContentBuilder.WriteString("视频摘要:\n")
parentContentBuilder.WriteString(videoSummary)
parentContentBuilder.WriteString("\n\n")
}
if combinedInput != "" {
parentContentBuilder.WriteString("详细时间轴解析:\n")
parentContentBuilder.WriteString(combinedInput)
}
parentContent := strings.TrimSpace(parentContentBuilder.String())
if parentContent == "" {
parentContent = "该视频未解析出有效内容"
}
// Construct a Parent Chunk representing the entire video
parentChunkID := uuid.New().String()
newChunks = append(newChunks, &types.Chunk{
ID: parentChunkID,
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: parentContent,
ChunkType: types.ChunkTypeParentText,
ParentChunkID: payload.ChunkID, // Link back to the original text chunk if extracted from a document
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
var chunkCfg chunker.SplitterConfig
if kb != nil {
baseCfg := buildSplitterConfig(kb)
if kb.ChunkingConfig.EnableParentChild {
// If parent-child is globally enabled, these chunks act as child chunks
// for the newly created video parent chunk.
_, childCfg := buildParentChildConfigs(kb.ChunkingConfig, baseCfg)
chunkCfg = childCfg
} else {
chunkCfg = baseCfg
}
} else {
chunkCfg = chunker.SplitterConfig{
ChunkSize: 512,
ChunkOverlap: 64,
Separators: []string{"\n\n", "\n", "。"},
}
}
if combinedInput != "" {
inputChunks := chunker.SplitText(combinedInput, chunkCfg)
for _, sc := range inputChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
ParentChunkID: parentChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
}
if videoSummary != "" {
summaryChunks := chunker.SplitText(videoSummary, chunkCfg)
for _, sc := range summaryChunks {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: sc.Content,
ChunkType: types.ChunkTypeText,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
VideoInfo: videoInfoJSON,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
}
return newChunks
}
// finalizeVideoKnowledge updates the knowledge after multimodal processing:
// - For standalone video files: sets Description from summary and marks ParseStatus as completed.
// - For videos extracted from documents: no-op (description comes from summary generation).
func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsVideoType(knowledge.FileType) {
return
}
if summary != "" {
knowledge.Description = summary
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created video chunks into the retrieval engine
// so they can participate in semantic search.
func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// resolveASR creates an asr.ASR instance for the given knowledge base.
func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
asrCfg := kb.ASRConfig
if !asrCfg.IsASREnabled() {
return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID)
}
return s.modelService.GetASRModel(ctx, asrCfg.ModelID)
}
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService {
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil || tenant == nil {
logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
return nil
}
provider := types.ParseProviderScheme(payload.VideoURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if kbErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
}
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
return nil
}
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the video knowledge.
func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}