feat(video): remove video upload support and update related terminology
- Removed support for video file uploads across the application, including updates to the knowledge base and document processing services. - Updated internationalization files to reflect the removal of video-related terminology, changing labels and descriptions to focus solely on audio processing. - Adjusted file type handling to exclude video formats, ensuring users receive accurate feedback regarding supported file types. - Enhanced error handling to provide clear messages when users attempt to upload unsupported video files. This update streamlines the application by focusing on audio processing capabilities and improving user experience through clearer messaging.
This commit is contained in:
@@ -203,6 +203,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName)
|
||||
|
||||
if IsVideoType(getFileType(fileName)) {
|
||||
logger.Error(ctx, "Video file upload is not supported")
|
||||
return nil, werrors.NewBadRequestError("暂不支持上传视频文件")
|
||||
}
|
||||
|
||||
// Get knowledge base configuration
|
||||
logger.Info(ctx, "Getting knowledge base configuration")
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
|
||||
@@ -1482,8 +1487,6 @@ type ProcessChunksOptions struct {
|
||||
QuestionCount int
|
||||
EnableMultimodel bool
|
||||
StoredImages []docparser.StoredImage
|
||||
IsVideo bool // true when knowledge is a standalone video file
|
||||
VideoFilePath string // storage path to video file (provider:// URL)
|
||||
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
|
||||
// When set, the chunks passed to processChunks are child chunks, and each
|
||||
// child's ParentIndex references an entry in this slice.
|
||||
@@ -1889,14 +1892,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
}
|
||||
|
||||
// Skip summary/question generation for image-type and video-type knowledge — the text chunk
|
||||
// is just a markdown reference, so LLM summary would be useless.
|
||||
// The multimodal task will provide a caption/summary as the description instead.
|
||||
// is just a markdown reference (or placeholder), so LLM summary would be useless.
|
||||
// For images, the multimodal task provides a caption/summary as the description instead.
|
||||
isImage := IsImageType(knowledge.FileType)
|
||||
isVideo := IsVideoType(knowledge.FileType)
|
||||
pendingMultimodal := (isImage && options.EnableMultimodel && len(options.StoredImages) > 0) ||
|
||||
(isVideo && options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "")
|
||||
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
|
||||
|
||||
// For image/video files with pending multimodal processing, keep "processing" status
|
||||
// For image files with pending multimodal processing, keep "processing" status
|
||||
// so the frontend waits until the description is ready before showing "completed".
|
||||
if pendingMultimodal {
|
||||
knowledge.ParseStatus = types.ParseStatusProcessing
|
||||
@@ -1942,11 +1944,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks)
|
||||
}
|
||||
|
||||
// Enqueue multimodal tasks for videos (async, non-blocking)
|
||||
if options.EnableMultimodel && options.IsVideo && options.VideoFilePath != "" {
|
||||
s.enqueueVideoMultimodalTasks(ctx, knowledge, kb, options.VideoFilePath, chunks)
|
||||
}
|
||||
|
||||
// Update tenant's storage usage
|
||||
tenantInfo.StorageUsed += totalStorageSize
|
||||
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
|
||||
@@ -3033,8 +3030,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
func isValidFileType(filename string) bool {
|
||||
switch strings.ToLower(getFileType(filename)) {
|
||||
case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
|
||||
"mp3", "wav", "m4a", "flac", "ogg",
|
||||
"mp4", "mov", "avi", "mkv", "webm", "wmv", "flv":
|
||||
"mp3", "wav", "m4a", "flac", "ogg":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -7706,6 +7702,17 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// 视频文件不再支持入库解析
|
||||
if payload.FilePath != "" && IsVideoType(payload.FileType) {
|
||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
||||
Errorf("processDocument video not supported")
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "暂不支持视频文件"
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
|
||||
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
|
||||
var convertResult *types.ReadResult
|
||||
var chunks []types.ParsedChunk
|
||||
@@ -7920,8 +7927,6 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
QuestionCount: payload.QuestionCount,
|
||||
EnableMultimodel: payload.EnableMultimodel,
|
||||
StoredImages: storedImages,
|
||||
IsVideo: convertResult != nil && convertResult.IsVideo,
|
||||
VideoFilePath: payload.FilePath,
|
||||
}
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
@@ -8150,50 +8155,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueVideoMultimodalTasks enqueues asynq tasks for multimodal video processing.
|
||||
func (s *knowledgeService) enqueueVideoMultimodalTasks(
|
||||
ctx context.Context,
|
||||
knowledge *types.Knowledge,
|
||||
kb *types.KnowledgeBase,
|
||||
videoFilePath string,
|
||||
chunks []types.ParsedChunk,
|
||||
) {
|
||||
if s.task == nil || videoFilePath == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the first chunk's ID as the parent chunk
|
||||
chunkID := ""
|
||||
if len(chunks) > 0 {
|
||||
chunkID = chunks[0].ChunkID
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
payload := types.VideoMultimodalPayload{
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kb.ID,
|
||||
ChunkID: chunkID,
|
||||
VideoURL: videoFilePath,
|
||||
EnableVLM: true,
|
||||
EnableASR: true,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to marshal video multimodal payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeVideoMultimodal, payloadBytes)
|
||||
if _, err := s.task.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "Failed to enqueue video multimodal task for %s: %v", videoFilePath, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Enqueued video:multimodal task for %s", videoFilePath)
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessFAQImport handles Asynq FAQ import tasks (including dry run mode)
|
||||
func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.FAQImportPayload
|
||||
|
||||
@@ -1,790 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/asr"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/video"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
const (
|
||||
vlmBatchPromptTemplate = `Analyze this sequence of {{count}} video frames and provide a detailed description in {{language}}.
|
||||
|
||||
{{previous_batch_instruction}}
|
||||
|
||||
<Requirements>
|
||||
1. The images are provided in chronological order.
|
||||
2. Describe visible objects, people, and their actions.
|
||||
3. Describe the scene and environment.
|
||||
4. Extract and describe any text on the screen.
|
||||
5. Note any notable events or changes across these frames.
|
||||
6. Be concise but comprehensive, keeping the word count under 150 words.
|
||||
7. The output must be in {{language}}.
|
||||
</Requirements>`
|
||||
|
||||
vlmFirstBatchInstruction = "<PreviousContext>\nNone. These are the first frames of the video. Describe them completely.\n</PreviousContext>"
|
||||
vlmNextBatchInstruction = "<PreviousContext>\n{{previous_description}}\n</PreviousContext>\n\nThese are the subsequent frames. Focus ONLY on what has changed or what is new compared to the previous context description provided above. If there are no significant changes, just state 'No significant changes'."
|
||||
|
||||
vlmSummarySystemPromptTemplate = `Please summarize the provided video frame descriptions into a coherent video summary in {{language}}.
|
||||
|
||||
<Requirements>
|
||||
1. The descriptions are ordered chronologically by timestamp. Please maintain this temporal flow.
|
||||
2. Focus on the main storyline, key events, and the interactions between people and objects.
|
||||
3. Identify and include any important text or dialogue that appears on screen.
|
||||
4. Provide a clear, structured summary that captures the essence of the video.
|
||||
5. Be concise but comprehensive, avoiding redundant descriptions of the same scene.
|
||||
6. Keep the summary under 1000 words.
|
||||
7. The output must be in {{language}}.
|
||||
</Requirements>`
|
||||
|
||||
vlmSummaryUserPromptTemplate = `Here are the video frame descriptions:
|
||||
|
||||
<Input>
|
||||
{{descriptions}}
|
||||
</Input>
|
||||
|
||||
Please provide the summary based on the requirements and input.`
|
||||
)
|
||||
|
||||
// VideoMultimodalService handles video:multimodal asynq tasks.
|
||||
// It reads videos from storage (via FileService for provider:// URLs),
|
||||
// extracts frames using FFmpeg, performs VLM analysis and ASR transcription,
|
||||
// and creates child chunks.
|
||||
type VideoMultimodalService struct {
|
||||
chunkService interfaces.ChunkService
|
||||
modelService interfaces.ModelService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
tenantRepo interfaces.TenantRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ollamaService *ollama.OllamaService
|
||||
taskEnqueuer interfaces.TaskEnqueuer
|
||||
}
|
||||
|
||||
func NewVideoMultimodalService(
|
||||
chunkService interfaces.ChunkService,
|
||||
modelService interfaces.ModelService,
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ollamaService *ollama.OllamaService,
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
) interfaces.TaskHandler {
|
||||
return &VideoMultimodalService{
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
kbService: kbService,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ollamaService: ollamaService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
}
|
||||
}
|
||||
|
||||
// Handle implements asynq handler for TypeVideoMultimodal.
|
||||
func (s *VideoMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
var payload types.VideoMultimodalPayload
|
||||
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
|
||||
return fmt.Errorf("unmarshal video multimodal payload: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[VideoMultimodal] Processing video: chunk=%s, url=%s, vlm=%v, asr=%v",
|
||||
payload.ChunkID, payload.VideoURL, payload.EnableVLM, payload.EnableASR)
|
||||
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
||||
}
|
||||
|
||||
var videoBytes []byte
|
||||
videoBytes, err := s.readVideoBytes(ctx, payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read video bytes: %w", err)
|
||||
}
|
||||
|
||||
if !payload.EnableVLM && !payload.EnableASR {
|
||||
logger.Infof(ctx, "[VideoMultimodal] Both VLM and ASR are disabled, nothing to do")
|
||||
return nil
|
||||
}
|
||||
|
||||
var extractor video.Extractor
|
||||
var extErr error
|
||||
if payload.EnableVLM || payload.EnableASR {
|
||||
extractor, extErr = video.NewExtractor(nil)
|
||||
if extErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable: %v", extErr)
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
type FrameAnalysis struct {
|
||||
StartTimestamp float64
|
||||
EndTimestamp float64
|
||||
Description string
|
||||
}
|
||||
var frameAnalyses []FrameAnalysis
|
||||
var asrSegments []asr.Segment
|
||||
var mu sync.Mutex // Mutex to protect concurrent access if needed
|
||||
|
||||
lang := payload.Language
|
||||
if lang == "" {
|
||||
lang = "zh-CN"
|
||||
}
|
||||
batchPrompt := strings.ReplaceAll(vlmBatchPromptTemplate, "{{language}}", lang)
|
||||
|
||||
if payload.EnableVLM {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
var frames []*video.Frame
|
||||
if extractor == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] FFmpeg not found or not executable, frame extraction disabled")
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Note: Please install FFmpeg to enable VLM frame analysis")
|
||||
} else {
|
||||
options := video.DefaultExtractOptions()
|
||||
if payload.MaxFrames > 0 {
|
||||
options.MaxFrames = payload.MaxFrames
|
||||
}
|
||||
var extractErr error
|
||||
frames, extractErr = extractor.ExtractFrames(ctx, videoBytes, options)
|
||||
if extractErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Frame extraction failed: %v", extractErr)
|
||||
frames = []*video.Frame{}
|
||||
} else {
|
||||
logger.Debugf(ctx, "[VideoMultimodal] Successfully extracted %d frames from video", len(frames))
|
||||
}
|
||||
}
|
||||
|
||||
if len(frames) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
vlmMdl, vlmErr := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
if vlmErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to resolve VLM: %v", vlmErr)
|
||||
} else {
|
||||
// Initialize the slice within the goroutine before appending
|
||||
localAnalyses := make([]FrameAnalysis, 0)
|
||||
var previousDescription string
|
||||
|
||||
// Batch size configuration (number of frames per API call)
|
||||
batchSize := 3
|
||||
for i := 0; i < len(frames); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(frames) {
|
||||
end = len(frames)
|
||||
}
|
||||
batchFrames := frames[i:end]
|
||||
|
||||
var batchImages [][]byte
|
||||
for _, f := range batchFrames {
|
||||
batchImages = append(batchImages, f.ImageData)
|
||||
}
|
||||
|
||||
startTs := batchFrames[0].Timestamp
|
||||
endTs := batchFrames[len(batchFrames)-1].Timestamp
|
||||
|
||||
logger.Debugf(ctx, "[VideoMultimodal] Starting VLM analysis for frames %d-%d/%d (%.2fs - %.2fs)", i+1, end, len(frames), startTs, endTs)
|
||||
|
||||
currentPrompt := strings.ReplaceAll(batchPrompt, "{{count}}", fmt.Sprintf("%d", len(batchFrames)))
|
||||
if i == 0 {
|
||||
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", vlmFirstBatchInstruction)
|
||||
} else {
|
||||
nextInstruction := strings.ReplaceAll(vlmNextBatchInstruction, "{{previous_description}}", previousDescription)
|
||||
currentPrompt = strings.ReplaceAll(currentPrompt, "{{previous_batch_instruction}}", nextInstruction)
|
||||
}
|
||||
|
||||
desc, descErr := vlmMdl.Predict(ctx, batchImages, currentPrompt)
|
||||
if descErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] VLM analysis failed for frames %d-%d at %.2fs - %.2fs: %v",
|
||||
i+1, end, startTs, endTs, descErr)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update previous description for the next iteration if there are changes
|
||||
if !strings.Contains(strings.ToLower(desc), "no significant changes") {
|
||||
previousDescription = desc
|
||||
}
|
||||
|
||||
localAnalyses = append(localAnalyses, FrameAnalysis{
|
||||
StartTimestamp: startTs,
|
||||
EndTimestamp: endTs,
|
||||
Description: desc,
|
||||
})
|
||||
logger.Infof(ctx, "[VideoMultimodal] Frames %d-%d/%d analyzed (%.2fs - %.2fs), desc len=%d",
|
||||
i+1, end, len(frames), startTs, endTs, len(desc))
|
||||
}
|
||||
mu.Lock()
|
||||
frameAnalyses = localAnalyses
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if payload.EnableASR {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
audioBytes := videoBytes
|
||||
fileName := "video.mp4"
|
||||
if extractor != nil {
|
||||
extractedAudio, err := extractor.ExtractAudio(ctx, videoBytes)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Audio extraction failed, falling back to original video: %v", err)
|
||||
} else if len(extractedAudio) == 0 {
|
||||
logger.Infof(ctx, "[VideoMultimodal] No audio stream found in video, skipping ASR transcription")
|
||||
return
|
||||
} else {
|
||||
audioBytes = extractedAudio
|
||||
fileName = "audio.mp3"
|
||||
logger.Debugf(ctx, "[VideoMultimodal] Extracted audio track for ASR, size: %d bytes", len(audioBytes))
|
||||
}
|
||||
}
|
||||
|
||||
asrRes, asrErr := s.transcribeAudio(ctx, audioBytes, payload.KnowledgeBaseID, fileName)
|
||||
if asrErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] ASR transcription failed: kb=%s file=%s audioBytes=%d err=%v",
|
||||
payload.KnowledgeBaseID, fileName, len(audioBytes), asrErr)
|
||||
} else if asrRes == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] ASR returned nil result without error: kb=%s file=%s audioBytes=%d",
|
||||
payload.KnowledgeBaseID, fileName, len(audioBytes))
|
||||
} else {
|
||||
asrSegments = asrRes.Segments
|
||||
nSeg := len(asrRes.Segments)
|
||||
nRunes := utf8.RuneCountInString(asrRes.Text)
|
||||
var spanStart, spanEnd float64
|
||||
if nSeg > 0 {
|
||||
spanStart = asrRes.Segments[0].Start
|
||||
spanEnd = asrRes.Segments[0].End
|
||||
for _, seg := range asrRes.Segments[1:] {
|
||||
if seg.Start < spanStart {
|
||||
spanStart = seg.Start
|
||||
}
|
||||
if seg.End > spanEnd {
|
||||
spanEnd = seg.End
|
||||
}
|
||||
}
|
||||
}
|
||||
const previewRunes = 200
|
||||
preview := strings.TrimSpace(asrRes.Text)
|
||||
if utf8.RuneCountInString(preview) > previewRunes {
|
||||
preview = string([]rune(preview)[:previewRunes]) + "..."
|
||||
}
|
||||
logger.Infof(ctx, "[VideoMultimodal] ASR transcription completed: kb=%s file=%s audioBytes=%d textRunes=%d segments=%d timeline=%.2fs-%.2fs preview=%q",
|
||||
payload.KnowledgeBaseID, fileName, len(audioBytes), nRunes, nSeg, spanStart, spanEnd, preview)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
logger.Debugf(ctx, "[VideoMultimodal] Waiting for VLM and ASR tasks to complete...")
|
||||
wg.Wait()
|
||||
logger.Debugf(ctx, "[VideoMultimodal] VLM and ASR tasks completed")
|
||||
|
||||
var videoSummary string
|
||||
var combinedInput string
|
||||
var kb *types.KnowledgeBase
|
||||
if (len(frameAnalyses) > 0 || len(asrSegments) > 0) && (payload.EnableVLM || payload.EnableASR) {
|
||||
var err error
|
||||
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
|
||||
type TimelineEvent struct {
|
||||
Start float64
|
||||
End float64
|
||||
Text string
|
||||
Type string
|
||||
}
|
||||
var events []TimelineEvent
|
||||
for _, f := range frameAnalyses {
|
||||
events = append(events, TimelineEvent{Start: f.StartTimestamp, End: f.EndTimestamp, Text: f.Description, Type: "frame"})
|
||||
}
|
||||
for _, seg := range asrSegments {
|
||||
events = append(events, TimelineEvent{Start: seg.Start, End: seg.End, Text: seg.Text, Type: "audio"})
|
||||
}
|
||||
sort.Slice(events, func(i, j int) bool {
|
||||
return events[i].Start < events[j].Start
|
||||
})
|
||||
|
||||
var combinedBuilder strings.Builder
|
||||
for _, ev := range events {
|
||||
if ev.Type == "frame" {
|
||||
if ev.Start == ev.End {
|
||||
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs] Visual: %s\n", ev.Start, ev.Text))
|
||||
} else {
|
||||
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Visual: %s\n", ev.Start, ev.End, ev.Text))
|
||||
}
|
||||
} else {
|
||||
combinedBuilder.WriteString(fmt.Sprintf("[%.2fs - %.2fs] Audio: %s\n", ev.Start, ev.End, ev.Text))
|
||||
}
|
||||
}
|
||||
combinedInput = combinedBuilder.String()
|
||||
|
||||
if err != nil || kb == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for summary: %v", err)
|
||||
videoSummary = combinedInput
|
||||
} else {
|
||||
chatMdl, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
|
||||
if err != nil || chatMdl == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get chat model for summary (modelID=%s): %v", kb.SummaryModelID, err)
|
||||
videoSummary = combinedInput
|
||||
} else {
|
||||
// Limit combinedInput to avoid context length overflow.
|
||||
// Using ~32000 runes to be safe for typical models.
|
||||
if utf8.RuneCountInString(combinedInput) > 32000 {
|
||||
runes := []rune(combinedInput)
|
||||
combinedInput = string(runes[:32000]) + "\n...[truncated due to length]..."
|
||||
}
|
||||
|
||||
systemPrompt := strings.ReplaceAll(vlmSummarySystemPromptTemplate, "{{language}}", lang)
|
||||
userPrompt := strings.ReplaceAll(vlmSummaryUserPromptTemplate, "{{descriptions}}", combinedInput)
|
||||
|
||||
logger.Debugf(ctx, "[VideoMultimodal] Generating video summary with model %s, input length: %d chars", kb.SummaryModelID, len(combinedInput))
|
||||
resp, sumErr := chatMdl.Chat(ctx, []chat.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: userPrompt},
|
||||
}, nil)
|
||||
|
||||
if sumErr != nil || resp == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Video summarization failed: %v", sumErr)
|
||||
videoSummary = combinedInput
|
||||
} else {
|
||||
videoSummary = resp.Content
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if kb == nil {
|
||||
var err error
|
||||
kb, err = s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for video chunking: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
videoInfo := types.VideoInfo{
|
||||
URL: payload.VideoURL,
|
||||
}
|
||||
videoInfoJSON, _ := json.Marshal([]types.VideoInfo{videoInfo})
|
||||
|
||||
newChunks := s.buildVideoChunks(payload, combinedInput, videoSummary, string(videoInfoJSON), kb)
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
// Even if VLM/ASR both failed, mark knowledge as completed
|
||||
s.finalizeVideoKnowledge(ctx, payload, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
|
||||
return fmt.Errorf("create video multimodal chunks: %w", err)
|
||||
}
|
||||
for _, c := range newChunks {
|
||||
logger.Infof(ctx, "[VideoMultimodal] Created %s chunk %s for video %s, len=%d",
|
||||
c.ChunkType, c.ID, payload.VideoURL, len(c.Content))
|
||||
}
|
||||
|
||||
// Index chunks so they can be retrieved
|
||||
s.indexChunks(ctx, payload, newChunks)
|
||||
|
||||
// For standalone video files, use summary as the knowledge description
|
||||
// and mark the knowledge as completed (it was kept in "processing" until now).
|
||||
s.finalizeVideoKnowledge(ctx, payload, videoSummary)
|
||||
|
||||
// Enqueue question generation for the frame descriptions/summary/ASR content if KB has it enabled.
|
||||
// During initial processChunks, question generation is skipped for video-type
|
||||
// knowledge because the text chunk is just a markdown reference. Now that we
|
||||
// have real textual content (frame descriptions, summary, ASR), we can generate questions.
|
||||
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *VideoMultimodalService) readVideoBytes(ctx context.Context, payload types.VideoMultimodalPayload) ([]byte, error) {
|
||||
var err error
|
||||
var videoBytes []byte
|
||||
if types.ParseProviderScheme(payload.VideoURL) != "" {
|
||||
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
|
||||
if fileSvc == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
|
||||
payload.TenantID, payload.KnowledgeBaseID)
|
||||
} else {
|
||||
// provider:// scheme — read via FileService
|
||||
reader, getErr := fileSvc.GetFile(ctx, payload.VideoURL)
|
||||
if getErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] FileService.GetFile(%s) failed: %v", payload.VideoURL, getErr)
|
||||
} else {
|
||||
videoBytes, err = io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Read provider file %s failed: %v", payload.VideoURL, err)
|
||||
videoBytes = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if videoBytes == nil && payload.VideoLocalPath != "" {
|
||||
videoBytes, err = os.ReadFile(payload.VideoLocalPath)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Local file %s not available (%v), trying URL", payload.VideoLocalPath, err)
|
||||
videoBytes = nil
|
||||
}
|
||||
}
|
||||
if videoBytes == nil {
|
||||
videoBytes, err = downloadVideoFromURL(payload.VideoURL)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[VideoMultimodal] Failed to download video from URL %s: %v", payload.VideoURL, err)
|
||||
return nil, fmt.Errorf("read video from URL %s failed: %w", payload.VideoURL, err)
|
||||
}
|
||||
logger.Infof(ctx, "[VideoMultimodal] Video downloaded from URL, len=%d", len(videoBytes))
|
||||
}
|
||||
return videoBytes, nil
|
||||
}
|
||||
|
||||
// downloadVideoFromURL downloads video bytes from an HTTP(S) URL.
|
||||
func downloadVideoFromURL(videoURL string) ([]byte, error) {
|
||||
return secutils.DownloadBytes(videoURL)
|
||||
}
|
||||
|
||||
// transcribeAudio transcribes audio from a video using ASR.
|
||||
func (s *VideoMultimodalService) transcribeAudio(ctx context.Context, audioBytes []byte, kbID string, fileName string) (*asr.TranscriptionResult, error) {
|
||||
asrMdl, err := s.resolveASR(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve ASR: %w", err)
|
||||
}
|
||||
return asrMdl.Transcribe(ctx, audioBytes, fileName)
|
||||
}
|
||||
|
||||
// buildVideoChunks creates child chunks for video multimodal analysis results:
|
||||
// - Video summary (split into multiple chunks if too long)
|
||||
// - Timeline combined input (split into multiple chunks if too long)
|
||||
func (s *VideoMultimodalService) buildVideoChunks(
|
||||
payload types.VideoMultimodalPayload,
|
||||
combinedInput string,
|
||||
videoSummary string,
|
||||
videoInfoJSON string,
|
||||
kb *types.KnowledgeBase,
|
||||
) []*types.Chunk {
|
||||
var newChunks []*types.Chunk
|
||||
|
||||
var parentContentBuilder strings.Builder
|
||||
if videoSummary != "" {
|
||||
parentContentBuilder.WriteString("视频摘要:\n")
|
||||
parentContentBuilder.WriteString(videoSummary)
|
||||
parentContentBuilder.WriteString("\n\n")
|
||||
}
|
||||
if combinedInput != "" {
|
||||
parentContentBuilder.WriteString("详细时间轴解析:\n")
|
||||
parentContentBuilder.WriteString(combinedInput)
|
||||
}
|
||||
parentContent := strings.TrimSpace(parentContentBuilder.String())
|
||||
if parentContent == "" {
|
||||
parentContent = "该视频未解析出有效内容"
|
||||
}
|
||||
|
||||
// Construct a Parent Chunk representing the entire video
|
||||
parentChunkID := uuid.New().String()
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: parentChunkID,
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: parentContent,
|
||||
ChunkType: types.ChunkTypeParentText,
|
||||
ParentChunkID: payload.ChunkID, // Link back to the original text chunk if extracted from a document
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
VideoInfo: videoInfoJSON,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
|
||||
var chunkCfg chunker.SplitterConfig
|
||||
if kb != nil {
|
||||
baseCfg := buildSplitterConfig(kb)
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
// If parent-child is globally enabled, these chunks act as child chunks
|
||||
// for the newly created video parent chunk.
|
||||
_, childCfg := buildParentChildConfigs(kb.ChunkingConfig, baseCfg)
|
||||
chunkCfg = childCfg
|
||||
} else {
|
||||
chunkCfg = baseCfg
|
||||
}
|
||||
} else {
|
||||
chunkCfg = chunker.SplitterConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 64,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
}
|
||||
}
|
||||
|
||||
if combinedInput != "" {
|
||||
inputChunks := chunker.SplitText(combinedInput, chunkCfg)
|
||||
for _, sc := range inputChunks {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: sc.Content,
|
||||
ChunkType: types.ChunkTypeText,
|
||||
ParentChunkID: parentChunkID,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
VideoInfo: videoInfoJSON,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if videoSummary != "" {
|
||||
summaryChunks := chunker.SplitText(videoSummary, chunkCfg)
|
||||
for _, sc := range summaryChunks {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: sc.Content,
|
||||
ChunkType: types.ChunkTypeText,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
VideoInfo: videoInfoJSON,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return newChunks
|
||||
}
|
||||
|
||||
// finalizeVideoKnowledge updates the knowledge after multimodal processing:
|
||||
// - For standalone video files: sets Description from summary and marks ParseStatus as completed.
|
||||
// - For videos extracted from documents: no-op (description comes from summary generation).
|
||||
func (s *VideoMultimodalService) finalizeVideoKnowledge(ctx context.Context, payload types.VideoMultimodalPayload, summary string) {
|
||||
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if knowledge == nil {
|
||||
return
|
||||
}
|
||||
if !IsVideoType(knowledge.FileType) {
|
||||
return
|
||||
}
|
||||
|
||||
if summary != "" {
|
||||
knowledge.Description = summary
|
||||
}
|
||||
knowledge.ParseStatus = types.ParseStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to finalize knowledge: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[VideoMultimodal] Finalized video knowledge %s (status=completed, description=%d chars)",
|
||||
payload.KnowledgeID, len(knowledge.Description))
|
||||
}
|
||||
}
|
||||
|
||||
// indexChunks indexes the newly created video chunks into the retrieval engine
|
||||
// so they can participate in semantic search.
|
||||
func (s *VideoMultimodalService) indexChunks(ctx context.Context, payload types.VideoMultimodalPayload, chunks []*types.Chunk) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil || kb == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get KB for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get embedding model for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to get tenant for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to init retrieve engine: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: chunk.Content,
|
||||
SourceID: chunk.ID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
|
||||
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
|
||||
logger.Errorf(ctx, "[VideoMultimodal] Failed to index video chunks: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Mark chunks as indexed.
|
||||
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
|
||||
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
|
||||
for _, chunk := range chunks {
|
||||
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
dbChunk.Status = int(types.ChunkStatusIndexed)
|
||||
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[VideoMultimodal] Indexed %d video chunks for video %s", len(chunks), payload.VideoURL)
|
||||
}
|
||||
|
||||
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
||||
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
||||
func (s *VideoMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
}
|
||||
if kb == nil {
|
||||
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
}
|
||||
|
||||
vlmCfg := kb.VLMConfig
|
||||
if !vlmCfg.IsEnabled() {
|
||||
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
||||
}
|
||||
|
||||
// New-style: resolve model through ModelService
|
||||
if vlmCfg.ModelID != "" {
|
||||
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
||||
}
|
||||
|
||||
// Legacy: create VLM from inline config
|
||||
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
||||
}
|
||||
|
||||
// resolveASR creates an asr.ASR instance for the given knowledge base.
|
||||
func (s *VideoMultimodalService) resolveASR(ctx context.Context, kbID string) (asr.ASR, error) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
}
|
||||
if kb == nil {
|
||||
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
}
|
||||
|
||||
asrCfg := kb.ASRConfig
|
||||
if !asrCfg.IsASREnabled() {
|
||||
return nil, fmt.Errorf("ASR is not enabled for knowledge base %s", kbID)
|
||||
}
|
||||
|
||||
return s.modelService.GetASRModel(ctx, asrCfg.ModelID)
|
||||
}
|
||||
|
||||
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
|
||||
func (s *VideoMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.VideoMultimodalPayload) interfaces.FileService {
|
||||
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil || tenant == nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
provider := types.ParseProviderScheme(payload.VideoURL)
|
||||
if provider == "" {
|
||||
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if kbErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
|
||||
} else if kb != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
|
||||
}
|
||||
}
|
||||
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
|
||||
if svcErr != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
|
||||
return nil
|
||||
}
|
||||
return fileSvc
|
||||
}
|
||||
|
||||
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
|
||||
// generation enabled and, if so, enqueues a task for the video knowledge.
|
||||
func (s *VideoMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.VideoMultimodalPayload) {
|
||||
if s.taskEnqueuer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil || kb == nil {
|
||||
return
|
||||
}
|
||||
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
questionCount := kb.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
if questionCount > 10 {
|
||||
questionCount = 10
|
||||
}
|
||||
|
||||
taskPayload := types.QuestionGenerationPayload{
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
QuestionCount: questionCount,
|
||||
Language: payload.Language,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to marshal question generation payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[VideoMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[VideoMultimodal] Enqueued question generation task for video knowledge %s (count=%d)",
|
||||
payload.KnowledgeID, questionCount)
|
||||
}
|
||||
}
|
||||
@@ -175,7 +175,6 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
|
||||
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
|
||||
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
|
||||
must(container.Provide(service.NewVideoMultimodalService, dig.Name("videoMultimodal")))
|
||||
|
||||
must(container.Provide(service.NewMessageService))
|
||||
must(container.Provide(service.NewMCPServiceService))
|
||||
|
||||
@@ -28,10 +28,6 @@ var audioFormats = map[string]bool{
|
||||
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
|
||||
}
|
||||
|
||||
var videoFormats = map[string]bool{
|
||||
"mp4": true, "mov": true, "avi": true, "mkv": true, "webm": true, "wmv": true, "flv": true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for k := range imageFormats {
|
||||
simpleFormats[k] = true
|
||||
@@ -39,9 +35,6 @@ func init() {
|
||||
for k := range audioFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
for k := range videoFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
}
|
||||
|
||||
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
|
||||
@@ -81,8 +74,6 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
|
||||
return imageToResult(req.FileName, req.FileContent), nil
|
||||
case audioFormats[ft]:
|
||||
return audioToResult(req.FileName, req.FileContent), nil
|
||||
case videoFormats[ft]:
|
||||
return videoToResult(req.FileName, req.FileContent), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported simple format: %s", ft)
|
||||
}
|
||||
@@ -122,11 +113,6 @@ func IsAudioFormat(fileType string) bool {
|
||||
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// IsVideoFormat returns true if the file type is a recognized video format.
|
||||
func IsVideoFormat(fileType string) bool {
|
||||
return videoFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// audioToResult wraps a standalone audio file. The actual transcription is
|
||||
// handled by the ASR model in the knowledge service pipeline. Here we just
|
||||
// return a placeholder markdown with the raw bytes preserved for upstream
|
||||
@@ -144,22 +130,6 @@ func audioToResult(fileName string, data []byte) *types.ReadResult {
|
||||
}
|
||||
}
|
||||
|
||||
// videoToResult wraps a standalone video file. The actual frame extraction,
|
||||
// VLM analysis and ASR transcription is handled by the VideoMultimodalService
|
||||
// in the knowledge service pipeline. Here we just return a placeholder markdown
|
||||
// with the raw bytes preserved for upstream processing.
|
||||
func videoToResult(fileName string, data []byte) *types.ReadResult {
|
||||
if fileName == "" {
|
||||
fileName = "video.mp4"
|
||||
}
|
||||
// Return a placeholder; the knowledge service will handle the actual processing.
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: fmt.Sprintf("[Video file: %s]", fileName),
|
||||
IsVideo: true,
|
||||
VideoData: data,
|
||||
}
|
||||
}
|
||||
|
||||
// ensureOriginalImageRef checks whether the input file is an image and, if the
|
||||
// returned markdown does not already contain a markdown image reference for it,
|
||||
// prepends one and appends the raw bytes to imageRefs. This guarantees that
|
||||
|
||||
@@ -43,7 +43,7 @@ func (e *builtinEngine) Description() string {
|
||||
return "DocReader built-in parser engine"
|
||||
}
|
||||
func (e *builtinEngine) FileTypes(_ bool) []string {
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
}
|
||||
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
|
||||
if docreaderConnected {
|
||||
@@ -71,7 +71,7 @@ func (e *simpleEngine) Description() string {
|
||||
return "Simple format & image parsing (no external service required)"
|
||||
}
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
}
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package video
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Extractor defines the interface for video frame extraction operations.
|
||||
type Extractor interface {
|
||||
// ExtractFrames extracts frames from a video file.
|
||||
// It returns a slice of Frame objects representing the extracted frames.
|
||||
ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error)
|
||||
|
||||
// ExtractFramesToReader extracts frames from a video and returns an io.Reader
|
||||
// for each frame's image data. This is useful for streaming processing.
|
||||
ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error)
|
||||
|
||||
// ExtractAudio extracts the audio track from a video file and returns it as bytes.
|
||||
ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error)
|
||||
|
||||
// GetVideoInfo retrieves basic information about a video file.
|
||||
// It returns a map containing duration, width, height, etc.
|
||||
GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error)
|
||||
}
|
||||
|
||||
// Frame represents a single extracted frame from a video.
|
||||
type Frame struct {
|
||||
// Index is the sequential index of the frame (0-based).
|
||||
Index int
|
||||
|
||||
// Timestamp is the time in seconds from the start of the video.
|
||||
Timestamp float64
|
||||
|
||||
// ImageData contains the raw image bytes of the frame.
|
||||
ImageData []byte
|
||||
|
||||
// Format specifies the image format (e.g., "jpeg", "png").
|
||||
Format string
|
||||
|
||||
// Width is the width of the frame in pixels.
|
||||
Width int
|
||||
|
||||
// Height is the height of the frame in pixels.
|
||||
Height int
|
||||
}
|
||||
|
||||
// ExtractOptions configures how frames are extracted from a video.
|
||||
type ExtractOptions struct {
|
||||
// FrameInterval specifies how often to extract frames (in seconds).
|
||||
// For example, 1.0 means extract one frame per second.
|
||||
FrameInterval float64
|
||||
|
||||
// MaxFrames specifies the maximum number of frames to extract.
|
||||
// If 0, extract all frames according to FrameInterval.
|
||||
MaxFrames int
|
||||
|
||||
// Format specifies the output image format for extracted frames.
|
||||
// Supported formats: "jpeg" (default), "png".
|
||||
Format string
|
||||
|
||||
// Quality specifies the image quality for JPEG output (1-100, default 80).
|
||||
Quality int
|
||||
|
||||
// Width specifies the target width for resizing frames (0 = no resize).
|
||||
Width int
|
||||
|
||||
// Height specifies the target height for resizing frames (0 = no resize).
|
||||
// If only one dimension is specified, the aspect ratio is preserved.
|
||||
Height int
|
||||
}
|
||||
|
||||
// DefaultExtractOptions returns the default frame extraction options.
|
||||
func DefaultExtractOptions() *ExtractOptions {
|
||||
return &ExtractOptions{
|
||||
FrameInterval: 1.0,
|
||||
MaxFrames: 0,
|
||||
Format: "jpeg",
|
||||
Quality: 80,
|
||||
Width: 0,
|
||||
Height: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// Config holds the configuration needed to create a video extractor.
|
||||
type Config struct {
|
||||
// FFMpegPath is the path to the ffmpeg executable.
|
||||
// If empty, it will try to find ffmpeg in the system PATH.
|
||||
FFMpegPath string
|
||||
|
||||
// TempDir is the directory to use for temporary files.
|
||||
// If empty, the system default temporary directory is used.
|
||||
TempDir string
|
||||
}
|
||||
|
||||
// NewExtractor creates a new video frame extractor based on the provided configuration.
|
||||
func NewExtractor(config *Config) (Extractor, error) {
|
||||
return NewFFMpegExtractor(config)
|
||||
}
|
||||
@@ -1,342 +0,0 @@
|
||||
package video
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// FFMpegExtractor implements the Extractor interface using FFmpeg.
|
||||
type FFMpegExtractor struct {
|
||||
ffmpegPath string
|
||||
tempDir string
|
||||
}
|
||||
|
||||
// NewFFMpegExtractor creates a new FFmpeg-based video extractor.
|
||||
func NewFFMpegExtractor(config *Config) (Extractor, error) {
|
||||
ffmpegPath := "ffmpeg"
|
||||
tempDir := os.TempDir()
|
||||
|
||||
if config != nil {
|
||||
if config.FFMpegPath != "" {
|
||||
ffmpegPath = config.FFMpegPath
|
||||
}
|
||||
if config.TempDir != "" {
|
||||
tempDir = config.TempDir
|
||||
}
|
||||
}
|
||||
|
||||
extractor := &FFMpegExtractor{ffmpegPath: ffmpegPath, tempDir: tempDir}
|
||||
if err := extractor.checkFFmpeg(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
// checkFFmpeg verifies that FFmpeg is available.
|
||||
func (e *FFMpegExtractor) checkFFmpeg() error {
|
||||
cmd := exec.Command(e.ffmpegPath, "-version")
|
||||
_, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ffmpeg not found or not executable: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractFrames extracts frames from a video using FFmpeg.
|
||||
func (e *FFMpegExtractor) ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error) {
|
||||
if options == nil {
|
||||
options = DefaultExtractOptions()
|
||||
}
|
||||
|
||||
tempVideoPath, err := e.writeTempVideo(videoBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tempVideoPath)
|
||||
|
||||
tempFrameDir, err := os.MkdirTemp(e.tempDir, "video-frames-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp frame directory: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tempFrameDir)
|
||||
|
||||
if _, err := e.getVideoDuration(ctx, tempVideoPath); err != nil {
|
||||
logger.Warnf(ctx, "[VideoFFMpeg] Failed to get video duration: %v", err)
|
||||
}
|
||||
|
||||
if err := e.extractFramesWithFFmpeg(ctx, tempVideoPath, tempFrameDir, options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return e.readExtractedFrames(ctx, tempFrameDir, options)
|
||||
}
|
||||
|
||||
// ExtractFramesToReaders extracts frames and returns them as io.Readers.
|
||||
func (e *FFMpegExtractor) ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error) {
|
||||
frames, err := e.ExtractFrames(ctx, videoBytes, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
readers := make([]io.Reader, 0, len(frames))
|
||||
for _, frame := range frames {
|
||||
readers = append(readers, bytes.NewReader(frame.ImageData))
|
||||
}
|
||||
|
||||
return readers, nil
|
||||
}
|
||||
|
||||
// writeTempVideo writes the video bytes to a temporary file.
|
||||
func (e *FFMpegExtractor) writeTempVideo(videoBytes []byte) (string, error) {
|
||||
tempFile, err := os.CreateTemp(e.tempDir, "video-input-*.mp4")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp video file: %w", err)
|
||||
}
|
||||
defer tempFile.Close()
|
||||
|
||||
if _, err := tempFile.Write(videoBytes); err != nil {
|
||||
os.Remove(tempFile.Name())
|
||||
return "", fmt.Errorf("failed to write video bytes: %w", err)
|
||||
}
|
||||
|
||||
return tempFile.Name(), nil
|
||||
}
|
||||
|
||||
// getVideoDuration gets the duration of a video using FFmpeg.
|
||||
func (e *FFMpegExtractor) getVideoDuration(ctx context.Context, videoPath string) (float64, error) {
|
||||
cmd := exec.CommandContext(ctx, e.ffmpegPath,
|
||||
"-i", videoPath,
|
||||
"-f", "null",
|
||||
"-")
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
outputStr := string(output)
|
||||
durationStr := ""
|
||||
lines := strings.Split(outputStr, "\n")
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "Duration:") {
|
||||
parts := strings.Split(line, "Duration:")
|
||||
if len(parts) > 1 {
|
||||
durationPart := strings.TrimSpace(parts[1])
|
||||
durationStr = strings.Split(durationPart, ",")[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if durationStr == "" {
|
||||
return 0, fmt.Errorf("could not parse duration")
|
||||
}
|
||||
|
||||
timeParts := strings.Split(durationStr, ":")
|
||||
if len(timeParts) != 3 {
|
||||
return 0, fmt.Errorf("invalid duration format: %s", durationStr)
|
||||
}
|
||||
|
||||
hours, _ := strconv.Atoi(timeParts[0])
|
||||
minutes, _ := strconv.Atoi(timeParts[1])
|
||||
seconds, _ := strconv.ParseFloat(timeParts[2], 64)
|
||||
|
||||
totalSeconds := float64(hours*3600) + float64(minutes*60) + seconds
|
||||
return totalSeconds, nil
|
||||
}
|
||||
|
||||
// extractFramesWithFFmpeg uses FFmpeg to extract frames from a video.
|
||||
func (e *FFMpegExtractor) extractFramesWithFFmpeg(ctx context.Context, videoPath, outputDir string, options *ExtractOptions) error {
|
||||
filters := []string{fmt.Sprintf("fps=1/%f", options.FrameInterval)}
|
||||
|
||||
if options.Width > 0 || options.Height > 0 {
|
||||
scaleFilter := "scale="
|
||||
if options.Width > 0 {
|
||||
scaleFilter += strconv.Itoa(options.Width)
|
||||
} else {
|
||||
scaleFilter += "-1"
|
||||
}
|
||||
scaleFilter += ":"
|
||||
if options.Height > 0 {
|
||||
scaleFilter += strconv.Itoa(options.Height)
|
||||
} else {
|
||||
scaleFilter += "-1"
|
||||
}
|
||||
filters = append(filters, scaleFilter)
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-i", videoPath,
|
||||
"-vf", strings.Join(filters, ","),
|
||||
}
|
||||
|
||||
quality := options.Quality
|
||||
if quality < 1 {
|
||||
quality = 1
|
||||
} else if quality > 100 {
|
||||
quality = 100
|
||||
}
|
||||
if options.Format == "jpeg" {
|
||||
args = append(args, "-q:v", strconv.Itoa(quality))
|
||||
}
|
||||
|
||||
outputPattern := filepath.Join(outputDir, "frame-%05d."+options.Format)
|
||||
args = append(args, outputPattern)
|
||||
|
||||
cmd := exec.CommandContext(ctx, e.ffmpegPath, args...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("ffmpeg extraction failed: %w, stderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[VideoFFMpeg] Extracted frames to: %s", outputDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// readExtractedFrames reads the extracted frames from the temporary directory.
|
||||
func (e *FFMpegExtractor) readExtractedFrames(ctx context.Context, frameDir string, options *ExtractOptions) ([]*Frame, error) {
|
||||
files, err := filepath.Glob(filepath.Join(frameDir, "frame-*."+options.Format))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list frame files: %w", err)
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
return nil, fmt.Errorf("no frames were extracted")
|
||||
}
|
||||
|
||||
frames := make([]*Frame, 0, len(files))
|
||||
for i, filePath := range files {
|
||||
imgData, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[VideoFFMpeg] Failed to read frame %d: %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
frame := &Frame{
|
||||
Index: i,
|
||||
Timestamp: float64(i) * options.FrameInterval,
|
||||
ImageData: imgData,
|
||||
Format: options.Format,
|
||||
Width: options.Width,
|
||||
Height: options.Height,
|
||||
}
|
||||
|
||||
frames = append(frames, frame)
|
||||
|
||||
if options.MaxFrames > 0 && len(frames) >= options.MaxFrames {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(frames) == 0 {
|
||||
return nil, fmt.Errorf("no frames could be read")
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[VideoFFMpeg] Successfully read %d frames", len(frames))
|
||||
return frames, nil
|
||||
}
|
||||
|
||||
// ExtractAudio extracts the audio track from a video using FFmpeg.
|
||||
func (e *FFMpegExtractor) ExtractAudio(ctx context.Context, videoBytes []byte) ([]byte, error) {
|
||||
tempVideoPath, err := e.writeTempVideo(videoBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tempVideoPath)
|
||||
|
||||
tempAudioFile, err := os.CreateTemp(e.tempDir, "audio-output-*.mp3")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp audio file: %w", err)
|
||||
}
|
||||
tempAudioPath := tempAudioFile.Name()
|
||||
tempAudioFile.Close()
|
||||
defer os.Remove(tempAudioPath)
|
||||
|
||||
cmd := exec.CommandContext(ctx, e.ffmpegPath,
|
||||
"-i", tempVideoPath,
|
||||
"-vn",
|
||||
"-c:a", "libmp3lame",
|
||||
"-q:a", "2",
|
||||
"-y",
|
||||
tempAudioPath,
|
||||
)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
if strings.Contains(stderr.String(), "Output file does not contain any stream") {
|
||||
logger.Infof(ctx, "[VideoFFMpeg] Video does not contain an audio stream")
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("ffmpeg audio extraction failed: %w, stderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
audioBytes, err := os.ReadFile(tempAudioPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read extracted audio: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[VideoFFMpeg] Extracted audio, size: %d bytes", len(audioBytes))
|
||||
return audioBytes, nil
|
||||
}
|
||||
|
||||
// GetVideoInfo returns basic information about a video.
|
||||
func (e *FFMpegExtractor) GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) {
|
||||
tempVideoPath, err := e.writeTempVideo(videoBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tempVideoPath)
|
||||
|
||||
duration, err := e.getVideoDuration(ctx, tempVideoPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, e.ffmpegPath, "-i", tempVideoPath, "-f", "null", "-")
|
||||
output, _ := cmd.CombinedOutput()
|
||||
outputStr := string(output)
|
||||
|
||||
width, height := 0, 0
|
||||
lines := strings.Split(outputStr, "\n")
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "Video:") {
|
||||
parts := strings.Split(line, ",")
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if strings.Contains(part, "x") && !strings.Contains(part, "SAR") {
|
||||
resParts := strings.Split(part, "x")
|
||||
if len(resParts) == 2 {
|
||||
width, _ = strconv.Atoi(strings.TrimSpace(resParts[0]))
|
||||
heightStr := strings.TrimSpace(resParts[1])
|
||||
heightParts := strings.Split(heightStr, " ")
|
||||
if len(heightParts) > 0 {
|
||||
height, _ = strconv.Atoi(heightParts[0])
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
videoInfo := map[string]interface{}{
|
||||
"duration": duration,
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
|
||||
return videoInfo, nil
|
||||
}
|
||||
@@ -23,7 +23,6 @@ type AsynqTaskParams struct {
|
||||
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
|
||||
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
|
||||
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
|
||||
VideoMultimodal interfaces.TaskHandler `name:"videoMultimodal"`
|
||||
}
|
||||
|
||||
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
@@ -110,9 +109,6 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register image multimodal handler
|
||||
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
|
||||
|
||||
// Register video multimodal handler
|
||||
mux.HandleFunc(types.TypeVideoMultimodal, params.VideoMultimodal.Handle)
|
||||
|
||||
// Register data source sync handler
|
||||
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
|
||||
|
||||
|
||||
@@ -22,8 +22,6 @@ type ReadResult struct {
|
||||
Error string
|
||||
IsAudio bool // true when the result contains raw audio data needing ASR transcription
|
||||
AudioData []byte // raw audio bytes for ASR processing
|
||||
IsVideo bool // true when the result contains raw video data needing multimodal processing
|
||||
VideoData []byte // raw video bytes for video multimodal processing
|
||||
}
|
||||
|
||||
// ImageRef represents an image reference extracted from the document.
|
||||
|
||||
@@ -13,7 +13,6 @@ const (
|
||||
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
|
||||
TypeVideoMultimodal = "video:multimodal" // 视频多模态处理任务(抽帧 + VLM + ASR)
|
||||
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
|
||||
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
|
||||
)
|
||||
@@ -156,20 +155,6 @@ type ImageMultimodalPayload struct {
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
}
|
||||
|
||||
// VideoMultimodalPayload represents the video multimodal processing task payload.
|
||||
type VideoMultimodalPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
VideoURL string `json:"video_url"` // provider:// URL (e.g. local://..., minio://...)
|
||||
VideoLocalPath string `json:"video_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
EnableVLM bool `json:"enable_vlm"` // enable VLM frame analysis
|
||||
EnableASR bool `json:"enable_asr"` // enable ASR audio transcription
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
MaxFrames int `json:"max_frames,omitempty"` // optional: max frames to extract, 0 = no limit
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
type KBCloneTaskStatus string
|
||||
|
||||
|
||||
Reference in New Issue
Block a user