feat: add video multimodal functionality
This commit is contained in:
@@ -28,6 +28,10 @@ var audioFormats = map[string]bool{
|
||||
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
|
||||
}
|
||||
|
||||
var videoFormats = map[string]bool{
|
||||
"mp4": true, "mov": true, "avi": true, "mkv": true, "webm": true, "wmv": true, "flv": true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for k := range imageFormats {
|
||||
simpleFormats[k] = true
|
||||
@@ -35,6 +39,9 @@ func init() {
|
||||
for k := range audioFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
for k := range videoFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
}
|
||||
|
||||
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
|
||||
@@ -74,6 +81,8 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
|
||||
return imageToResult(req.FileName, req.FileContent), nil
|
||||
case audioFormats[ft]:
|
||||
return audioToResult(req.FileName, req.FileContent), nil
|
||||
case videoFormats[ft]:
|
||||
return videoToResult(req.FileName, req.FileContent), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported simple format: %s", ft)
|
||||
}
|
||||
@@ -113,6 +122,11 @@ func IsAudioFormat(fileType string) bool {
|
||||
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// IsVideoFormat returns true if the file type is a recognized video format.
|
||||
func IsVideoFormat(fileType string) bool {
|
||||
return videoFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// audioToResult wraps a standalone audio file. The actual transcription is
|
||||
// handled by the ASR model in the knowledge service pipeline. Here we just
|
||||
// return a placeholder markdown with the raw bytes preserved for upstream
|
||||
@@ -130,6 +144,22 @@ func audioToResult(fileName string, data []byte) *types.ReadResult {
|
||||
}
|
||||
}
|
||||
|
||||
// videoToResult wraps a standalone video file. The actual frame extraction,
|
||||
// VLM analysis and ASR transcription is handled by the VideoMultimodalService
|
||||
// in the knowledge service pipeline. Here we just return a placeholder markdown
|
||||
// with the raw bytes preserved for upstream processing.
|
||||
func videoToResult(fileName string, data []byte) *types.ReadResult {
|
||||
if fileName == "" {
|
||||
fileName = "video.mp4"
|
||||
}
|
||||
// Return a placeholder; the knowledge service will handle the actual processing.
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: fmt.Sprintf("[Video file: %s]", fileName),
|
||||
IsVideo: true,
|
||||
VideoData: data,
|
||||
}
|
||||
}
|
||||
|
||||
// ensureOriginalImageRef checks whether the input file is an image and, if the
|
||||
// returned markdown does not already contain a markdown image reference for it,
|
||||
// prepends one and appends the raw bytes to imageRefs. This guarantees that
|
||||
|
||||
@@ -42,7 +42,7 @@ func (e *builtinEngine) Description() string {
|
||||
return "DocReader built-in parser engine"
|
||||
}
|
||||
func (e *builtinEngine) FileTypes(_ bool) []string {
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"}
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
|
||||
}
|
||||
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
|
||||
if docreaderConnected {
|
||||
@@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string {
|
||||
return "Simple format & image parsing (no external service required)"
|
||||
}
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
|
||||
}
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
|
||||
Reference in New Issue
Block a user