feat: add video multimodal functionality

This commit is contained in:
DaWesen
2026-04-08 19:52:28 +08:00
committed by lyingbug
parent 2b10cc54de
commit b80bdb1079
20 changed files with 1213 additions and 28 deletions
@@ -28,6 +28,10 @@ var audioFormats = map[string]bool{
"mp3": true, "wav": true, "m4a": true, "flac": true, "ogg": true,
}
var videoFormats = map[string]bool{
"mp4": true, "mov": true, "avi": true, "mkv": true, "webm": true, "wmv": true, "flv": true,
}
func init() {
for k := range imageFormats {
simpleFormats[k] = true
@@ -35,6 +39,9 @@ func init() {
for k := range audioFormats {
simpleFormats[k] = true
}
for k := range videoFormats {
simpleFormats[k] = true
}
}
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
@@ -74,6 +81,8 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
return imageToResult(req.FileName, req.FileContent), nil
case audioFormats[ft]:
return audioToResult(req.FileName, req.FileContent), nil
case videoFormats[ft]:
return videoToResult(req.FileName, req.FileContent), nil
default:
return nil, fmt.Errorf("unsupported simple format: %s", ft)
}
@@ -113,6 +122,11 @@ func IsAudioFormat(fileType string) bool {
return audioFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// IsVideoFormat returns true if the file type is a recognized video format.
func IsVideoFormat(fileType string) bool {
return videoFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
}
// audioToResult wraps a standalone audio file. The actual transcription is
// handled by the ASR model in the knowledge service pipeline. Here we just
// return a placeholder markdown with the raw bytes preserved for upstream
@@ -130,6 +144,22 @@ func audioToResult(fileName string, data []byte) *types.ReadResult {
}
}
// videoToResult wraps a standalone video file. The actual frame extraction,
// VLM analysis and ASR transcription is handled by the VideoMultimodalService
// in the knowledge service pipeline. Here we just return a placeholder markdown
// with the raw bytes preserved for upstream processing.
func videoToResult(fileName string, data []byte) *types.ReadResult {
if fileName == "" {
fileName = "video.mp4"
}
// Return a placeholder; the knowledge service will handle the actual processing.
return &types.ReadResult{
MarkdownContent: fmt.Sprintf("[Video file: %s]", fileName),
IsVideo: true,
VideoData: data,
}
}
// ensureOriginalImageRef checks whether the input file is an image and, if the
// returned markdown does not already contain a markdown image reference for it,
// prepends one and appends the raw bytes to imageRefs. This guarantees that
@@ -42,7 +42,7 @@ func (e *builtinEngine) Description() string {
return "DocReader built-in parser engine"
}
func (e *builtinEngine) FileTypes(_ bool) []string {
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"}
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
}
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
if docreaderConnected {
@@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string {
return "Simple format & image parsing (no external service required)"
}
func (e *simpleEngine) FileTypes(_ bool) []string {
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", "wmv", "flv"}
}
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
return true, ""