feat: add video multimodal functionality

This commit is contained in:
DaWesen
2026-04-08 19:52:28 +08:00
committed by lyingbug
parent 2b10cc54de
commit b80bdb1079
20 changed files with 1213 additions and 28 deletions
+95
View File
@@ -0,0 +1,95 @@
package video
import (
"context"
"io"
)
// Extractor defines the interface for video frame extraction operations.
type Extractor interface {
// ExtractFrames extracts frames from a video file.
// It returns a slice of Frame objects representing the extracted frames.
ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error)
// ExtractFramesToReader extracts frames from a video and returns an io.Reader
// for each frame's image data. This is useful for streaming processing.
ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error)
// GetVideoInfo retrieves basic information about a video file.
// It returns a map containing duration, width, height, etc.
GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error)
}
// Frame represents a single extracted frame from a video.
type Frame struct {
// Index is the sequential index of the frame (0-based).
Index int
// Timestamp is the time in seconds from the start of the video.
Timestamp float64
// ImageData contains the raw image bytes of the frame.
ImageData []byte
// Format specifies the image format (e.g., "jpeg", "png").
Format string
// Width is the width of the frame in pixels.
Width int
// Height is the height of the frame in pixels.
Height int
}
// ExtractOptions configures how frames are extracted from a video.
type ExtractOptions struct {
// FrameInterval specifies how often to extract frames (in seconds).
// For example, 1.0 means extract one frame per second.
FrameInterval float64
// MaxFrames specifies the maximum number of frames to extract.
// If 0, extract all frames according to FrameInterval.
MaxFrames int
// Format specifies the output image format for extracted frames.
// Supported formats: "jpeg" (default), "png".
Format string
// Quality specifies the image quality for JPEG output (1-100, default 80).
Quality int
// Width specifies the target width for resizing frames (0 = no resize).
Width int
// Height specifies the target height for resizing frames (0 = no resize).
// If only one dimension is specified, the aspect ratio is preserved.
Height int
}
// DefaultExtractOptions returns the default frame extraction options.
func DefaultExtractOptions() *ExtractOptions {
return &ExtractOptions{
FrameInterval: 1.0,
MaxFrames: 0,
Format: "jpeg",
Quality: 80,
Width: 0,
Height: 0,
}
}
// Config holds the configuration needed to create a video extractor.
type Config struct {
// FFMpegPath is the path to the ffmpeg executable.
// If empty, it will try to find ffmpeg in the system PATH.
FFMpegPath string
// TempDir is the directory to use for temporary files.
// If empty, the system default temporary directory is used.
TempDir string
}
// NewExtractor creates a new video frame extractor based on the provided configuration.
func NewExtractor(config *Config) (Extractor, error) {
return NewFFMpegExtractor(config)
}
+298
View File
@@ -0,0 +1,298 @@
package video
import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
)
// FFMpegExtractor implements the Extractor interface using FFmpeg.
type FFMpegExtractor struct {
ffmpegPath string
tempDir string
}
// NewFFMpegExtractor creates a new FFmpeg-based video extractor.
func NewFFMpegExtractor(config *Config) (Extractor, error) {
ffmpegPath := "ffmpeg"
tempDir := os.TempDir()
if config != nil {
if config.FFMpegPath != "" {
ffmpegPath = config.FFMpegPath
}
if config.TempDir != "" {
tempDir = config.TempDir
}
}
extractor := &FFMpegExtractor{ffmpegPath: ffmpegPath, tempDir: tempDir}
if err := extractor.checkFFmpeg(); err != nil {
return nil, err
}
return extractor, nil
}
// checkFFmpeg verifies that FFmpeg is available.
func (e *FFMpegExtractor) checkFFmpeg() error {
cmd := exec.Command(e.ffmpegPath, "-version")
_, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("ffmpeg not found or not executable: %w", err)
}
return nil
}
// ExtractFrames extracts frames from a video using FFmpeg.
func (e *FFMpegExtractor) ExtractFrames(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]*Frame, error) {
if options == nil {
options = DefaultExtractOptions()
}
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
tempFrameDir, err := os.MkdirTemp(e.tempDir, "video-frames-")
if err != nil {
return nil, fmt.Errorf("failed to create temp frame directory: %w", err)
}
defer os.RemoveAll(tempFrameDir)
if _, err := e.getVideoDuration(ctx, tempVideoPath); err != nil {
logger.Warnf(ctx, "[VideoFFMpeg] Failed to get video duration: %v", err)
}
if err := e.extractFramesWithFFmpeg(ctx, tempVideoPath, tempFrameDir, options); err != nil {
return nil, err
}
return e.readExtractedFrames(ctx, tempFrameDir, options)
}
// ExtractFramesToReaders extracts frames and returns them as io.Readers.
func (e *FFMpegExtractor) ExtractFramesToReaders(ctx context.Context, videoBytes []byte, options *ExtractOptions) ([]io.Reader, error) {
frames, err := e.ExtractFrames(ctx, videoBytes, options)
if err != nil {
return nil, err
}
readers := make([]io.Reader, 0, len(frames))
for _, frame := range frames {
readers = append(readers, bytes.NewReader(frame.ImageData))
}
return readers, nil
}
// writeTempVideo writes the video bytes to a temporary file.
func (e *FFMpegExtractor) writeTempVideo(videoBytes []byte) (string, error) {
tempFile, err := os.CreateTemp(e.tempDir, "video-input-*.mp4")
if err != nil {
return "", fmt.Errorf("failed to create temp video file: %w", err)
}
defer tempFile.Close()
if _, err := tempFile.Write(videoBytes); err != nil {
os.Remove(tempFile.Name())
return "", fmt.Errorf("failed to write video bytes: %w", err)
}
return tempFile.Name(), nil
}
// getVideoDuration gets the duration of a video using FFmpeg.
func (e *FFMpegExtractor) getVideoDuration(ctx context.Context, videoPath string) (float64, error) {
cmd := exec.CommandContext(ctx, e.ffmpegPath,
"-i", videoPath,
"-f", "null",
"-")
output, err := cmd.CombinedOutput()
if err != nil {
return 0, err
}
outputStr := string(output)
durationStr := ""
lines := strings.Split(outputStr, "\n")
for _, line := range lines {
if strings.Contains(line, "Duration:") {
parts := strings.Split(line, "Duration:")
if len(parts) > 1 {
durationPart := strings.TrimSpace(parts[1])
durationStr = strings.Split(durationPart, ",")[0]
break
}
}
}
if durationStr == "" {
return 0, fmt.Errorf("could not parse duration")
}
timeParts := strings.Split(durationStr, ":")
if len(timeParts) != 3 {
return 0, fmt.Errorf("invalid duration format: %s", durationStr)
}
hours, _ := strconv.Atoi(timeParts[0])
minutes, _ := strconv.Atoi(timeParts[1])
seconds, _ := strconv.ParseFloat(timeParts[2], 64)
totalSeconds := float64(hours*3600) + float64(minutes*60) + seconds
return totalSeconds, nil
}
// extractFramesWithFFmpeg uses FFmpeg to extract frames from a video.
func (e *FFMpegExtractor) extractFramesWithFFmpeg(ctx context.Context, videoPath, outputDir string, options *ExtractOptions) error {
filters := []string{fmt.Sprintf("fps=1/%f", options.FrameInterval)}
if options.Width > 0 || options.Height > 0 {
scaleFilter := "scale="
if options.Width > 0 {
scaleFilter += strconv.Itoa(options.Width)
} else {
scaleFilter += "-1"
}
scaleFilter += ":"
if options.Height > 0 {
scaleFilter += strconv.Itoa(options.Height)
} else {
scaleFilter += "-1"
}
filters = append(filters, scaleFilter)
}
args := []string{
"-i", videoPath,
"-vf", strings.Join(filters, ","),
}
quality := options.Quality
if quality < 1 {
quality = 1
} else if quality > 100 {
quality = 100
}
if options.Format == "jpeg" {
args = append(args, "-q:v", strconv.Itoa(quality))
}
outputPattern := filepath.Join(outputDir, "frame-%05d."+options.Format)
args = append(args, outputPattern)
cmd := exec.CommandContext(ctx, e.ffmpegPath, args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("ffmpeg extraction failed: %w, stderr: %s", err, stderr.String())
}
logger.Infof(ctx, "[VideoFFMpeg] Extracted frames to: %s", outputDir)
return nil
}
// readExtractedFrames reads the extracted frames from the temporary directory.
func (e *FFMpegExtractor) readExtractedFrames(ctx context.Context, frameDir string, options *ExtractOptions) ([]*Frame, error) {
files, err := filepath.Glob(filepath.Join(frameDir, "frame-*."+options.Format))
if err != nil {
return nil, fmt.Errorf("failed to list frame files: %w", err)
}
if len(files) == 0 {
return nil, fmt.Errorf("no frames were extracted")
}
frames := make([]*Frame, 0, len(files))
for i, filePath := range files {
imgData, err := os.ReadFile(filePath)
if err != nil {
logger.Warnf(ctx, "[VideoFFMpeg] Failed to read frame %d: %v", i, err)
continue
}
frame := &Frame{
Index: i,
Timestamp: float64(i) * options.FrameInterval,
ImageData: imgData,
Format: options.Format,
Width: options.Width,
Height: options.Height,
}
frames = append(frames, frame)
if options.MaxFrames > 0 && len(frames) >= options.MaxFrames {
break
}
}
if len(frames) == 0 {
return nil, fmt.Errorf("no frames could be read")
}
logger.Infof(ctx, "[VideoFFMpeg] Successfully read %d frames", len(frames))
return frames, nil
}
// GetVideoInfo returns basic information about a video.
func (e *FFMpegExtractor) GetVideoInfo(ctx context.Context, videoBytes []byte) (map[string]interface{}, error) {
tempVideoPath, err := e.writeTempVideo(videoBytes)
if err != nil {
return nil, err
}
defer os.Remove(tempVideoPath)
duration, err := e.getVideoDuration(ctx, tempVideoPath)
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, e.ffmpegPath, "-i", tempVideoPath, "-f", "null", "-")
output, _ := cmd.CombinedOutput()
outputStr := string(output)
width, height := 0, 0
lines := strings.Split(outputStr, "\n")
for _, line := range lines {
if strings.Contains(line, "Video:") {
parts := strings.Split(line, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
if strings.Contains(part, "x") && !strings.Contains(part, "SAR") {
resParts := strings.Split(part, "x")
if len(resParts) == 2 {
width, _ = strconv.Atoi(strings.TrimSpace(resParts[0]))
heightStr := strings.TrimSpace(resParts[1])
heightParts := strings.Split(heightStr, " ")
if len(heightParts) > 0 {
height, _ = strconv.Atoi(heightParts[0])
}
}
break
}
}
break
}
}
videoInfo := map[string]interface{}{
"duration": duration,
"width": width,
"height": height,
}
return videoInfo, nil
}