feat: add video multimodal functionality

This commit is contained in:
DaWesen
2026-04-08 19:52:28 +08:00
committed by lyingbug
parent 2b10cc54de
commit b80bdb1079
20 changed files with 1213 additions and 28 deletions
+26
View File
@@ -20,6 +20,12 @@ const (
ChunkTypeImageOCR ChunkType = "image_ocr"
// ChunkTypeImageCaption 表示图片描述的 Chunk
ChunkTypeImageCaption ChunkType = "image_caption"
// ChunkTypeVideoFrame 表示视频帧分析内容的 Chunk
ChunkTypeVideoFrame ChunkType = "video_frame"
// ChunkTypeVideoCaption 表示视频整体描述的 Chunk
ChunkTypeVideoCaption ChunkType = "video_caption"
// ChunkTypeVideoASR 表示视频音频转录的 Chunk
ChunkTypeVideoASR ChunkType = "video_asr"
// ChunkTypeSummary 表示摘要类型的 Chunk
ChunkTypeSummary = "summary"
// ChunkTypeEntity 表示实体类型的 Chunk
@@ -95,6 +101,24 @@ type ImageInfo struct {
OCRText string `json:"ocr_text"`
}
// VideoInfo 表示与 Chunk 关联的视频信息
type VideoInfo struct {
// 视频URL
URL string `json:"url" gorm:"type:text"`
// 帧数量
FrameCount int `json:"frame_count"`
// 是否有VLM分析
HasVLMAnalysis bool `json:"has_vlm_analysis"`
// 是否有ASR转录
HasASR bool `json:"has_asr"`
// 视频摘要
VideoSummary string `json:"video_summary"`
// ASR转录文本
ASRText string `json:"asr_text"`
// 帧描述列表
FrameDescriptions []string `json:"frame_descriptions"`
}
// Chunk represents a document chunk
// Chunks are meaningful text segments extracted from original documents
// and are the basic units of knowledge base retrieval
@@ -147,6 +171,8 @@ type Chunk struct {
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
// 图片信息,存储为 JSON
ImageInfo string `json:"image_info" gorm:"type:text"`
// 视频信息,存储为 JSON
VideoInfo string `json:"video_info" gorm:"type:text"`
// Chunk creation time
CreatedAt time.Time `json:"created_at"`
// Chunk last update time
+2
View File
@@ -22,6 +22,8 @@ type ReadResult struct {
Error string
IsAudio bool // true when the result contains raw audio data needing ASR transcription
AudioData []byte // raw audio bytes for ASR processing
IsVideo bool // true when the result contains raw video data needing multimodal processing
VideoData []byte // raw video bytes for video multimodal processing
}
// ImageRef represents an image reference extracted from the document.
+21 -6
View File
@@ -13,6 +13,7 @@ const (
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
TypeVideoMultimodal = "video:multimodal" // 视频多模态处理任务(抽帧 + VLM + ASR)
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
)
@@ -121,7 +122,7 @@ type KnowledgeMoveProgress struct {
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Failed int `json:"failed"` // 失败数
@@ -138,8 +139,8 @@ type ManualProcessPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Content string `json:"content"` // cleaned markdown content
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
Content string `json:"content"` // cleaned markdown content
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
}
// ImageMultimodalPayload represents the image multimodal processing task payload.
@@ -147,14 +148,28 @@ type ImageMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
ChunkID string `json:"chunk_id"` // parent text chunk
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
EnableOCR bool `json:"enable_ocr"`
EnableCaption bool `json:"enable_caption"`
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// VideoMultimodalPayload represents the video multimodal processing task payload.
type VideoMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
VideoURL string `json:"video_url"` // provider:// URL (e.g. local://..., minio://...)
VideoLocalPath string `json:"video_local_path"` // deprecated: kept for backward compat with in-flight tasks
EnableVLM bool `json:"enable_vlm"` // enable VLM frame analysis
EnableASR bool `json:"enable_asr"` // enable ASR audio transcription
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
MaxFrames int `json:"max_frames,omitempty"` // optional: max frames to extract, 0 = no limit
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string