feat: add video multimodal functionality
This commit is contained in:
@@ -20,6 +20,12 @@ const (
|
||||
ChunkTypeImageOCR ChunkType = "image_ocr"
|
||||
// ChunkTypeImageCaption 表示图片描述的 Chunk
|
||||
ChunkTypeImageCaption ChunkType = "image_caption"
|
||||
// ChunkTypeVideoFrame 表示视频帧分析内容的 Chunk
|
||||
ChunkTypeVideoFrame ChunkType = "video_frame"
|
||||
// ChunkTypeVideoCaption 表示视频整体描述的 Chunk
|
||||
ChunkTypeVideoCaption ChunkType = "video_caption"
|
||||
// ChunkTypeVideoASR 表示视频音频转录的 Chunk
|
||||
ChunkTypeVideoASR ChunkType = "video_asr"
|
||||
// ChunkTypeSummary 表示摘要类型的 Chunk
|
||||
ChunkTypeSummary = "summary"
|
||||
// ChunkTypeEntity 表示实体类型的 Chunk
|
||||
@@ -95,6 +101,24 @@ type ImageInfo struct {
|
||||
OCRText string `json:"ocr_text"`
|
||||
}
|
||||
|
||||
// VideoInfo 表示与 Chunk 关联的视频信息
|
||||
type VideoInfo struct {
|
||||
// 视频URL
|
||||
URL string `json:"url" gorm:"type:text"`
|
||||
// 帧数量
|
||||
FrameCount int `json:"frame_count"`
|
||||
// 是否有VLM分析
|
||||
HasVLMAnalysis bool `json:"has_vlm_analysis"`
|
||||
// 是否有ASR转录
|
||||
HasASR bool `json:"has_asr"`
|
||||
// 视频摘要
|
||||
VideoSummary string `json:"video_summary"`
|
||||
// ASR转录文本
|
||||
ASRText string `json:"asr_text"`
|
||||
// 帧描述列表
|
||||
FrameDescriptions []string `json:"frame_descriptions"`
|
||||
}
|
||||
|
||||
// Chunk represents a document chunk
|
||||
// Chunks are meaningful text segments extracted from original documents
|
||||
// and are the basic units of knowledge base retrieval
|
||||
@@ -147,6 +171,8 @@ type Chunk struct {
|
||||
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
|
||||
// 图片信息,存储为 JSON
|
||||
ImageInfo string `json:"image_info" gorm:"type:text"`
|
||||
// 视频信息,存储为 JSON
|
||||
VideoInfo string `json:"video_info" gorm:"type:text"`
|
||||
// Chunk creation time
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Chunk last update time
|
||||
|
||||
@@ -22,6 +22,8 @@ type ReadResult struct {
|
||||
Error string
|
||||
IsAudio bool // true when the result contains raw audio data needing ASR transcription
|
||||
AudioData []byte // raw audio bytes for ASR processing
|
||||
IsVideo bool // true when the result contains raw video data needing multimodal processing
|
||||
VideoData []byte // raw video bytes for video multimodal processing
|
||||
}
|
||||
|
||||
// ImageRef represents an image reference extracted from the document.
|
||||
|
||||
@@ -13,6 +13,7 @@ const (
|
||||
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
|
||||
TypeVideoMultimodal = "video:multimodal" // 视频多模态处理任务(抽帧 + VLM + ASR)
|
||||
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
|
||||
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
|
||||
)
|
||||
@@ -121,7 +122,7 @@ type KnowledgeMoveProgress struct {
|
||||
SourceKBID string `json:"source_kb_id"`
|
||||
TargetKBID string `json:"target_kb_id"`
|
||||
Status KBCloneTaskStatus `json:"status"`
|
||||
Progress int `json:"progress"` // 0-100
|
||||
Progress int `json:"progress"` // 0-100
|
||||
Total int `json:"total"` // 总知识数
|
||||
Processed int `json:"processed"` // 已处理数
|
||||
Failed int `json:"failed"` // 失败数
|
||||
@@ -138,8 +139,8 @@ type ManualProcessPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
Content string `json:"content"` // cleaned markdown content
|
||||
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
|
||||
Content string `json:"content"` // cleaned markdown content
|
||||
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
|
||||
}
|
||||
|
||||
// ImageMultimodalPayload represents the image multimodal processing task payload.
|
||||
@@ -147,14 +148,28 @@ type ImageMultimodalPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
|
||||
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
|
||||
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
EnableOCR bool `json:"enable_ocr"`
|
||||
EnableCaption bool `json:"enable_caption"`
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
}
|
||||
|
||||
// VideoMultimodalPayload represents the video multimodal processing task payload.
|
||||
type VideoMultimodalPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
VideoURL string `json:"video_url"` // provider:// URL (e.g. local://..., minio://...)
|
||||
VideoLocalPath string `json:"video_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
EnableVLM bool `json:"enable_vlm"` // enable VLM frame analysis
|
||||
EnableASR bool `json:"enable_asr"` // enable ASR audio transcription
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
MaxFrames int `json:"max_frames,omitempty"` // optional: max frames to extract, 0 = no limit
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
type KBCloneTaskStatus string
|
||||
|
||||
|
||||
Reference in New Issue
Block a user