Files
Enterprise-Knowledge-Base/internal/application/service/image_multimodal.go
T
wizardchen 4a9977b11f feat: enhance document loading and state management in doc-content component
- Introduced a loading state (`loadingChunks`) to manage document chunk loading more effectively.
- Updated the scroll handling logic to prevent multiple requests while loading chunks.
- Adjusted the pagination logic to ensure correct page increments based on the total document count.

This update improves the user experience by ensuring smoother document loading and better state management during scrolling interactions.
2026-03-02 21:21:49 +08:00

359 lines
13 KiB
Go

package service
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmOCRPrompt = "请提取这张文档图片中的所有正文内容,用纯 Markdown 格式输出。要求:\n" +
"1. 忽略页眉、页脚\n" +
"2. 表格使用 Markdown 表格语法\n" +
"3. 公式使用 LaTeX 格式(用 $ 或 $$ 包裹)\n" +
"4. 按照原文阅读顺序组织\n" +
"5. 只输出提取到的文本内容,不要添加任何 HTML 标签\n" +
"如果图片中没有可识别的文字内容,请回复:无文字内容。"
vlmCaptionPrompt = "简单凝炼的描述图片的主要内容"
)
// ImageMultimodalService handles image:multimodal asynq tasks.
// It reads images from local storage, performs OCR and VLM caption,
// and creates child chunks for the processed content.
type ImageMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
}
func NewImageMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
}
}
// Handle implements asynq handler for TypeImageMultimodal.
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.ImageMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
}
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, local_path=%s, ocr=%v, caption=%v",
payload.ChunkID, payload.ImageURL, payload.ImageLocalPath, payload.EnableOCR, payload.EnableCaption)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if err != nil {
return fmt.Errorf("resolve VLM: %w", err)
}
imgBytes, err := os.ReadFile(payload.ImageLocalPath)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying to download from URL", payload.ImageLocalPath, err)
imgBytes, err = downloadImageFromURL(payload.ImageURL)
if err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
return fmt.Errorf("read image: local file %s not found and URL download failed: %w", payload.ImageLocalPath, err)
}
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
} else {
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
// Only clean up the local file when images have been re-uploaded to
// external object storage (COS/TOS) — the serving URL will be an
// absolute http(s) URL in that case. For local and MinIO storage the
// local file IS the serving file and must be kept.
if strings.HasPrefix(payload.ImageURL, "http://") || strings.HasPrefix(payload.ImageURL, "https://") {
defer os.Remove(payload.ImageLocalPath)
}
}
imageInfo := types.ImageInfo{
URL: payload.ImageURL,
OriginalURL: payload.ImageURL,
}
if payload.EnableOCR {
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
if ocrErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
} else {
ocrText = sanitizeOCRText(ocrText)
if ocrText != "" {
imageInfo.OCRText = ocrText
} else {
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
}
}
}
if payload.EnableCaption {
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
if capErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
} else if caption != "" {
imageInfo.Caption = caption
}
}
// Build child chunks for OCR and caption results
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
var newChunks []*types.Chunk
if imageInfo.OCRText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.OCRText,
ChunkType: types.ChunkTypeImageOCR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if imageInfo.Caption != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.Caption,
ChunkType: types.ChunkTypeImageCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if len(newChunks) == 0 {
// Even if OCR/caption both failed, mark knowledge as completed
s.finalizeImageKnowledge(ctx, payload, "")
return nil
}
// Persist chunks
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
// For standalone image files, use caption as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
return nil
}
// finalizeImageKnowledge updates the knowledge after multimodal processing:
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
// (processChunks kept it in "processing" to wait for multimodal results).
// - For images extracted from PDFs: no-op (description comes from summary generation).
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsImageType(knowledge.FileType) {
return
}
if caption != "" {
knowledge.Description = caption
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
// so they can participate in semantic search.
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
}
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full image metadata (URL, OCR, caption).
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.ImageInfo
if chunk.ImageInfo != "" {
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == imageInfo.URL {
existingInfos[i] = imageInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, imageInfo)
}
imageInfoJSON, _ := json.Marshal(existingInfos)
chunk.ImageInfo = string(imageInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
return secutils.DownloadBytes(imageURL)
}