package service import ( "context" "encoding/json" "fmt" "io" "os" "strings" "time" filesvc "github.com/Tencent/WeKnora/internal/application/service/file" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" ) const ( vlmOCRPrompt = "\n" + "You are an OCR assistant. Your task is to extract all body text content from this document image and output in pure Markdown format.\n" + "\n\n" + "\n" + "1. Ignore headers and footers.\n" + "2. Use Markdown table syntax for tables.\n" + "3. Use LaTeX format for formulas (wrapped with $ or $$).\n" + "4. Organize content in the original reading order.\n" + "5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" + "6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" + "" vlmOCRScannedPDFPrompt = "\n" + "You are an OCR and document layout extraction assistant. The input image is a page from a scanned PDF document.\n" + "Your task is to carefully extract all text and layout structure from the image, and output the result in pure Markdown format.\n" + "\n\n" + "\n" + "1. Ignore headers, footers, and page numbers.\n" + "2. Preserve the original document's paragraph and hierarchical structure as much as possible.\n" + "3. If there are tables, use Markdown table syntax to represent them.\n" + "4. If there are mathematical formulas, use LaTeX format wrapped in $ or $$.\n" + "5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" + "6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" + "" vlmCaptionPrompt = "Provide a brief and concise description of the main content of the image in Chinese" ) // ImageMultimodalService handles image:multimodal asynq tasks. // It reads images from storage (via FileService for provider:// URLs), // performs OCR and VLM caption, and creates child chunks. type ImageMultimodalService struct { chunkService interfaces.ChunkService modelService interfaces.ModelService kbService interfaces.KnowledgeBaseService knowledgeRepo interfaces.KnowledgeRepository tenantRepo interfaces.TenantRepository retrieveEngine interfaces.RetrieveEngineRegistry ownership retriever.TenantStoreOwnership ollamaService *ollama.OllamaService taskEnqueuer interfaces.TaskEnqueuer redisClient *redis.Client // fileSvc is the globally configured default FileService used as a fallback // when the tenant-scoped storage config cannot produce a usable service // (e.g. images were saved using the global MINIO_* env vars while the // tenant's StorageEngineConfig.MinIO is empty). Mirrors the write-side // fallback in knowledgeService.resolveFileService. fileSvc interfaces.FileService } func NewImageMultimodalService( chunkService interfaces.ChunkService, modelService interfaces.ModelService, kbService interfaces.KnowledgeBaseService, knowledgeRepo interfaces.KnowledgeRepository, tenantRepo interfaces.TenantRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ownership retriever.TenantStoreOwnership, ollamaService *ollama.OllamaService, taskEnqueuer interfaces.TaskEnqueuer, redisClient *redis.Client, fileSvc interfaces.FileService, ) interfaces.TaskHandler { return &ImageMultimodalService{ chunkService: chunkService, modelService: modelService, kbService: kbService, knowledgeRepo: knowledgeRepo, tenantRepo: tenantRepo, retrieveEngine: retrieveEngine, ownership: ownership, ollamaService: ollamaService, taskEnqueuer: taskEnqueuer, redisClient: redisClient, fileSvc: fileSvc, } } // Handle implements asynq handler for TypeImageMultimodal. func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error { var payload types.ImageMultimodalPayload if err := json.Unmarshal(task.Payload(), &payload); err != nil { return fmt.Errorf("unmarshal image multimodal payload: %w", err) } logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, ocr=%v, caption=%v", payload.ChunkID, payload.ImageURL, payload.EnableOCR, payload.EnableCaption) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID) if err != nil { return fmt.Errorf("resolve VLM: %w", err) } // Read image bytes. A provider:// URL must be resolved via FileService — // it must NEVER be handed to the HTTP downloader (which would fail with // "unsupported URL scheme"). On unrecoverable read failure for a single // image, skip it and still trigger finalize so the parent knowledge // doesn't get stuck in "processing" forever (see issue #1282). imgBytes, readErr := s.readImageBytes(ctx, payload) if readErr != nil { logger.Errorf(ctx, "[ImageMultimodal] Skip unreadable image %s: %v", payload.ImageURL, readErr) s.checkAndFinalizeAllImages(ctx, payload) return nil } imageInfo := types.ImageInfo{ URL: payload.ImageURL, OriginalURL: payload.ImageURL, } if payload.EnableOCR { prompt := vlmOCRPrompt if payload.ImageSourceType == "scanned_pdf" { prompt = vlmOCRScannedPDFPrompt logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL) } ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt) if ocrErr != nil { logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr) } else { ocrText = sanitizeOCRText(ocrText) if ocrText != "" { imageInfo.OCRText = ocrText } else { logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL) } } } caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt) if capErr != nil { logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr) } else if caption != "" { imageInfo.Caption = caption } // Build child chunks for OCR and caption results imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo}) var newChunks []*types.Chunk if imageInfo.OCRText != "" { newChunks = append(newChunks, &types.Chunk{ ID: uuid.New().String(), TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Content: imageInfo.OCRText, ChunkType: types.ChunkTypeImageOCR, ParentChunkID: payload.ChunkID, IsEnabled: true, Flags: types.ChunkFlagRecommended, ImageInfo: string(imageInfoJSON), CreatedAt: time.Now(), UpdatedAt: time.Now(), }) } if imageInfo.Caption != "" { newChunks = append(newChunks, &types.Chunk{ ID: uuid.New().String(), TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Content: imageInfo.Caption, ChunkType: types.ChunkTypeImageCaption, ParentChunkID: payload.ChunkID, IsEnabled: true, Flags: types.ChunkFlagRecommended, ImageInfo: string(imageInfoJSON), CreatedAt: time.Now(), UpdatedAt: time.Now(), }) } if len(newChunks) == 0 { s.checkAndFinalizeAllImages(ctx, payload) return nil } // Persist chunks if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil { return fmt.Errorf("create multimodal chunks: %w", err) } for _, c := range newChunks { logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d", c.ChunkType, c.ID, payload.ImageURL, len(c.Content)) } // Index chunks so they can be retrieved s.indexChunks(ctx, payload, newChunks) // Enqueue question generation for the caption/OCR content if KB has it enabled. // During initial processChunks, question generation is skipped for image-type // knowledge because the text chunk is just a markdown reference. Now that we // have real textual content (caption/OCR), we can generate questions. // Note: for documents with multiple images (e.g. PDFs), we also wait until // all images are processed before triggering summary/question generation. s.checkAndFinalizeAllImages(ctx, payload) return nil } // indexChunks indexes the newly created multimodal chunks into the retrieval engine // so they can participate in semantic search. func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) { kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil || kb == nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err) return } // Skip vector/keyword indexing when the KB has no embedding-based pipeline enabled // (e.g. Wiki-only KBs). Without this check, GetEmbeddingModel would fail because // EmbeddingModelID is intentionally empty for such KBs. The multimodal chunks // themselves are already persisted in the DB above, so skipping index here is safe. if !kb.NeedsEmbeddingModel() { logger.Infof(ctx, "[ImageMultimodal] Vector/keyword indexing disabled for KB %s, skipping index for %d multimodal chunks", kb.ID, len(chunks)) // Still mark chunks as indexed so downstream finalization sees a consistent state. for _, chunk := range chunks { dbChunk, gerr := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID) if gerr != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, gerr) continue } dbChunk.Status = int(types.ChunkStatusIndexed) if uerr := s.chunkService.UpdateChunk(ctx, dbChunk); uerr != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, uerr) } } return } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err) return } tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err) return } // The factory's unbound path reads TenantInfo from ctx; make sure it's there. ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) // Resolve engine via the factory using the KB's VectorStore binding // (nil → tenant effective engines fallback; verified tenant ownership otherwise). engine, err := retriever.CreateRetrieveEngineForKB( ctx, s.retrieveEngine, s.ownership, payload.TenantID, kb.VectorStoreID) if err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err) return } indexInfoList := make([]*types.IndexInfo, 0, len(chunks)) for _, chunk := range chunks { indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) } if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil { logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err) return } // Mark chunks as indexed. // Must re-fetch from DB because the in-memory objects lack auto-generated fields // (e.g. seq_id), and GORM Save would overwrite them with zero values. for _, chunk := range chunks { dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID) if err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err) continue } dbChunk.Status = int(types.ChunkStatusIndexed) if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err) } } logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL) } // resolveVLM creates a vlm.VLM instance for the given knowledge base, // supporting both new-style (ModelID) and legacy (inline BaseURL) configs. func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) { kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) if err != nil { return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err) } if kb == nil { return nil, fmt.Errorf("knowledge base %s not found", kbID) } vlmCfg := kb.VLMConfig if !vlmCfg.IsEnabled() { return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID) } // New-style: resolve model through ModelService if vlmCfg.ModelID != "" { return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID) } // Legacy: create VLM from inline config return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService) } // resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs. // Falls back to the globally configured default FileService when the tenant's // StorageEngineConfig does not carry a usable configuration for the URL's provider. // This mirrors the write-side fallback in knowledgeService.resolveFileService // and is required because images can be saved using global STORAGE_TYPE/MINIO_* // env vars while tenant.StorageEngineConfig.MinIO is left empty (issue #1282). func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.ImageMultimodalPayload) interfaces.FileService { tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil || tenant == nil { logger.Warnf(ctx, "[ImageMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err) return s.fileSvc } provider := types.ParseProviderScheme(payload.ImageURL) if provider == "" { kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if kbErr != nil { logger.Warnf(ctx, "[ImageMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr) } else if kb != nil { provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider())) } } baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir) if svcErr != nil { logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr) return s.fileSvc } return fileSvc } // readImageBytes loads the image bytes for a multimodal payload. // - For provider:// URLs (local://, minio://, s3://, cos://, ...) it reads via // the resolved FileService and NEVER falls back to HTTP — handing a // provider:// URL to the HTTP downloader is what caused issue #1282. // - For legacy in-flight payloads with ImageLocalPath set, it tries the local // file before falling back to the URL. // - For plain http(s):// URLs it uses the SSRF-safe downloader. func (s *ImageMultimodalService) readImageBytes(ctx context.Context, payload types.ImageMultimodalPayload) ([]byte, error) { if types.ParseProviderScheme(payload.ImageURL) != "" { fileSvc := s.resolveFileServiceForPayload(ctx, payload) if fileSvc == nil { return nil, fmt.Errorf("no file service available for %s", payload.ImageURL) } reader, err := fileSvc.GetFile(ctx, payload.ImageURL) if err != nil { return nil, fmt.Errorf("file service get %s: %w", payload.ImageURL, err) } defer reader.Close() data, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("read %s: %w", payload.ImageURL, err) } return data, nil } if payload.ImageLocalPath != "" { if data, err := os.ReadFile(payload.ImageLocalPath); err == nil { return data, nil } else { logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), falling back to URL", payload.ImageLocalPath, err) } } data, err := downloadImageFromURL(payload.ImageURL) if err != nil { return nil, fmt.Errorf("download %s: %w", payload.ImageURL, err) } logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(data)) return data, nil } // downloadImageFromURL downloads image bytes from an HTTP(S) URL. func downloadImageFromURL(imageURL string) ([]byte, error) { return secutils.DownloadBytes(imageURL) } func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, payload types.ImageMultimodalPayload) { if s.redisClient == nil { s.enqueueKnowledgePostProcessTask(ctx, payload) return } redisKey := fmt.Sprintf("multimodal:pending:%s", payload.KnowledgeID) pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result() if err != nil && err != redis.Nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to decrement pending count for %s: %v", payload.KnowledgeID, err) return } if pendingCount <= 0 { logger.Infof(ctx, "[ImageMultimodal] All images processed for knowledge %s. Finalizing...", payload.KnowledgeID) s.redisClient.Del(ctx, redisKey) s.enqueueKnowledgePostProcessTask(ctx, payload) } } func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Context, payload types.ImageMultimodalPayload) { if s.taskEnqueuer == nil { return } taskPayload := types.KnowledgePostProcessPayload{ TenantID: payload.TenantID, KnowledgeID: payload.KnowledgeID, KnowledgeBaseID: payload.KnowledgeBaseID, Language: payload.Language, } langfuse.InjectTracing(ctx, &taskPayload) payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal post process payload: %v", err) return } task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err) } else { logger.Infof(ctx, "[ImageMultimodal] Enqueued post process task for %s", payload.KnowledgeID) } }