From 23290efcecea1ff9e4d25980a91b2c82c0c2b4c5 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 25 Mar 2026 20:58:51 +0800 Subject: [PATCH] feat(image-multimodal): add task enqueuing for question generation based on image knowledge - Introduced a new task enqueuer in the ImageMultimodalService to handle question generation for image knowledge. - Implemented logic to enqueue question generation tasks when valid captions or OCR content are available, enhancing the knowledge base's interactivity. - Updated image reference handling to ensure markdown URLs are correctly formatted, accommodating filenames with spaces. - Enhanced regex pattern in image resolver to support filenames with spaces, improving image processing accuracy. --- .../application/service/image_multimodal.go | 53 +++++++++++++++++++ .../docparser/builtin_converter.go | 6 ++- .../docparser/image_resolver.go | 4 +- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 4987da84..ce146fab 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -43,6 +43,7 @@ type ImageMultimodalService struct { tenantRepo interfaces.TenantRepository retrieveEngine interfaces.RetrieveEngineRegistry ollamaService *ollama.OllamaService + taskEnqueuer interfaces.TaskEnqueuer } func NewImageMultimodalService( @@ -53,6 +54,7 @@ func NewImageMultimodalService( tenantRepo interfaces.TenantRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ollamaService *ollama.OllamaService, + taskEnqueuer interfaces.TaskEnqueuer, ) interfaces.TaskHandler { return &ImageMultimodalService{ chunkService: chunkService, @@ -62,6 +64,7 @@ func NewImageMultimodalService( tenantRepo: tenantRepo, retrieveEngine: retrieveEngine, ollamaService: ollamaService, + taskEnqueuer: taskEnqueuer, } } @@ -212,6 +215,12 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e // and mark the knowledge as completed (it was kept in "processing" until now). s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption) + // Enqueue question generation for the caption/OCR content if KB has it enabled. + // During initial processChunks, question generation is skipped for image-type + // knowledge because the text chunk is just a markdown reference. Now that we + // have real textual content (caption/OCR), we can generate questions. + s.enqueueQuestionGenerationIfEnabled(ctx, payload) + return nil } @@ -400,6 +409,50 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex return fileSvc } +// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question +// generation enabled and, if so, enqueues a task for the image knowledge. +func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.ImageMultimodalPayload) { + if s.taskEnqueuer == nil { + return + } + + kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if err != nil || kb == nil { + return + } + if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled { + return + } + + questionCount := kb.QuestionGenerationConfig.QuestionCount + if questionCount <= 0 { + questionCount = 3 + } + if questionCount > 10 { + questionCount = 10 + } + + taskPayload := types.QuestionGenerationPayload{ + TenantID: payload.TenantID, + KnowledgeBaseID: payload.KnowledgeBaseID, + KnowledgeID: payload.KnowledgeID, + QuestionCount: questionCount, + } + payloadBytes, err := json.Marshal(taskPayload) + if err != nil { + logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal question generation payload: %v", err) + return + } + + task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) + if _, err := s.taskEnqueuer.Enqueue(task); err != nil { + logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err) + } else { + logger.Infof(ctx, "[ImageMultimodal] Enqueued question generation task for image knowledge %s (count=%d)", + payload.KnowledgeID, questionCount) + } +} + // downloadImageFromURL downloads image bytes from an HTTP(S) URL. func downloadImageFromURL(imageURL string) ([]byte, error) { return secutils.DownloadBytes(imageURL) diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go index 70480ee5..385116ff 100644 --- a/internal/infrastructure/docparser/builtin_converter.go +++ b/internal/infrastructure/docparser/builtin_converter.go @@ -77,14 +77,16 @@ func imageToResult(fileName string, data []byte) *types.ReadResult { fileName = "image.png" } refPath := "images/" + fileName + // Encode spaces so the markdown URL is valid and matches the regex in ResolveAndStore. + safeRef := strings.ReplaceAll(refPath, " ", "%20") mime := http.DetectContentType(data) return &types.ReadResult{ - MarkdownContent: fmt.Sprintf("![%s](%s)", fileName, refPath), + MarkdownContent: fmt.Sprintf("![%s](%s)", fileName, safeRef), ImageRefs: []types.ImageRef{ { Filename: fileName, - OriginalRef: refPath, + OriginalRef: safeRef, MimeType: mime, ImageData: data, }, diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go index aa8f86b4..fa031c1d 100644 --- a/internal/infrastructure/docparser/image_resolver.go +++ b/internal/infrastructure/docparser/image_resolver.go @@ -104,7 +104,9 @@ func (r *ImageResolver) ResolveAndStore( // Process each image reference found in the markdown. // The URL group supports one level of balanced parentheses so that URLs // like https://example.com/item_(abc)/123 are captured in full. - imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) + // Allow spaces in URLs (exclude only parens and newlines) to handle + // filenames with spaces, e.g. "images/第 1 页.jpg". + imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`) matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1) // Process in reverse order to preserve positions when replacing