feat(image-multimodal): add task enqueuing for question generation based on image knowledge

- Introduced a new task enqueuer in the ImageMultimodalService to handle question generation for image knowledge.
- Implemented logic to enqueue question generation tasks when valid captions or OCR content are available, enhancing the knowledge base's interactivity.
- Updated image reference handling to ensure markdown URLs are correctly formatted, accommodating filenames with spaces.
- Enhanced regex pattern in image resolver to support filenames with spaces, improving image processing accuracy.
This commit is contained in:
wizardchen
2026-03-25 20:58:51 +08:00
committed by lyingbug
parent 1960fc2d90
commit 23290efcec
3 changed files with 60 additions and 3 deletions
@@ -43,6 +43,7 @@ type ImageMultimodalService struct {
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewImageMultimodalService(
@@ -53,6 +54,7 @@ func NewImageMultimodalService(
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
@@ -62,6 +64,7 @@ func NewImageMultimodalService(
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
@@ -212,6 +215,12 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
// Enqueue question generation for the caption/OCR content if KB has it enabled.
// During initial processChunks, question generation is skipped for image-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (caption/OCR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
@@ -400,6 +409,50 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the image knowledge.
func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Enqueued question generation task for image knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
return secutils.DownloadBytes(imageURL)
@@ -77,14 +77,16 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {
fileName = "image.png"
}
refPath := "images/" + fileName
// Encode spaces so the markdown URL is valid and matches the regex in ResolveAndStore.
safeRef := strings.ReplaceAll(refPath, " ", "%20")
mime := http.DetectContentType(data)
return &types.ReadResult{
MarkdownContent: fmt.Sprintf("![%s](%s)", fileName, refPath),
MarkdownContent: fmt.Sprintf("![%s](%s)", fileName, safeRef),
ImageRefs: []types.ImageRef{
{
Filename: fileName,
OriginalRef: refPath,
OriginalRef: safeRef,
MimeType: mime,
ImageData: data,
},
@@ -104,7 +104,9 @@ func (r *ImageResolver) ResolveAndStore(
// Process each image reference found in the markdown.
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
// Allow spaces in URLs (exclude only parens and newlines) to handle
// filenames with spaces, e.g. "images/第 1 页.jpg".
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
// Process in reverse order to preserve positions when replacing