feat(image-multimodal): add task enqueuing for question generation based on image knowledge

- Introduced a new task enqueuer in the ImageMultimodalService to handle question generation for image knowledge.
- Implemented logic to enqueue question generation tasks when valid captions or OCR content are available, enhancing the knowledge base's interactivity.
- Updated image reference handling to ensure markdown URLs are correctly formatted, accommodating filenames with spaces.
- Enhanced regex pattern in image resolver to support filenames with spaces, improving image processing accuracy.
This commit is contained in:
wizardchen
2026-03-25 20:58:51 +08:00
committed by lyingbug
parent 1960fc2d90
commit 23290efcec
3 changed files with 60 additions and 3 deletions
@@ -43,6 +43,7 @@ type ImageMultimodalService struct {
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewImageMultimodalService(
@@ -53,6 +54,7 @@ func NewImageMultimodalService(
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
@@ -62,6 +64,7 @@ func NewImageMultimodalService(
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
}
}
@@ -212,6 +215,12 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
// Enqueue question generation for the caption/OCR content if KB has it enabled.
// During initial processChunks, question generation is skipped for image-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (caption/OCR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
return nil
}
@@ -400,6 +409,50 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the image knowledge.
func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Enqueued question generation task for image knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
return secutils.DownloadBytes(imageURL)