feat(image-multimodal): add task enqueuing for question generation based on image knowledge
- Introduced a new task enqueuer in the ImageMultimodalService to handle question generation for image knowledge. - Implemented logic to enqueue question generation tasks when valid captions or OCR content are available, enhancing the knowledge base's interactivity. - Updated image reference handling to ensure markdown URLs are correctly formatted, accommodating filenames with spaces. - Enhanced regex pattern in image resolver to support filenames with spaces, improving image processing accuracy.
This commit is contained in:
@@ -43,6 +43,7 @@ type ImageMultimodalService struct {
|
||||
tenantRepo interfaces.TenantRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ollamaService *ollama.OllamaService
|
||||
taskEnqueuer interfaces.TaskEnqueuer
|
||||
}
|
||||
|
||||
func NewImageMultimodalService(
|
||||
@@ -53,6 +54,7 @@ func NewImageMultimodalService(
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ollamaService *ollama.OllamaService,
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
) interfaces.TaskHandler {
|
||||
return &ImageMultimodalService{
|
||||
chunkService: chunkService,
|
||||
@@ -62,6 +64,7 @@ func NewImageMultimodalService(
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ollamaService: ollamaService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,6 +215,12 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
// and mark the knowledge as completed (it was kept in "processing" until now).
|
||||
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
|
||||
|
||||
// Enqueue question generation for the caption/OCR content if KB has it enabled.
|
||||
// During initial processChunks, question generation is skipped for image-type
|
||||
// knowledge because the text chunk is just a markdown reference. Now that we
|
||||
// have real textual content (caption/OCR), we can generate questions.
|
||||
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -400,6 +409,50 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
|
||||
return fileSvc
|
||||
}
|
||||
|
||||
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
|
||||
// generation enabled and, if so, enqueues a task for the image knowledge.
|
||||
func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.ImageMultimodalPayload) {
|
||||
if s.taskEnqueuer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil || kb == nil {
|
||||
return
|
||||
}
|
||||
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
questionCount := kb.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
if questionCount > 10 {
|
||||
questionCount = 10
|
||||
}
|
||||
|
||||
taskPayload := types.QuestionGenerationPayload{
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal question generation payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Enqueued question generation task for image knowledge %s (count=%d)",
|
||||
payload.KnowledgeID, questionCount)
|
||||
}
|
||||
}
|
||||
|
||||
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
|
||||
func downloadImageFromURL(imageURL string) ([]byte, error) {
|
||||
return secutils.DownloadBytes(imageURL)
|
||||
|
||||
@@ -77,14 +77,16 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {
|
||||
fileName = "image.png"
|
||||
}
|
||||
refPath := "images/" + fileName
|
||||
// Encode spaces so the markdown URL is valid and matches the regex in ResolveAndStore.
|
||||
safeRef := strings.ReplaceAll(refPath, " ", "%20")
|
||||
mime := http.DetectContentType(data)
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: fmt.Sprintf("", fileName, refPath),
|
||||
MarkdownContent: fmt.Sprintf("", fileName, safeRef),
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: fileName,
|
||||
OriginalRef: refPath,
|
||||
OriginalRef: safeRef,
|
||||
MimeType: mime,
|
||||
ImageData: data,
|
||||
},
|
||||
|
||||
@@ -104,7 +104,9 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
// Process each image reference found in the markdown.
|
||||
// The URL group supports one level of balanced parentheses so that URLs
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||||
// Allow spaces in URLs (exclude only parens and newlines) to handle
|
||||
// filenames with spaces, e.g. "images/第 1 页.jpg".
|
||||
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
|
||||
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
|
||||
// Process in reverse order to preserve positions when replacing
|
||||
|
||||
Reference in New Issue
Block a user