Refactor knowledge post-processing into orchestrator task

Introduce KnowledgePostProcess task to decouple image multimodal
processing from downstream operations like summary, question
generation, and knowledge graph extraction.

This allows unifying the processing logic across documents and
images and ensures all text content, including from images,
is properly extracted before post-processing.
This commit is contained in:
wizardchen
2026-04-16 14:59:23 +08:00
parent 7c40fb6ac9
commit a1a07e6818
8 changed files with 505 additions and 278 deletions
@@ -19,16 +19,33 @@ import (
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
const (
vlmOCRPrompt = "Extract all body text content from this document image and output in pure Markdown format. Requirements:\n" +
"1. Ignore headers and footers\n" +
"2. Use Markdown table syntax for tables\n" +
"3. Use LaTeX format for formulas (wrapped with $ or $$)\n" +
"4. Organize content in the original reading order\n" +
"5. Only output extracted text content, do not add any HTML tags\n" +
"If there is no recognizable text content in the image, reply: No text content."
vlmOCRPrompt = "<system_prompt>\n" +
"You are an OCR assistant. Your task is to extract all body text content from this document image and output in pure Markdown format.\n" +
"</system_prompt>\n\n" +
"<instructions>\n" +
"1. Ignore headers and footers.\n" +
"2. Use Markdown table syntax for tables.\n" +
"3. Use LaTeX format for formulas (wrapped with $ or $$).\n" +
"4. Organize content in the original reading order.\n" +
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
"</instructions>"
vlmOCRScannedPDFPrompt = "<system_prompt>\n" +
"You are an OCR and document layout extraction assistant. The input image is a page from a scanned PDF document.\n" +
"Your task is to carefully extract all text and layout structure from the image, and output the result in pure Markdown format.\n" +
"</system_prompt>\n\n" +
"<instructions>\n" +
"1. Ignore headers, footers, and page numbers.\n" +
"2. Preserve the original document's paragraph and hierarchical structure as much as possible.\n" +
"3. If there are tables, use Markdown table syntax to represent them.\n" +
"4. If there are mathematical formulas, use LaTeX format wrapped in $ or $$.\n" +
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
"</instructions>"
vlmCaptionPrompt = "Provide a brief and concise description of the main content of the image in Chinese"
)
@@ -44,6 +61,7 @@ type ImageMultimodalService struct {
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
redisClient *redis.Client
}
func NewImageMultimodalService(
@@ -55,6 +73,7 @@ func NewImageMultimodalService(
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
redisClient *redis.Client,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
@@ -65,6 +84,7 @@ func NewImageMultimodalService(
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
}
}
@@ -133,7 +153,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
if payload.EnableOCR {
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmOCRPrompt)
prompt := vlmOCRPrompt
if payload.ImageSourceType == "scanned_pdf" {
prompt = vlmOCRScannedPDFPrompt
logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL)
}
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
if ocrErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
} else {
@@ -196,6 +222,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
if len(newChunks) == 0 {
// Even if OCR/caption both failed, mark knowledge as completed
s.finalizeImageKnowledge(ctx, payload, "")
s.checkAndFinalizeAllImages(ctx, payload)
return nil
}
@@ -222,7 +249,9 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
// During initial processChunks, question generation is skipped for image-type
// knowledge because the text chunk is just a markdown reference. Now that we
// have real textual content (caption/OCR), we can generate questions.
s.enqueueQuestionGenerationIfEnabled(ctx, payload)
// Note: for documents with multiple images (e.g. PDFs), we also wait until
// all images are processed before triggering summary/question generation.
s.checkAndFinalizeAllImages(ctx, payload)
return nil
}
@@ -412,52 +441,55 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
return fileSvc
}
// enqueueQuestionGenerationIfEnabled checks if the knowledge base has question
// generation enabled and, if so, enqueues a task for the image knowledge.
func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Enqueued question generation task for image knowledge %s (count=%d)",
payload.KnowledgeID, questionCount)
}
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
return secutils.DownloadBytes(imageURL)
}
func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.redisClient == nil {
s.enqueueKnowledgePostProcessTask(ctx, payload)
return
}
redisKey := fmt.Sprintf("multimodal:pending:%s", payload.KnowledgeID)
pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result()
if err != nil && err != redis.Nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to decrement pending count for %s: %v", payload.KnowledgeID, err)
return
}
if pendingCount <= 0 {
logger.Infof(ctx, "[ImageMultimodal] All images processed for knowledge %s. Finalizing...", payload.KnowledgeID)
s.redisClient.Del(ctx, redisKey)
// Enqueue the post process task to handle all downstream tasks (summary, question generation, etc)
s.enqueueKnowledgePostProcessTask(ctx, payload)
}
}
func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.taskEnqueuer == nil {
return
}
taskPayload := types.KnowledgePostProcessPayload{
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal post process payload: %v", err)
return
}
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Enqueued post process task for %s", payload.KnowledgeID)
}
}
+38 -44
View File
@@ -1500,6 +1500,7 @@ type ProcessChunksOptions struct {
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
ParentChunks []types.ParsedParentChunk
Metadata map[string]string
}
// buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase.
@@ -1875,17 +1876,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
logger.Infof(ctx, "processChunks create relationship rag task")
if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled {
for _, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed")
span.RecordError(err)
}
}
}
// Final check before marking as completed - if deleted during processing, don't update status
if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) {
logger.Infof(ctx, "Knowledge was deleted during processing, skipping completion update: %s", knowledge.ID)
@@ -1900,19 +1890,15 @@ func (s *knowledgeService) processChunks(ctx context.Context,
return
}
// Skip summary/question generation for image-type and video-type knowledge — the text chunk
// is just a markdown reference (or placeholder), so LLM summary would be useless.
// For images, the multimodal task provides a caption/summary as the description instead.
// Check if this document has extracted images that will be processed asynchronously
isImage := IsImageType(knowledge.FileType)
isVideo := IsVideoType(knowledge.FileType)
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
pendingPDFMultimodal := !isImage && !isVideo && options.EnableMultimodel && len(options.StoredImages) > 0
// For image files with pending multimodal processing, keep "processing" status
// so the frontend waits until the description is ready before showing "completed".
if pendingMultimodal {
// For image files or documents with pending multimodal processing, keep "processing" status
if pendingMultimodal || pendingPDFMultimodal {
knowledge.ParseStatus = types.ParseStatusProcessing
} else {
knowledge.ParseStatus = types.ParseStatusCompleted
}
knowledge.EnableStatus = "enabled"
knowledge.StorageSize = totalStorageSize
@@ -1920,37 +1906,32 @@ func (s *knowledgeService) processChunks(ctx context.Context,
knowledge.ProcessedAt = &now
knowledge.UpdatedAt = now
// Set summary status based on whether summary generation will be triggered
if len(textChunks) > 0 && !isImage && !isVideo {
knowledge.SummaryStatus = types.SummaryStatusPending
} else {
knowledge.SummaryStatus = types.SummaryStatusNone
}
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed")
}
// Enqueue question generation task if enabled (async, non-blocking)
if options.EnableQuestionGeneration && len(textChunks) > 0 && !isImage && !isVideo {
questionCount := options.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
s.enqueueQuestionGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID, questionCount)
}
// Enqueue summary generation task (async, non-blocking)
if len(textChunks) > 0 && !isImage && !isVideo {
s.enqueueSummaryGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
}
// Enqueue multimodal tasks for images (async, non-blocking)
if options.EnableMultimodel && len(options.StoredImages) > 0 {
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks)
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks, options.Metadata)
} else {
// If there are no multimodal tasks, enqueue the post process task immediately
lang, _ := types.LanguageFromContext(ctx)
payloadBytes, err := json.Marshal(types.KnowledgePostProcessPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Language: lang,
})
if err == nil {
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
if _, err := s.task.Enqueue(task); err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
} else {
logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)
}
} else {
logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)
}
}
// Update tenant's storage usage
@@ -7952,6 +7933,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
StoredImages: storedImages,
}
if convertResult != nil {
processOpts.Metadata = convertResult.Metadata
}
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
@@ -8132,11 +8117,19 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
kb *types.KnowledgeBase,
images []docparser.StoredImage,
chunks []types.ParsedChunk,
metadata map[string]string,
) {
if s.task == nil || len(images) == 0 {
return
}
redisKey := fmt.Sprintf("multimodal:pending:%s", knowledge.ID)
if s.redisClient != nil {
if err := s.redisClient.Set(ctx, redisKey, len(images), 24*time.Hour).Err(); err != nil {
logger.Warnf(ctx, "Failed to set multimodal pending count for %s: %v", knowledge.ID, err)
}
}
for _, img := range images {
// Match image to the ParsedChunk whose content contains the image URL.
// ChunkID was populated by processChunks with the real DB UUID.
@@ -8161,6 +8154,7 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
EnableOCR: true,
EnableCaption: true,
Language: lang,
ImageSourceType: metadata["image_source_type"],
}
payloadBytes, err := json.Marshal(payload)
@@ -0,0 +1,182 @@
package service
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
)
// KnowledgePostProcessService acts as an orchestrator for all post-processing tasks
// after a document has been parsed and split into chunks (including multimodal OCR/Caption).
type KnowledgePostProcessService struct {
knowledgeRepo interfaces.KnowledgeRepository
kbService interfaces.KnowledgeBaseService
chunkService interfaces.ChunkService
taskEnqueuer interfaces.TaskEnqueuer
}
func NewKnowledgePostProcessService(
knowledgeRepo interfaces.KnowledgeRepository,
kbService interfaces.KnowledgeBaseService,
chunkService interfaces.ChunkService,
taskEnqueuer interfaces.TaskEnqueuer,
) interfaces.TaskHandler {
return &KnowledgePostProcessService{
knowledgeRepo: knowledgeRepo,
kbService: kbService,
chunkService: chunkService,
taskEnqueuer: taskEnqueuer,
}
}
// Handle implements asynq handler for TypeKnowledgePostProcess.
func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.KnowledgePostProcessPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal knowledge post process payload: %w", err)
}
logger.Infof(ctx, "[KnowledgePostProcess] Orchestrating post processing for knowledge: %s", payload.KnowledgeID)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// 1. Fetch Knowledge and KB
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err)
}
if knowledge == nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID)
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err)
}
// 2. Fetch all chunks
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("list chunks for knowledge %s: %w", payload.KnowledgeID, err)
}
// Gather all text-like chunks (including newly added OCR and Caption from multimodal tasks)
var textChunks []*types.Chunk
for _, c := range chunks {
if c.ChunkType == types.ChunkTypeText || c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
textChunks = append(textChunks, c)
}
}
// 3. Update ParseStatus to Completed
// (Except if it's already completed or if it was marked as failed/deleting, but we'll just set it to completed if it's processing)
if knowledge.ParseStatus == types.ParseStatusProcessing {
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
// Setup summary status
if len(textChunks) > 0 {
knowledge.SummaryStatus = types.SummaryStatusPending
} else {
knowledge.SummaryStatus = types.SummaryStatusNone
}
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to update knowledge status to completed: %v", err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s marked as completed.", payload.KnowledgeID)
}
}
// 4. Spawn Summary and Question Tasks
if len(textChunks) > 0 {
s.enqueueSummaryGenerationTask(ctx, payload)
s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb)
}
// 5. Spawn Graph RAG Tasks
if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled {
logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks))
for _, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID)
if err != nil {
logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err)
}
}
}
return nil
}
func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload) {
if s.taskEnqueuer == nil {
return
}
taskPayload := types.SummaryGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal summary generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue summary generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued summary generation task for %s", payload.KnowledgeID)
}
}
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) {
if s.taskEnqueuer == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount)
}
}
+2
View File
@@ -175,6 +175,8 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
must(container.Provide(service.NewKnowledgePostProcessService, dig.Name("knowledgePostProcess")))
must(container.Provide(service.NewMessageService))
must(container.Provide(service.NewMCPServiceService))
+2
View File
@@ -79,6 +79,7 @@ type SyncTaskParams struct {
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"`
}
// RegisterSyncHandlers registers all task handlers on the SyncTaskExecutor.
@@ -97,6 +98,7 @@ func RegisterSyncHandlers(params SyncTaskParams) {
params.Executor.RegisterHandler(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
params.Executor.RegisterHandler(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete)
params.Executor.RegisterHandler(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
params.Executor.RegisterHandler(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
params.Executor.RegisterHandler(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
logger.Infof(context.Background(), "[SyncTask] All task handlers registered (Lite mode, no Redis)")
}
+4
View File
@@ -23,6 +23,7 @@ type AsynqTaskParams struct {
ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"`
DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"`
ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"`
KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"`
}
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
@@ -109,6 +110,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register image multimodal handler
mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
// Register knowledge post process handler
mux.HandleFunc(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
// Register data source sync handler
mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
-180
View File
@@ -1,185 +1,5 @@
package types
const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
TypeIndexDelete = "index:delete" // 索引删除任务
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
)
// ExtractChunkPayload represents the extract chunk task payload
type ExtractChunkPayload struct {
TenantID uint64 `json:"tenant_id"`
ChunkID string `json:"chunk_id"`
ModelID string `json:"model_id"`
}
// DocumentProcessPayload represents the document process task payload
type DocumentProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
URL string `json:"url,omitempty"` // URL(URL导入时使用)
FileURL string `json:"file_url,omitempty"` // 文件资源链接(file_url导入时使用)
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
EnableMultimodel bool `json:"enable_multimodel"`
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// FAQImportPayload represents the FAQ import task payload (including dry run mode)
type FAQImportPayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id,omitempty"` // 仅非 dry run 模式需要
Entries []FAQEntryPayload `json:"entries,omitempty"` // 小数据量时直接存储在 payload 中
EntriesURL string `json:"entries_url,omitempty"` // 大数据量时存储到对象存储,这里存储 URL
EntryCount int `json:"entry_count,omitempty"` // 条目总数(使用 EntriesURL 时需要)
Mode string `json:"mode"`
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
}
// QuestionGenerationPayload represents the question generation task payload
type QuestionGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
QuestionCount int `json:"question_count"`
// Language is the request locale (e.g. zh-CN, en-US) when the task was enqueued, used for {{language}} / {{lang}} in templates.
Language string `json:"language,omitempty"`
}
// SummaryGenerationPayload represents the summary generation task payload
type SummaryGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
Language string `json:"language,omitempty"`
}
// KBClonePayload represents the knowledge base clone task payload
type KBClonePayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
}
// IndexDeletePayload represents the index delete task payload
type IndexDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EmbeddingModelID string `json:"embedding_model_id"`
KBType string `json:"kb_type"`
ChunkIDs []string `json:"chunk_ids"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
}
// KBDeletePayload represents the knowledge base delete task payload
type KBDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
}
// KnowledgeListDeletePayload represents the batch knowledge delete task payload
type KnowledgeListDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeIDs []string `json:"knowledge_ids"`
}
// KnowledgeMovePayload represents the knowledge move task payload
type KnowledgeMovePayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
KnowledgeIDs []string `json:"knowledge_ids"`
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
Mode string `json:"mode"` // "reuse_vectors" or "reparse"
}
// KnowledgeMoveProgress represents the progress of a knowledge move task
type KnowledgeMoveProgress struct {
TaskID string `json:"task_id"`
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Failed int `json:"failed"` // 失败数
Message string `json:"message"` // 状态消息
Error string `json:"error"` // 错误信息
CreatedAt int64 `json:"created_at"` // 任务创建时间
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}
// ManualProcessPayload represents the manual knowledge processing task payload.
// Used for both create (publish) and update operations.
type ManualProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Content string `json:"content"` // cleaned markdown content
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
}
// ImageMultimodalPayload represents the image multimodal processing task payload.
type ImageMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
EnableOCR bool `json:"enable_ocr"`
EnableCaption bool `json:"enable_caption"`
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string
const (
KBCloneStatusPending KBCloneTaskStatus = "pending"
KBCloneStatusProcessing KBCloneTaskStatus = "processing"
KBCloneStatusCompleted KBCloneTaskStatus = "completed"
KBCloneStatusFailed KBCloneTaskStatus = "failed"
)
// KBCloneProgress represents the progress of a knowledge base clone task
type KBCloneProgress struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Message string `json:"message"` // 状态消息
Error string `json:"error"` // 错误信息
CreatedAt int64 `json:"created_at"` // 任务创建时间
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}
// ChunkContext represents chunk content with surrounding context
type ChunkContext struct {
ChunkID string `json:"chunk_id"`
+191
View File
@@ -0,0 +1,191 @@
package types
const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
TypeIndexDelete = "index:delete" // 索引删除任务
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
TypeKnowledgePostProcess = "knowledge:post_process" // 知识后处理任务(统一调度)
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
)
// ExtractChunkPayload represents the extract chunk task payload
type ExtractChunkPayload struct {
TenantID uint64 `json:"tenant_id"`
ChunkID string `json:"chunk_id"`
ModelID string `json:"model_id"`
}
// DocumentProcessPayload represents the document process task payload
type DocumentProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
URL string `json:"url,omitempty"` // URL(URL导入时使用)
FileURL string `json:"file_url,omitempty"` // 文件资源链接(file_url导入时使用)
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
EnableMultimodel bool `json:"enable_multimodel"`
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// FAQImportPayload represents the FAQ import task payload (including dry run mode)
type FAQImportPayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id,omitempty"` // 仅非 dry run 模式需要
Entries []FAQEntryPayload `json:"entries,omitempty"` // 小数据量时直接存储在 payload 中
EntriesURL string `json:"entries_url,omitempty"` // 大数据量时存储到对象存储,这里存储 URL
EntryCount int `json:"entry_count,omitempty"` // 条目总数(使用 EntriesURL 时需要)
Mode string `json:"mode"`
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
}
// QuestionGenerationPayload represents the question generation task payload
type QuestionGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
QuestionCount int `json:"question_count"`
// Language is the request locale (e.g. zh-CN, en-US) when the task was enqueued, used for {{language}} / {{lang}} in templates.
Language string `json:"language,omitempty"`
}
// SummaryGenerationPayload represents the summary generation task payload
type SummaryGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
Language string `json:"language,omitempty"`
}
// KBClonePayload represents the knowledge base clone task payload
type KBClonePayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
}
// IndexDeletePayload represents the index delete task payload
type IndexDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EmbeddingModelID string `json:"embedding_model_id"`
KBType string `json:"kb_type"`
ChunkIDs []string `json:"chunk_ids"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
}
// KBDeletePayload represents the knowledge base delete task payload
type KBDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
}
// KnowledgeListDeletePayload represents the batch knowledge delete task payload
type KnowledgeListDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeIDs []string `json:"knowledge_ids"`
}
// KnowledgeMovePayload represents the knowledge move task payload
type KnowledgeMovePayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
KnowledgeIDs []string `json:"knowledge_ids"`
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
Mode string `json:"mode"` // "reuse_vectors" or "reparse"
}
// KnowledgeMoveProgress represents the progress of a knowledge move task
type KnowledgeMoveProgress struct {
TaskID string `json:"task_id"`
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Failed int `json:"failed"` // 失败数
Message string `json:"message"` // 状态消息
Error string `json:"error"` // 错误信息
CreatedAt int64 `json:"created_at"` // 任务创建时间
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}
// ManualProcessPayload represents the manual knowledge processing task payload.
// Used for both create (publish) and update operations.
type ManualProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Content string `json:"content"` // cleaned markdown content
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
}
// ImageMultimodalPayload represents the image multimodal processing task payload.
type ImageMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
EnableOCR bool `json:"enable_ocr"`
EnableCaption bool `json:"enable_caption"`
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
ImageSourceType string `json:"image_source_type,omitempty"` // Source type of the image (e.g., "scanned_pdf")
}
// KnowledgePostProcessPayload represents the knowledge post process task payload.
type KnowledgePostProcessPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string
const (
KBCloneStatusPending KBCloneTaskStatus = "pending"
KBCloneStatusProcessing KBCloneTaskStatus = "processing"
KBCloneStatusCompleted KBCloneTaskStatus = "completed"
KBCloneStatusFailed KBCloneTaskStatus = "failed"
)
// KBCloneProgress represents the progress of a knowledge base clone task
type KBCloneProgress struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Message string `json:"message"` // 状态消息
Error string `json:"error"` // 错误信息
CreatedAt int64 `json:"created_at"` // 任务创建时间
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}