package service import ( "context" "encoding/json" "fmt" "time" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" ) // KnowledgePostProcessService acts as an orchestrator for all post-processing tasks // after a document has been parsed and split into chunks (including multimodal OCR/Caption). type KnowledgePostProcessService struct { knowledgeRepo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService chunkService interfaces.ChunkService taskEnqueuer interfaces.TaskEnqueuer pendingRepo interfaces.TaskPendingOpsRepository redisClient *redis.Client spanTracker SpanTracker } func NewKnowledgePostProcessService( knowledgeRepo interfaces.KnowledgeRepository, kbService interfaces.KnowledgeBaseService, chunkService interfaces.ChunkService, taskEnqueuer interfaces.TaskEnqueuer, pendingRepo interfaces.TaskPendingOpsRepository, redisClient *redis.Client, spanTracker SpanTracker, ) interfaces.TaskHandler { return &KnowledgePostProcessService{ knowledgeRepo: knowledgeRepo, kbService: kbService, chunkService: chunkService, taskEnqueuer: taskEnqueuer, pendingRepo: pendingRepo, redisClient: redisClient, spanTracker: spanTracker, } } func (s *KnowledgePostProcessService) tracker() SpanTracker { if s.spanTracker == nil { return noopSpanTracker{} } return s.spanTracker } // Handle implements asynq handler for TypeKnowledgePostProcess. func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Task) error { var payload types.KnowledgePostProcessPayload if err := json.Unmarshal(task.Payload(), &payload); err != nil { return fmt.Errorf("unmarshal knowledge post process payload: %w", err) } logger.Infof(ctx, "[KnowledgePostProcess] Orchestrating post processing for knowledge: %s", payload.KnowledgeID) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } // Resolve attempt: payload carries it from the upstream stage, but // fall back to the latest known attempt for compatibility with // in-flight tasks queued before this code shipped. attempt := payload.Attempt if attempt <= 0 { attempt = s.tracker().LatestAttempt(ctx, payload.KnowledgeID) } // Close the multimodal stage span (parent enqueued it as "running" // and we never see the per-image fan-in here other than by reaching // post-process). If the parent skipped multimodal entirely, the // stage row will already be in "skipped" state and EndSpan is a // no-op for missing rows. Per-image success/failure counts are NOT // aggregated here — the frontend already walks the children when // rendering the multimodal stage detail and counts them itself, // avoiding an extra query path. if mm := s.tracker().LookupStage(ctx, payload.KnowledgeID, attempt, types.StageMultimodal); mm != nil && mm.Kind == types.SpanKindStage { s.tracker().EndSpan(ctx, mm, nil) } postSpan := s.tracker().BeginStage(ctx, payload.KnowledgeID, attempt, types.StagePostProcess, nil) // 1. Fetch Knowledge and KB knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID) if err != nil { return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err) } if knowledge == nil { logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID) return nil } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil || kb == nil { return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err) } // 2. Fetch all chunks chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID) if err != nil { return fmt.Errorf("list chunks for knowledge %s: %w", payload.KnowledgeID, err) } // Gather all text-like chunks (including newly added OCR and Caption from multimodal tasks) var textChunks []*types.Chunk for _, c := range chunks { if c.ChunkType == types.ChunkTypeText || c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption { textChunks = append(textChunks, c) } } // 3. Update ParseStatus to Completed // (Except if it's already completed or if it was marked as failed/deleting, but we'll just set it to completed if it's processing) if knowledge.ParseStatus == types.ParseStatusProcessing { knowledge.ParseStatus = types.ParseStatusCompleted knowledge.UpdatedAt = time.Now() // Setup summary status if len(textChunks) > 0 { knowledge.SummaryStatus = types.SummaryStatusPending } else { knowledge.SummaryStatus = types.SummaryStatusNone } if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to update knowledge status to completed: %v", err) } else { logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s marked as completed.", payload.KnowledgeID) } } // 4. Spawn Summary and Question Tasks enqueuedSummary := false enqueuedQuestion := false if len(textChunks) > 0 { s.enqueueSummaryGenerationTask(ctx, payload, attempt) enqueuedSummary = true // Question generation only makes sense for RAG indexing (improves chunk recall). // Skip when only Wiki/Graph is enabled without vector/keyword search. if kb.NeedsEmbeddingModel() { enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb, attempt) } } // 5. Spawn Graph RAG Tasks — only when graph indexing is enabled in IndexingStrategy enqueuedGraph := false if kb.IsGraphEnabled() { logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks)) for i, chunk := range textChunks { err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID, payload.KnowledgeID, attempt, i) if err != nil { logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err) } } enqueuedGraph = len(textChunks) > 0 } // 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy enqueuedWiki := false if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 { EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) enqueuedWiki = true } postOutput := types.JSONMap{ "chunks_total": len(textChunks), "enqueued_summary": enqueuedSummary, "enqueued_question": enqueuedQuestion, "enqueued_wiki": enqueuedWiki, "enqueued_graph": enqueuedGraph, } s.tracker().EndSpan(ctx, postSpan, postOutput) // Close the root span — the parse pipeline is done. Async // downstream stages (summary/question/wiki/graph) record their // own spans independently; their finishing extends the trace's // end-time but does not reopen the root. A late failure in one // of those stages does not poison the parse result. s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt, types.SpanStatusDone, postOutput, "", "") return nil } func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload, attempt int) { if s.taskEnqueuer == nil { return } taskPayload := types.SummaryGenerationPayload{ TenantID: payload.TenantID, KnowledgeBaseID: payload.KnowledgeBaseID, KnowledgeID: payload.KnowledgeID, Language: payload.Language, Attempt: attempt, } langfuse.InjectTracing(ctx, &taskPayload) payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal summary generation payload: %v", err) return } task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue summary generation for %s: %v", payload.KnowledgeID, err) } else { logger.Infof(ctx, "[KnowledgePostProcess] Enqueued summary generation task for %s", payload.KnowledgeID) } } func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase, attempt int) bool { if s.taskEnqueuer == nil { return false } if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled { return false } questionCount := kb.QuestionGenerationConfig.QuestionCount if questionCount <= 0 { questionCount = 3 } if questionCount > 10 { questionCount = 10 } taskPayload := types.QuestionGenerationPayload{ TenantID: payload.TenantID, KnowledgeBaseID: payload.KnowledgeBaseID, KnowledgeID: payload.KnowledgeID, QuestionCount: questionCount, Language: payload.Language, Attempt: attempt, } langfuse.InjectTracing(ctx, &taskPayload) payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err) return false } task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err) return false } logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount) return true }