Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_post_process.go
T
wizardchen 72272b2d62 feat(knowledge): count wiki ingest in the finalizing subtask counter
Wiki ingest runs asynchronously after the parse pipeline and was not
counted in pending_subtasks_count, so a document flipped to "completed"
while wiki generation was still minutes away (30s debounce + batch +
retries). That hid the in-progress state and dropped the stop-parse
affordance before wiki actually finished.

Count wiki as a single enrichment subtask (when WikiEnabled and the doc
has text chunks, matching the enqueue condition) so the row stays in
"finalizing" until wiki is done. The batch worker drains the slot at the
op's terminal state: once on successful map, and once when an op is
dead-lettered after exhausting in-batch retries. Retract ops (deleted
knowledge) are skipped. FinalizeSubtask guards both the decrement and the
promote, so a wiki op enqueued before this accounting shipped is a
harmless no-op on an already-completed row. A wiki op that never drains is
bounded by the existing housekeeping finalizing sweep.
2026-05-29 19:32:13 +08:00

368 lines
14 KiB
Go

package service
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
// KnowledgePostProcessService acts as an orchestrator for all post-processing tasks
// after a document has been parsed and split into chunks (including multimodal OCR/Caption).
type KnowledgePostProcessService struct {
knowledgeRepo interfaces.KnowledgeRepository
kbService interfaces.KnowledgeBaseService
chunkService interfaces.ChunkService
taskEnqueuer interfaces.TaskEnqueuer
pendingRepo interfaces.TaskPendingOpsRepository
redisClient *redis.Client
spanTracker SpanTracker
}
func NewKnowledgePostProcessService(
knowledgeRepo interfaces.KnowledgeRepository,
kbService interfaces.KnowledgeBaseService,
chunkService interfaces.ChunkService,
taskEnqueuer interfaces.TaskEnqueuer,
pendingRepo interfaces.TaskPendingOpsRepository,
redisClient *redis.Client,
spanTracker SpanTracker,
) interfaces.TaskHandler {
return &KnowledgePostProcessService{
knowledgeRepo: knowledgeRepo,
kbService: kbService,
chunkService: chunkService,
taskEnqueuer: taskEnqueuer,
pendingRepo: pendingRepo,
redisClient: redisClient,
spanTracker: spanTracker,
}
}
func (s *KnowledgePostProcessService) tracker() SpanTracker {
if s.spanTracker == nil {
return noopSpanTracker{}
}
return s.spanTracker
}
// Handle implements asynq handler for TypeKnowledgePostProcess.
func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.KnowledgePostProcessPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal knowledge post process payload: %w", err)
}
logger.Infof(ctx, "[KnowledgePostProcess] Orchestrating post processing for knowledge: %s", payload.KnowledgeID)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// Resolve attempt: payload carries it from the upstream stage, but
// fall back to the latest known attempt for compatibility with
// in-flight tasks queued before this code shipped.
attempt := payload.Attempt
if attempt <= 0 {
attempt = s.tracker().LatestAttempt(ctx, payload.KnowledgeID)
}
// Close the multimodal stage span (parent enqueued it as "running"
// and we never see the per-image fan-in here other than by reaching
// post-process). If the parent skipped multimodal entirely, the
// stage row will already be in "skipped" state and EndSpan is a
// no-op for missing rows. Per-image success/failure counts are NOT
// aggregated here — the frontend already walks the children when
// rendering the multimodal stage detail and counts them itself,
// avoiding an extra query path.
if mm := s.tracker().LookupStage(ctx, payload.KnowledgeID, attempt, types.StageMultimodal); mm != nil &&
mm.Kind == types.SpanKindStage {
s.tracker().EndSpan(ctx, mm, nil)
}
postSpan := s.tracker().BeginStage(ctx, payload.KnowledgeID, attempt, types.StagePostProcess, nil)
// 1. Fetch Knowledge and KB
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err)
}
if knowledge == nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID)
return nil
}
// Skip post-processing entirely when the knowledge has been cancelled
// by the user or marked for deletion. We must NOT enqueue summary /
// question / graph / wiki child tasks for an aborted knowledge. We
// MUST also close postSpan before returning, otherwise it stays in
// running state forever and the trace viewer shows an orange bar
// long after the user cancelled (the AbortAttempt sweep ran before
// we opened postSpan, so the sweep didn't catch this row).
switch knowledge.ParseStatus {
case types.ParseStatusCancelled, types.ParseStatusDeleting:
logger.Infof(ctx,
"[KnowledgePostProcess] Knowledge %s aborted (%s), skipping post-processing.",
payload.KnowledgeID, knowledge.ParseStatus,
)
s.tracker().SkipSpan(ctx, postSpan,
"knowledge "+knowledge.ParseStatus+" before postprocess started")
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err)
}
// 2. Fetch all chunks
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("list chunks for knowledge %s: %w", payload.KnowledgeID, err)
}
// Gather all text-like chunks (including newly added OCR and Caption from multimodal tasks)
var textChunks []*types.Chunk
for _, c := range chunks {
if c.ChunkType == types.ChunkTypeText || c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
textChunks = append(textChunks, c)
}
}
// 3. Compute the enrichment subtask count up front so we can flip to
// "finalizing" with the right counter BEFORE spawning any subtasks.
// Each subtask handler atomically decrements pending_subtasks_count
// on its terminal exit; the row promotes itself to "completed" when
// the counter hits zero (see knowledgeRepository.FinalizeSubtask).
//
// Wiki ingest IS counted (as a single subtask): although it's a
// KB-scoped debounced batch, each upload enqueues exactly one
// per-knowledge op, and the batch worker calls FinalizeSubtask once
// when that op reaches a terminal state (mapped successfully or
// dead-lettered after exhausting retries). Counting it keeps the row
// in "finalizing" — i.e. shown as in-progress and still cancellable —
// until wiki generation actually finishes instead of flipping to
// completed while wiki runs minutes later. A wiki op that never
// drains is bounded by the housekeeping finalizing sweep.
willSpawnSummary := len(textChunks) > 0
willSpawnQuestion := willSpawnSummary && kb.NeedsEmbeddingModel() &&
kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled
willSpawnWiki := kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0
graphChunkCount := 0
if kb.IsGraphEnabled() {
graphChunkCount = len(textChunks)
}
expectedSubtasks := 0
if willSpawnSummary {
expectedSubtasks++
}
if willSpawnQuestion {
expectedSubtasks++
}
if willSpawnWiki {
expectedSubtasks++
}
expectedSubtasks += graphChunkCount
switch {
case knowledge.ParseStatus != types.ParseStatusProcessing:
// The row was already in some other state (deleting / cancelled /
// failed / completed) when we arrived. Don't touch parse_status
// and don't spawn enrichment — the upstream that put the row in
// that state has already decided this attempt is over.
logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s is in %s, skipping enrichment fan-out.",
payload.KnowledgeID, knowledge.ParseStatus)
s.tracker().EndSpan(ctx, postSpan, types.JSONMap{
"skipped": "non_processing_status",
"observed_status": knowledge.ParseStatus,
})
s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt,
types.SpanStatusDone, types.JSONMap{
"skipped": "non_processing_status",
"observed_status": knowledge.ParseStatus,
}, "", "")
return nil
case expectedSubtasks == 0:
// Nothing to enrich — fast path keeps the previous behavior so
// users without summary/question/graph see 'completed' immediately.
updates := map[string]interface{}{
"parse_status": types.ParseStatusCompleted,
"updated_at": time.Now(),
}
if len(textChunks) > 0 {
updates["summary_status"] = types.SummaryStatusNone
}
if err := s.knowledgeRepo.UpdateKnowledgeColumns(ctx, payload.KnowledgeID, updates); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to mark %s completed (no subtasks): %v",
payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s marked completed (no enrichment subtasks).",
payload.KnowledgeID)
}
default:
// Flip processing → finalizing in one statement so a parallel
// cancel/delete cannot race us into completed.
promoted, err := s.knowledgeRepo.SetFinalizing(ctx, payload.KnowledgeID, expectedSubtasks)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",
payload.KnowledgeID, err)
}
if promoted {
// Reflect summary status separately so the UI shows the
// summary as queued for users who already had it visible.
summaryStatus := types.SummaryStatusNone
if willSpawnSummary {
summaryStatus = types.SummaryStatusPending
}
if err := s.knowledgeRepo.UpdateKnowledgeColumn(ctx,
payload.KnowledgeID, "summary_status", summaryStatus); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to update summary_status for %s: %v",
payload.KnowledgeID, err)
}
logger.Infof(ctx,
"[KnowledgePostProcess] Knowledge %s entered finalizing (pending_subtasks=%d).",
payload.KnowledgeID, expectedSubtasks)
} else {
// Row was no longer 'processing' (cancel / delete won the race).
// Skip enrichment entirely so we don't waste LLM quota on a row
// the user already abandoned.
logger.Infof(ctx,
"[KnowledgePostProcess] Knowledge %s no longer in processing, skipping enrichment fan-out.",
payload.KnowledgeID)
s.tracker().EndSpan(ctx, postSpan, types.JSONMap{
"skipped": "knowledge_no_longer_processing",
})
s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt,
types.SpanStatusDone, types.JSONMap{
"skipped": "knowledge_no_longer_processing",
}, "", "")
return nil
}
}
// 4. Spawn Summary and Question Tasks
enqueuedSummary := false
enqueuedQuestion := false
if willSpawnSummary {
s.enqueueSummaryGenerationTask(ctx, payload, attempt)
enqueuedSummary = true
if willSpawnQuestion {
enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb, attempt)
}
}
// 5. Spawn Graph RAG Tasks — only when graph indexing is enabled in IndexingStrategy
enqueuedGraph := false
if graphChunkCount > 0 {
logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks))
for i, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID,
payload.KnowledgeID, attempt, i)
if err != nil {
logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err)
}
}
enqueuedGraph = true
}
// 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy
enqueuedWiki := false
if willSpawnWiki {
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
enqueuedWiki = true
}
postOutput := types.JSONMap{
"chunks_total": len(textChunks),
"enqueued_summary": enqueuedSummary,
"enqueued_question": enqueuedQuestion,
"enqueued_wiki": enqueuedWiki,
"enqueued_graph": enqueuedGraph,
}
s.tracker().EndSpan(ctx, postSpan, postOutput)
// Close the root span — the parse pipeline is done. Async
// downstream stages (summary/question/wiki/graph) record their
// own spans independently; their finishing extends the trace's
// end-time but does not reopen the root. A late failure in one
// of those stages does not poison the parse result.
s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt,
types.SpanStatusDone, postOutput, "", "")
return nil
}
func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload, attempt int) {
if s.taskEnqueuer == nil {
return
}
taskPayload := types.SummaryGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
Language: payload.Language,
Attempt: attempt,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal summary generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue summary generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued summary generation task for %s", payload.KnowledgeID)
}
}
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase, attempt int) bool {
if s.taskEnqueuer == nil {
return false
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return false
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
Attempt: attempt,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err)
return false
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
return false
}
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount)
return true
}