Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_post_process.go
T
wizardchen c9941a6688 refactor(knowledge): replace flat stage table with langfuse-style span tree
Addresses review feedback that the PR② design had four shortcomings:

  1. The pipeline is a DAG, not a sequence — Embedding and Multimodal
     are independent of each other, both downstream of Chunking, both
     upstream of PostProcess. The flat (knowledge_id, stage) table
     couldn't represent that, so a Chunking failure left dependents
     stranded as "pending" forever instead of being marked as
     impossible-to-run.

  2. No history across attempts. A reparse erased the previous run's
     status before the new run started, leaving operators with no way
     to investigate "why did this fail twice?".

  3. Stages had only status + duration. Operators want to know how big
     the work was — pages parsed, chunks created, tokens embedded, VLM
     calls made — to distinguish "slow because the file is huge" from
     "slow because the docreader is wedged".

  4. Multimodal fans out N image tasks; Embedding fans out M batches;
     PostProcess fans out into Summary/Question/Wiki/Graph. Each unit
     is interesting on its own (Langfuse already captures this for
     LLM calls). The flat model couldn't express it.

The redesign mirrors Langfuse's trace/span/generation hierarchy:

  * Migration 000053 supersedes 000052: knowledge_processing_spans
    table with (knowledge_id, attempt, span_id) primary key, plus
    parent_span_id, kind ∈ {root, stage, subspan, generation},
    status ∈ {pending,running,done,failed,skipped,cancelled}, and
    JSONB input/output/metadata fields.

  * SpanTracker (replacing StageTracker) exposes OpenAttempt /
    BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan /
    LookupStage. Cross-process workers (image_multimodal) get the
    parent's attempt + span via payload + LookupStage so subspans
    attach correctly.

  * StageDependencies declares the DAG; FailSpan now cascades —
    descendants of the failed span and dependent stages are flipped
    to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear
    blast radius instead of orphan spinners.

  * Reparse now calls OpenAttempt up front so the timeline reflects
    "new attempt, all pending" instead of letting the previous run's
    status linger until the worker picks up the task.

  * Image_multimodal records each image as a generation subspan with
    its own success/failure on the parent attempt's multimodal stage.
    The finalize-on-last-attempt counter logic is preserved unchanged.

  * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns
    a tree shape with synthesized pending placeholders so the
    frontend always renders five timeline segments. ?attempt=N
    enables history navigation.
2026-05-28 15:14:45 +08:00

235 lines
8.5 KiB
Go

package service
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
// KnowledgePostProcessService acts as an orchestrator for all post-processing tasks
// after a document has been parsed and split into chunks (including multimodal OCR/Caption).
type KnowledgePostProcessService struct {
knowledgeRepo interfaces.KnowledgeRepository
kbService interfaces.KnowledgeBaseService
chunkService interfaces.ChunkService
taskEnqueuer interfaces.TaskEnqueuer
pendingRepo interfaces.TaskPendingOpsRepository
redisClient *redis.Client
spanTracker SpanTracker
}
func NewKnowledgePostProcessService(
knowledgeRepo interfaces.KnowledgeRepository,
kbService interfaces.KnowledgeBaseService,
chunkService interfaces.ChunkService,
taskEnqueuer interfaces.TaskEnqueuer,
pendingRepo interfaces.TaskPendingOpsRepository,
redisClient *redis.Client,
spanTracker SpanTracker,
) interfaces.TaskHandler {
return &KnowledgePostProcessService{
knowledgeRepo: knowledgeRepo,
kbService: kbService,
chunkService: chunkService,
taskEnqueuer: taskEnqueuer,
pendingRepo: pendingRepo,
redisClient: redisClient,
spanTracker: spanTracker,
}
}
func (s *KnowledgePostProcessService) tracker() SpanTracker {
if s.spanTracker == nil {
return noopSpanTracker{}
}
return s.spanTracker
}
// Handle implements asynq handler for TypeKnowledgePostProcess.
func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.KnowledgePostProcessPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal knowledge post process payload: %w", err)
}
logger.Infof(ctx, "[KnowledgePostProcess] Orchestrating post processing for knowledge: %s", payload.KnowledgeID)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// Resolve attempt: payload carries it from the upstream stage, but
// fall back to the latest known attempt for compatibility with
// in-flight tasks queued before this code shipped.
attempt := payload.Attempt
if attempt <= 0 {
attempt = s.tracker().LatestAttempt(ctx, payload.KnowledgeID)
}
// Close the multimodal stage span (parent enqueued it as "running"
// and we never see the per-image fan-in here other than by reaching
// post-process). If the parent skipped multimodal entirely, the
// stage row will already be in "skipped" state and EndSpan is a
// no-op for missing rows.
if mm := s.tracker().LookupStage(ctx, payload.KnowledgeID, attempt, types.StageMultimodal); mm != nil &&
mm.Kind == types.SpanKindStage {
s.tracker().EndSpan(ctx, mm, nil)
}
postSpan := s.tracker().BeginStage(ctx, payload.KnowledgeID, attempt, types.StagePostProcess, nil)
// 1. Fetch Knowledge and KB
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err)
}
if knowledge == nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID)
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err)
}
// 2. Fetch all chunks
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
return fmt.Errorf("list chunks for knowledge %s: %w", payload.KnowledgeID, err)
}
// Gather all text-like chunks (including newly added OCR and Caption from multimodal tasks)
var textChunks []*types.Chunk
for _, c := range chunks {
if c.ChunkType == types.ChunkTypeText || c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
textChunks = append(textChunks, c)
}
}
// 3. Update ParseStatus to Completed
// (Except if it's already completed or if it was marked as failed/deleting, but we'll just set it to completed if it's processing)
if knowledge.ParseStatus == types.ParseStatusProcessing {
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
// Setup summary status
if len(textChunks) > 0 {
knowledge.SummaryStatus = types.SummaryStatusPending
} else {
knowledge.SummaryStatus = types.SummaryStatusNone
}
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to update knowledge status to completed: %v", err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s marked as completed.", payload.KnowledgeID)
}
}
// 4. Spawn Summary and Question Tasks
if len(textChunks) > 0 {
s.enqueueSummaryGenerationTask(ctx, payload)
// Question generation only makes sense for RAG indexing (improves chunk recall).
// Skip when only Wiki/Graph is enabled without vector/keyword search.
if kb.NeedsEmbeddingModel() {
s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb)
}
}
// 5. Spawn Graph RAG Tasks — only when graph indexing is enabled in IndexingStrategy
if kb.IsGraphEnabled() {
logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks))
for _, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID)
if err != nil {
logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err)
}
}
}
// 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy
if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 {
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
}
s.tracker().EndSpan(ctx, postSpan, types.JSONMap{
"chunks_total": len(textChunks),
})
return nil
}
func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload) {
if s.taskEnqueuer == nil {
return
}
taskPayload := types.SummaryGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
Language: payload.Language,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal summary generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue summary generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued summary generation task for %s", payload.KnowledgeID)
}
}
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) {
if s.taskEnqueuer == nil {
return
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
return
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
taskPayload := types.QuestionGenerationPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
QuestionCount: questionCount,
Language: payload.Language,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
} else {
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount)
}
}