fix(knowledge): close root span on terminal state, enrich stage metadata
The root span created by OpenAttempt was never closed: PostProcess only
ended the postprocess stage, so the root row stayed at status=running
forever even after parse_status flipped to completed/failed. The
timeline rendered "进行中" indefinitely on the root, defeating the
whole "is the document done" question the timeline is meant to answer.
- SpanTracker.FinalizeAttempt(kid, attempt, status, output, code, msg):
closes the root row idempotently. Re-closing a terminal root no-ops
so success / cascade-fail / dead-letter paths can fire without
coordination.
- PostProcess.Handle calls FinalizeAttempt(done) after EndSpan(postprocess)
on the success path. Async downstream work (summary/question/wiki/
graph) still records its own spans; their completion extends the
trace's wall-clock end-time but doesn't reopen the root.
- FailSpan auto-closes the root when a MAIN pipeline stage fails
(docreader / chunking / embedding / multimodal / postprocess).
Cascade-cancelled siblings stay closed-with-the-cascade as before.
- Dead-letter callback (router/task.go) accepts the SpanTracker via
DI and calls FinalizeAttempt(failed, TASK_TIMEOUT) when a
document-related task exhausts retries. The probe payload now
extracts the Attempt field that Document/Manual/PostProcess
payloads already thread through.
Stage spans were also being recorded with nil input/output, leaving
the new detail panel with timestamps only. Each Begin/End site now
emits useful work metrics:
- DocReader input: file_name, file_type, is_url, url
output: text_length, images_found, is_audio, pages
- Chunking input: chunks_planned
output: chunks_written, total_text_chars
- Embedding input: chunks_to_embed, model_id, dim
output: vectors_written, storage_bytes
- Multimodal input: image_count, enable_ocr, enable_caption
- PostProcess output: chunks_total, enqueued_summary, _question, _wiki, _graph
i18n: add knowledgeStages.root ("Knowledge processing") so the UI
can render a localized name instead of the raw span identifier.
This commit is contained in:
@@ -79,7 +79,10 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
// and we never see the per-image fan-in here other than by reaching
|
||||
// post-process). If the parent skipped multimodal entirely, the
|
||||
// stage row will already be in "skipped" state and EndSpan is a
|
||||
// no-op for missing rows.
|
||||
// no-op for missing rows. Per-image success/failure counts are NOT
|
||||
// aggregated here — the frontend already walks the children when
|
||||
// rendering the multimodal stage detail and counts them itself,
|
||||
// avoiding an extra query path.
|
||||
if mm := s.tracker().LookupStage(ctx, payload.KnowledgeID, attempt, types.StageMultimodal); mm != nil &&
|
||||
mm.Kind == types.SpanKindStage {
|
||||
s.tracker().EndSpan(ctx, mm, nil)
|
||||
@@ -137,16 +140,20 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
}
|
||||
|
||||
// 4. Spawn Summary and Question Tasks
|
||||
enqueuedSummary := false
|
||||
enqueuedQuestion := false
|
||||
if len(textChunks) > 0 {
|
||||
s.enqueueSummaryGenerationTask(ctx, payload)
|
||||
enqueuedSummary = true
|
||||
// Question generation only makes sense for RAG indexing (improves chunk recall).
|
||||
// Skip when only Wiki/Graph is enabled without vector/keyword search.
|
||||
if kb.NeedsEmbeddingModel() {
|
||||
s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb)
|
||||
enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Spawn Graph RAG Tasks — only when graph indexing is enabled in IndexingStrategy
|
||||
enqueuedGraph := false
|
||||
if kb.IsGraphEnabled() {
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks))
|
||||
for _, chunk := range textChunks {
|
||||
@@ -155,16 +162,31 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
enqueuedGraph = len(textChunks) > 0
|
||||
}
|
||||
|
||||
// 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy
|
||||
enqueuedWiki := false
|
||||
if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 {
|
||||
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
|
||||
enqueuedWiki = true
|
||||
}
|
||||
s.tracker().EndSpan(ctx, postSpan, types.JSONMap{
|
||||
"chunks_total": len(textChunks),
|
||||
})
|
||||
postOutput := types.JSONMap{
|
||||
"chunks_total": len(textChunks),
|
||||
"enqueued_summary": enqueuedSummary,
|
||||
"enqueued_question": enqueuedQuestion,
|
||||
"enqueued_wiki": enqueuedWiki,
|
||||
"enqueued_graph": enqueuedGraph,
|
||||
}
|
||||
s.tracker().EndSpan(ctx, postSpan, postOutput)
|
||||
// Close the root span — the parse pipeline is done. Async
|
||||
// downstream stages (summary/question/wiki/graph) record their
|
||||
// own spans independently; their finishing extends the trace's
|
||||
// end-time but does not reopen the root. A late failure in one
|
||||
// of those stages does not poison the parse result.
|
||||
s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt,
|
||||
types.SpanStatusDone, postOutput, "", "")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -194,13 +216,13 @@ func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.C
|
||||
}
|
||||
}
|
||||
|
||||
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) {
|
||||
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) bool {
|
||||
if s.taskEnqueuer == nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
questionCount := kb.QuestionGenerationConfig.QuestionCount
|
||||
@@ -222,13 +244,14 @@ func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx con
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount)
|
||||
return false
|
||||
}
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount)
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user