fix(knowledge): close root span on terminal state, enrich stage metadata
The root span created by OpenAttempt was never closed: PostProcess only
ended the postprocess stage, so the root row stayed at status=running
forever even after parse_status flipped to completed/failed. The
timeline rendered "进行中" indefinitely on the root, defeating the
whole "is the document done" question the timeline is meant to answer.
- SpanTracker.FinalizeAttempt(kid, attempt, status, output, code, msg):
closes the root row idempotently. Re-closing a terminal root no-ops
so success / cascade-fail / dead-letter paths can fire without
coordination.
- PostProcess.Handle calls FinalizeAttempt(done) after EndSpan(postprocess)
on the success path. Async downstream work (summary/question/wiki/
graph) still records its own spans; their completion extends the
trace's wall-clock end-time but doesn't reopen the root.
- FailSpan auto-closes the root when a MAIN pipeline stage fails
(docreader / chunking / embedding / multimodal / postprocess).
Cascade-cancelled siblings stay closed-with-the-cascade as before.
- Dead-letter callback (router/task.go) accepts the SpanTracker via
DI and calls FinalizeAttempt(failed, TASK_TIMEOUT) when a
document-related task exhausts retries. The probe payload now
extracts the Attempt field that Document/Manual/PostProcess
payloads already thread through.
Stage spans were also being recorded with nil input/output, leaving
the new detail panel with timestamps only. Each Begin/End site now
emits useful work metrics:
- DocReader input: file_name, file_type, is_url, url
output: text_length, images_found, is_audio, pages
- Chunking input: chunks_planned
output: chunks_written, total_text_chars
- Embedding input: chunks_to_embed, model_id, dim
output: vectors_written, storage_bytes
- Multimodal input: image_count, enable_ocr, enable_caption
- PostProcess output: chunks_total, enqueued_summary, _question, _wiki, _graph
i18n: add knowledgeStages.root ("Knowledge processing") so the UI
can render a localized name instead of the raw span identifier.
This commit is contained in:
@@ -466,7 +466,9 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
// Chunks are needed for wiki generation, graph extraction, and summary generation
|
||||
// even when vector/keyword indexing is disabled.
|
||||
span.AddEvent("create chunks")
|
||||
s.beginStage(ctx, knowledge.ID, types.StageChunking, nil)
|
||||
s.beginStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{
|
||||
"chunks_planned": len(insertChunks),
|
||||
})
|
||||
if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil {
|
||||
knowledge.ParseStatus = types.ParseStatusFailed
|
||||
knowledge.ErrorMessage = err.Error()
|
||||
@@ -477,13 +479,27 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
werrors.ErrCodeChunkingFailed, "create chunks failed", err)
|
||||
return
|
||||
}
|
||||
s.endStage(ctx, knowledge.ID, types.StageChunking, nil)
|
||||
totalChunkChars := 0
|
||||
for _, c := range insertChunks {
|
||||
totalChunkChars += len(c.Content)
|
||||
}
|
||||
s.endStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{
|
||||
"chunks_written": len(insertChunks),
|
||||
"total_text_chars": totalChunkChars,
|
||||
})
|
||||
|
||||
// Create index information and perform vector indexing — only when vector/keyword is enabled.
|
||||
// Chunks are ALWAYS saved to DB (above) because wiki and graph need them even without vector indexing.
|
||||
var totalStorageSize int64
|
||||
if kb.NeedsEmbeddingModel() && embeddingModel != nil {
|
||||
s.beginStage(ctx, knowledge.ID, types.StageEmbedding, nil)
|
||||
embedInput := types.JSONMap{
|
||||
"chunks_to_embed": len(textChunks),
|
||||
"model_id": kb.EmbeddingModelID,
|
||||
}
|
||||
if dim := embeddingModel.GetDimensions(); dim > 0 {
|
||||
embedInput["dim"] = dim
|
||||
}
|
||||
s.beginStage(ctx, knowledge.ID, types.StageEmbedding, embedInput)
|
||||
// Create index information — only for child/flat chunks, NOT parent chunks.
|
||||
// Parent chunks are stored for context retrieval but do not need vector embeddings.
|
||||
// Prepend the document title to improve semantic alignment between
|
||||
@@ -576,7 +592,10 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
return
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
|
||||
s.endStage(ctx, knowledge.ID, types.StageEmbedding, nil)
|
||||
s.endStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{
|
||||
"vectors_written": len(indexInfoList),
|
||||
"storage_bytes": totalStorageSize,
|
||||
})
|
||||
|
||||
// Final check before marking as completed - if deleted during processing, don't update status
|
||||
if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) {
|
||||
@@ -617,7 +636,11 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
|
||||
// Enqueue multimodal tasks for images (async, non-blocking)
|
||||
if options.EnableMultimodel && len(options.StoredImages) > 0 {
|
||||
s.beginStage(ctx, knowledge.ID, types.StageMultimodal, nil)
|
||||
s.beginStage(ctx, knowledge.ID, types.StageMultimodal, types.JSONMap{
|
||||
"image_count": len(options.StoredImages),
|
||||
"enable_ocr": true,
|
||||
"enable_caption": true,
|
||||
})
|
||||
s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks, options.Metadata)
|
||||
} else {
|
||||
s.skipStage(ctx, knowledge.ID, types.StageMultimodal, "skipped")
|
||||
@@ -2336,7 +2359,15 @@ func (s *knowledgeService) convert(
|
||||
// up — before that, the stage stays "pending" from the initial
|
||||
// upload. Failure/skip transitions are emitted at the specific
|
||||
// failure points below; success is emitted at the bottom.
|
||||
s.beginStage(ctx, knowledge.ID, types.StageDocReader, nil)
|
||||
docInput := types.JSONMap{
|
||||
"file_name": payload.FileName,
|
||||
"file_type": payload.FileType,
|
||||
"is_url": payload.URL != "",
|
||||
}
|
||||
if payload.URL != "" {
|
||||
docInput["url"] = payload.URL
|
||||
}
|
||||
s.beginStage(ctx, knowledge.ID, types.StageDocReader, docInput)
|
||||
isURL := payload.URL != ""
|
||||
fileType := payload.FileType
|
||||
overrides := s.getParserEngineOverridesFromContext(ctx)
|
||||
@@ -2426,7 +2457,15 @@ func (s *knowledgeService) convert(
|
||||
werrors.ErrCodeDocReaderParseFailed, result.Error, nil)
|
||||
return nil, nil
|
||||
}
|
||||
s.endStage(ctx, knowledge.ID, types.StageDocReader, nil)
|
||||
docOutput := types.JSONMap{
|
||||
"text_length": len(result.MarkdownContent),
|
||||
"images_found": len(result.ImageRefs),
|
||||
"is_audio": result.IsAudio,
|
||||
}
|
||||
if pages := result.Metadata["pages"]; pages != "" {
|
||||
docOutput["pages"] = pages
|
||||
}
|
||||
s.endStage(ctx, knowledge.ID, types.StageDocReader, docOutput)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user