diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 177c285c..083fb86b 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -390,6 +390,7 @@ export default { knowledgeStages: { title: 'Processing pipeline', + root: 'Knowledge processing', attempt: 'Attempt {n}', attemptLatest: 'Attempt {n} (latest)', retry: 'Retry parsing', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 6eae382e..9082eebe 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -396,6 +396,7 @@ export default { }, knowledgeStages: { title: "처리 파이프라인", + root: "지식 처리", attempt: "{n}번째 시도", attemptLatest: "{n}번째 시도 (최신)", retry: "다시 파싱", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 33a6fb25..700f53fc 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -360,6 +360,7 @@ export default { }, knowledgeStages: { title: 'Конвейер обработки', + root: 'Обработка знаний', attempt: 'Попытка {n}', attemptLatest: 'Попытка {n} (последняя)', retry: 'Повторить парсинг', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 7c330053..fd086acf 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -393,6 +393,7 @@ export default { }, knowledgeStages: { title: "处理流水线", + root: "知识处理", attempt: "第 {n} 次尝试", attemptLatest: "第 {n} 次尝试(最新)", retry: "重新解析", diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index c3a1e6a4..887349d6 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -79,7 +79,10 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // and we never see the per-image fan-in here other than by reaching // post-process). If the parent skipped multimodal entirely, the // stage row will already be in "skipped" state and EndSpan is a - // no-op for missing rows. + // no-op for missing rows. Per-image success/failure counts are NOT + // aggregated here — the frontend already walks the children when + // rendering the multimodal stage detail and counts them itself, + // avoiding an extra query path. if mm := s.tracker().LookupStage(ctx, payload.KnowledgeID, attempt, types.StageMultimodal); mm != nil && mm.Kind == types.SpanKindStage { s.tracker().EndSpan(ctx, mm, nil) @@ -137,16 +140,20 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta } // 4. Spawn Summary and Question Tasks + enqueuedSummary := false + enqueuedQuestion := false if len(textChunks) > 0 { s.enqueueSummaryGenerationTask(ctx, payload) + enqueuedSummary = true // Question generation only makes sense for RAG indexing (improves chunk recall). // Skip when only Wiki/Graph is enabled without vector/keyword search. if kb.NeedsEmbeddingModel() { - s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb) + enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb) } } // 5. Spawn Graph RAG Tasks — only when graph indexing is enabled in IndexingStrategy + enqueuedGraph := false if kb.IsGraphEnabled() { logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks)) for _, chunk := range textChunks { @@ -155,16 +162,31 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err) } } + enqueuedGraph = len(textChunks) > 0 } // 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy + enqueuedWiki := false if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 { EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) + enqueuedWiki = true } - s.tracker().EndSpan(ctx, postSpan, types.JSONMap{ - "chunks_total": len(textChunks), - }) + postOutput := types.JSONMap{ + "chunks_total": len(textChunks), + "enqueued_summary": enqueuedSummary, + "enqueued_question": enqueuedQuestion, + "enqueued_wiki": enqueuedWiki, + "enqueued_graph": enqueuedGraph, + } + s.tracker().EndSpan(ctx, postSpan, postOutput) + // Close the root span — the parse pipeline is done. Async + // downstream stages (summary/question/wiki/graph) record their + // own spans independently; their finishing extends the trace's + // end-time but does not reopen the root. A late failure in one + // of those stages does not poison the parse result. + s.tracker().FinalizeAttempt(ctx, payload.KnowledgeID, attempt, + types.SpanStatusDone, postOutput, "", "") return nil } @@ -194,13 +216,13 @@ func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.C } } -func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) { +func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) bool { if s.taskEnqueuer == nil { - return + return false } if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled { - return + return false } questionCount := kb.QuestionGenerationConfig.QuestionCount @@ -222,13 +244,14 @@ func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx con payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to marshal question generation payload: %v", err) - return + return false } task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation for %s: %v", payload.KnowledgeID, err) - } else { - logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount) + return false } + logger.Infof(ctx, "[KnowledgePostProcess] Enqueued question generation task for %s (count=%d)", payload.KnowledgeID, questionCount) + return true } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index ebbcb0aa..dffed0e1 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -466,7 +466,9 @@ func (s *knowledgeService) processChunks(ctx context.Context, // Chunks are needed for wiki generation, graph extraction, and summary generation // even when vector/keyword indexing is disabled. span.AddEvent("create chunks") - s.beginStage(ctx, knowledge.ID, types.StageChunking, nil) + s.beginStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{ + "chunks_planned": len(insertChunks), + }) if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = err.Error() @@ -477,13 +479,27 @@ func (s *knowledgeService) processChunks(ctx context.Context, werrors.ErrCodeChunkingFailed, "create chunks failed", err) return } - s.endStage(ctx, knowledge.ID, types.StageChunking, nil) + totalChunkChars := 0 + for _, c := range insertChunks { + totalChunkChars += len(c.Content) + } + s.endStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{ + "chunks_written": len(insertChunks), + "total_text_chars": totalChunkChars, + }) // Create index information and perform vector indexing — only when vector/keyword is enabled. // Chunks are ALWAYS saved to DB (above) because wiki and graph need them even without vector indexing. var totalStorageSize int64 if kb.NeedsEmbeddingModel() && embeddingModel != nil { - s.beginStage(ctx, knowledge.ID, types.StageEmbedding, nil) + embedInput := types.JSONMap{ + "chunks_to_embed": len(textChunks), + "model_id": kb.EmbeddingModelID, + } + if dim := embeddingModel.GetDimensions(); dim > 0 { + embedInput["dim"] = dim + } + s.beginStage(ctx, knowledge.ID, types.StageEmbedding, embedInput) // Create index information — only for child/flat chunks, NOT parent chunks. // Parent chunks are stored for context retrieval but do not need vector embeddings. // Prepend the document title to improve semantic alignment between @@ -576,7 +592,10 @@ func (s *knowledgeService) processChunks(ctx context.Context, return } logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList)) - s.endStage(ctx, knowledge.ID, types.StageEmbedding, nil) + s.endStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{ + "vectors_written": len(indexInfoList), + "storage_bytes": totalStorageSize, + }) // Final check before marking as completed - if deleted during processing, don't update status if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) { @@ -617,7 +636,11 @@ func (s *knowledgeService) processChunks(ctx context.Context, // Enqueue multimodal tasks for images (async, non-blocking) if options.EnableMultimodel && len(options.StoredImages) > 0 { - s.beginStage(ctx, knowledge.ID, types.StageMultimodal, nil) + s.beginStage(ctx, knowledge.ID, types.StageMultimodal, types.JSONMap{ + "image_count": len(options.StoredImages), + "enable_ocr": true, + "enable_caption": true, + }) s.enqueueImageMultimodalTasks(ctx, knowledge, kb, options.StoredImages, chunks, options.Metadata) } else { s.skipStage(ctx, knowledge.ID, types.StageMultimodal, "skipped") @@ -2336,7 +2359,15 @@ func (s *knowledgeService) convert( // up — before that, the stage stays "pending" from the initial // upload. Failure/skip transitions are emitted at the specific // failure points below; success is emitted at the bottom. - s.beginStage(ctx, knowledge.ID, types.StageDocReader, nil) + docInput := types.JSONMap{ + "file_name": payload.FileName, + "file_type": payload.FileType, + "is_url": payload.URL != "", + } + if payload.URL != "" { + docInput["url"] = payload.URL + } + s.beginStage(ctx, knowledge.ID, types.StageDocReader, docInput) isURL := payload.URL != "" fileType := payload.FileType overrides := s.getParserEngineOverridesFromContext(ctx) @@ -2426,7 +2457,15 @@ func (s *knowledgeService) convert( werrors.ErrCodeDocReaderParseFailed, result.Error, nil) return nil, nil } - s.endStage(ctx, knowledge.ID, types.StageDocReader, nil) + docOutput := types.JSONMap{ + "text_length": len(result.MarkdownContent), + "images_found": len(result.ImageRefs), + "is_audio": result.IsAudio, + } + if pages := result.Metadata["pages"]; pages != "" { + docOutput["pages"] = pages + } + s.endStage(ctx, knowledge.ID, types.StageDocReader, docOutput) return result, nil } diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go index eb05e25b..b81f6cda 100644 --- a/internal/application/service/knowledge_span_tracker.go +++ b/internal/application/service/knowledge_span_tracker.go @@ -93,6 +93,15 @@ type SpanTracker interface { // image_multimodal) attach subspans to the parent stage span // created by the upstream pipeline. LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span + + // FinalizeAttempt closes the root span for (knowledgeID, attempt) + // with the given terminal status (done | failed). Idempotent: + // re-closing an already-terminal root is a no-op so callers from + // multiple paths (success orchestrator, dead-letter handler, + // housekeeping) can fire without coordination. status defaults to + // done; output/error are written verbatim. + FinalizeAttempt(ctx context.Context, knowledgeID string, attempt int, status string, + output types.JSONMap, errorCode, errorMessage string) } type spanTracker struct { @@ -456,6 +465,16 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error // in StageDependencies (those are siblings, not descendants). if span.Kind == types.SpanKindStage { t.cascadeDependentStages(ctx, span, reason) + // Any failure in a MAIN pipeline stage means the attempt is + // done — the parse cannot succeed past this point. Close the + // root span as failed so the UI doesn't show "进行中" forever. + // Optional downstream stages (summary/question/wiki/graph) do + // not poison the attempt: they can fail without invalidating + // the parsed document. + if isMainPipelineStage(span.Name) { + t.FinalizeAttempt(ctx, span.KnowledgeID, span.Attempt, + types.SpanStatusFailed, nil, errorCode, errorMessage) + } } t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind) } @@ -596,6 +615,21 @@ func contains(haystack []string, needle string) bool { return false } +// isMainPipelineStage reports whether stage is one of the 5 mandatory +// pipeline stages (docreader / chunking / embedding / multimodal / +// postprocess). A failure in any of these terminally invalidates the +// attempt and must close the root as failed. Optional downstream stages +// added later (summary, question, wiki, graph) do NOT match — those +// can fail individually without poisoning the parse result. +func isMainPipelineStage(name string) bool { + for _, s := range types.AllStages { + if s == name { + return true + } + } + return false +} + // durationSince computes elapsed ms preferring the in-process cache; // falls back to the *Span's StartedAt for cross-process callers. func durationSince(t *spanTracker, span *Span, now time.Time) int64 { @@ -608,6 +642,88 @@ func durationSince(t *spanTracker, span *Span, now time.Time) int64 { return 0 } +// FinalizeAttempt closes the root span for (knowledgeID, attempt). The +// pipeline calls this from two places: the success orchestrator +// (PostProcess) when the parse reaches Completed, and FailSpan when a +// MAIN stage fails terminally. Without this, the root row created by +// OpenAttempt would stay in "running" forever even after parse_status +// flips to completed/failed — operators would see a perpetually +// "running" trace despite a terminal knowledge state. +// +// Idempotent: re-closing a root that's already done/failed is a no-op +// so callers from different paths (success vs. cascade-fail vs. +// dead-letter) don't have to coordinate. We deliberately avoid the +// recordStart cache here (cross-process callers won't have it); we +// recompute duration from the persisted row's started_at. +func (t *spanTracker) FinalizeAttempt(ctx context.Context, knowledgeID string, attempt int, status string, + output types.JSONMap, errorCode, errorMessage string, +) { + if knowledgeID == "" || attempt <= 0 { + return + } + if status == "" { + status = types.SpanStatusDone + } + rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt) + if err != nil { + logger.Warnf(ctx, "[SpanTracker] FinalizeAttempt list failed kid=%s attempt=%d: %v", + knowledgeID, attempt, err) + return + } + var root *types.KnowledgeProcessingSpan + for i := range rows { + if rows[i].Kind == types.SpanKindRoot { + cp := rows[i] + root = &cp + break + } + } + if root == nil { + // No root means nothing to close — likely an attempt that + // predates the tracker or whose OpenAttempt write failed. + return + } + if root.Status == types.SpanStatusDone || root.Status == types.SpanStatusFailed || + root.Status == types.SpanStatusCancelled || root.Status == types.SpanStatusSkipped { + return + } + now := time.Now() + var started time.Time + if root.StartedAt != nil { + started = *root.StartedAt + } + dur := int64(0) + if !started.IsZero() { + dur = now.Sub(started).Milliseconds() + } + if len(errorMessage) > 1024 { + errorMessage = errorMessage[:1024] + } + row := &types.KnowledgeProcessingSpan{ + KnowledgeID: root.KnowledgeID, + Attempt: root.Attempt, + SpanID: root.SpanID, + ParentSpanID: root.ParentSpanID, + Name: root.Name, + Kind: root.Kind, + Status: status, + Input: root.Input, + Output: output, + Metadata: root.Metadata, + ErrorCode: strings.TrimSpace(errorCode), + ErrorMessage: errorMessage, + StartedAt: root.StartedAt, + FinishedAt: &now, + DurationMs: dur, + } + if err := t.repo.Upsert(ctx, row); err != nil { + logger.Warnf(ctx, "[SpanTracker] FinalizeAttempt upsert failed kid=%s attempt=%d: %v", + knowledgeID, attempt, err) + return + } + t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindRoot) +} + // noopSpanTracker collapses every method to a no-op for tests/lite. type noopSpanTracker struct{} @@ -625,3 +741,5 @@ func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {} func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {} func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil } +func (noopSpanTracker) FinalizeAttempt(_ context.Context, _ string, _ int, _ string, _ types.JSONMap, _, _ string) { +} diff --git a/internal/router/task.go b/internal/router/task.go index ce1fb1eb..505e6a4b 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -34,6 +34,7 @@ type AsynqTaskParams struct { KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"` WikiIngest interfaces.TaskHandler `name:"wikiIngest"` DeadLetterRepo interfaces.TaskDeadLetterRepository + SpanTracker service.SpanTracker } // defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms @@ -165,7 +166,7 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // a permanently-failing task left its parent knowledge stranded in // "processing" until housekeeping cron caught it minutes later — the // UI signal users actually see. - knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService) + knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker) mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer)) // Install Langfuse middleware BEFORE handler registration so every task @@ -236,6 +237,12 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // depend on the full payload schema and survive future field churn. type deadLetterKnowledgePayload struct { KnowledgeID string `json:"knowledge_id,omitempty"` + // Attempt threads through DocumentProcess / ManualProcess / + // KnowledgePostProcess payloads (added when span tracking shipped) + // — extracted here so the dead-letter callback can also close the + // matching root span as failed. Older in-flight payloads without + // this field decode as 0 and the tracker call no-ops. + Attempt int `json:"attempt,omitempty"` } // taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose @@ -269,7 +276,7 @@ var taskTypesAffectingKnowledgeStatus = map[string]struct{}{ // errors are logged and swallowed. The dead-letter record is the source of // truth — this is purely a UX shortcut so users don't wait for the // housekeeping cron's next sweep. -func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDeadLetter { +func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker service.SpanTracker) asynqdl.OnDeadLetter { if ks == nil { return nil } @@ -303,6 +310,13 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDead logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err) return } + // Close the matching root span so the timeline stops showing + // "进行中" after dead-letter exhaustion. Best-effort: nil + // tracker / missing attempt / missing root all no-op cleanly. + if tracker != nil && probe.Attempt > 0 { + tracker.FinalizeAttempt(ctx, probe.KnowledgeID, probe.Attempt, + types.SpanStatusFailed, nil, "TASK_TIMEOUT", errMsg) + } logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type()) } }