Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_span_tracker.go
T
wizardchen c9941a6688 refactor(knowledge): replace flat stage table with langfuse-style span tree
Addresses review feedback that the PR② design had four shortcomings:

  1. The pipeline is a DAG, not a sequence — Embedding and Multimodal
     are independent of each other, both downstream of Chunking, both
     upstream of PostProcess. The flat (knowledge_id, stage) table
     couldn't represent that, so a Chunking failure left dependents
     stranded as "pending" forever instead of being marked as
     impossible-to-run.

  2. No history across attempts. A reparse erased the previous run's
     status before the new run started, leaving operators with no way
     to investigate "why did this fail twice?".

  3. Stages had only status + duration. Operators want to know how big
     the work was — pages parsed, chunks created, tokens embedded, VLM
     calls made — to distinguish "slow because the file is huge" from
     "slow because the docreader is wedged".

  4. Multimodal fans out N image tasks; Embedding fans out M batches;
     PostProcess fans out into Summary/Question/Wiki/Graph. Each unit
     is interesting on its own (Langfuse already captures this for
     LLM calls). The flat model couldn't express it.

The redesign mirrors Langfuse's trace/span/generation hierarchy:

  * Migration 000053 supersedes 000052: knowledge_processing_spans
    table with (knowledge_id, attempt, span_id) primary key, plus
    parent_span_id, kind ∈ {root, stage, subspan, generation},
    status ∈ {pending,running,done,failed,skipped,cancelled}, and
    JSONB input/output/metadata fields.

  * SpanTracker (replacing StageTracker) exposes OpenAttempt /
    BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan /
    LookupStage. Cross-process workers (image_multimodal) get the
    parent's attempt + span via payload + LookupStage so subspans
    attach correctly.

  * StageDependencies declares the DAG; FailSpan now cascades —
    descendants of the failed span and dependent stages are flipped
    to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear
    blast radius instead of orphan spinners.

  * Reparse now calls OpenAttempt up front so the timeline reflects
    "new attempt, all pending" instead of letting the previous run's
    status linger until the worker picks up the task.

  * Image_multimodal records each image as a generation subspan with
    its own success/failure on the parent attempt's multimodal stage.
    The finalize-on-last-attempt counter logic is preserved unchanged.

  * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns
    a tree shape with synthesized pending placeholders so the
    frontend always renders five timeline segments. ?attempt=N
    enables history navigation.
2026-05-28 15:14:45 +08:00

509 lines
16 KiB
Go

// Package service: span tracker.
//
// SpanTracker is the pipeline-facing facade for recording per-attempt
// progress trees. It mirrors Langfuse's vocabulary (root / span /
// generation) so the UI's mental model matches what operators already use
// for LLM call observability.
//
// Lifecycle:
//
// attempt := tracker.OpenAttempt(ctx, knowledgeID, langfuseTraceID)
// // creates the root span; every subsequent Begin* call uses this attempt
//
// stage := tracker.BeginStage(ctx, knowledgeID, attempt, types.StageDocReader, input)
// // ...do work...
// tracker.EndSpan(ctx, stage, output) // success
// tracker.FailSpan(ctx, stage, code, msg, err) // error
// tracker.SkipSpan(ctx, stage, reason) // intentionally not run
//
// sub := tracker.BeginSubSpan(ctx, parentSpan, "multimodal.image[0]", types.SpanKindGeneration, input)
// // ...
//
// All operations are best-effort: a DB error is logged and swallowed so a
// tracker hiccup never breaks the parsing pipeline. Knowledge.parse_status
// remains the authoritative source of truth for completion.
package service
import (
"context"
"strings"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
)
// Span is the in-memory handle the pipeline holds while a stage / subspan
// is executing. It carries enough context for End/Fail/Skip to write back
// without re-querying the DB. Returned (and required) from every Begin*.
type Span struct {
KnowledgeID string
Attempt int
SpanID string
ParentSpanID string
Name string
Kind string
StartedAt time.Time
}
// SpanTracker is the only public surface — kept as an interface so tests
// can swap in a no-op without spinning up a database.
type SpanTracker interface {
// OpenAttempt creates a new root span for (knowledgeID,
// nextAttempt) and returns its number plus the root *Span. Call
// at the start of a parse / reparse, before any other Begin*.
OpenAttempt(ctx context.Context, knowledgeID, langfuseTraceID string) (root *Span, attempt int, err error)
// LatestAttempt returns the highest attempt number recorded for
// the knowledge, or 0 if it's never been parsed. Used by the API
// layer to default to "show me the most recent run".
LatestAttempt(ctx context.Context, knowledgeID string) int
// BeginStage starts one of the canonical stages. Looks up the
// root span for (kid, attempt) — caller passes attempt to make
// the wiring explicit and let cross-process workers join an
// existing attempt without new repo lookups.
BeginStage(ctx context.Context, knowledgeID string, attempt int, stage string, input types.JSONMap) *Span
// BeginSubSpan creates a child span under parent. parent may be a
// stage span (for multimodal.image[i] / embedding.batch[i]) or
// another subspan. kind is "subspan" or "generation" — generations
// will be stitched to a Langfuse generation by trace_id.
BeginSubSpan(ctx context.Context, parent *Span, name, kind string, input types.JSONMap) *Span
// EndSpan marks span as done with optional output. Safe with nil.
EndSpan(ctx context.Context, span *Span, output types.JSONMap)
// FailSpan marks span as failed and cascade-cancels its
// descendants. errorDetail (a Go error) is recorded verbatim in
// error_detail (truncated to 8 KB) for admin views.
FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)
// SkipSpan marks an intentionally not-run span (e.g. multimodal
// on a text-only document). Distinct from cancelled — skipped is
// "we chose not to" while cancelled is "an upstream broke".
SkipSpan(ctx context.Context, span *Span, reason string)
// LookupStage returns the stage's *Span for an in-flight attempt
// — the cross-process bridge that lets an asynq worker (e.g.
// image_multimodal) attach subspans to the parent stage span
// created by the upstream pipeline.
LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span
}
type spanTracker struct {
repo repository.KnowledgeSpanRepository
// startsMu guards the in-process duration cache. Cross-process
// workers won't find their parent's start here — that's fine,
// duration_ms falls back to (FinishedAt - row.StartedAt) computed
// at write time when the cache misses.
startsMu sync.Mutex
starts map[string]time.Time // span_id → started_at
}
// NewSpanTracker constructs the GORM-backed tracker. A nil repo collapses
// to a no-op so test harnesses don't need to spin up a database.
func NewSpanTracker(repo repository.KnowledgeSpanRepository) SpanTracker {
if repo == nil {
return noopSpanTracker{}
}
return &spanTracker{
repo: repo,
starts: make(map[string]time.Time),
}
}
func newSpanID() string {
// Stripping the dashes saves 4 bytes per row — JSON parsers don't
// care, and operators paste these into queries / Langfuse where a
// hex-only ID is friendlier.
return strings.ReplaceAll(uuid.NewString(), "-", "")
}
func (t *spanTracker) recordStart(spanID string, at time.Time) {
t.startsMu.Lock()
t.starts[spanID] = at
t.startsMu.Unlock()
}
func (t *spanTracker) takeStart(spanID string) (time.Time, bool) {
t.startsMu.Lock()
defer t.startsMu.Unlock()
v, ok := t.starts[spanID]
if ok {
delete(t.starts, spanID)
}
return v, ok
}
func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTraceID string) (*Span, int, error) {
attempt, err := t.repo.NextAttempt(ctx, knowledgeID)
if err != nil {
return nil, 0, err
}
now := time.Now()
rootID := newSpanID()
meta := types.JSONMap{}
if langfuseTraceID != "" {
// The frontend renders a "open in Langfuse" link from this.
meta["langfuse_trace_id"] = langfuseTraceID
}
row := &types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: rootID,
Name: "knowledge_processing",
Kind: types.SpanKindRoot,
Status: types.SpanStatusRunning,
Metadata: meta,
StartedAt: &now,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] OpenAttempt failed kid=%s: %v", knowledgeID, err)
return nil, attempt, err
}
t.recordStart(rootID, now)
return &Span{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: rootID,
Name: "knowledge_processing",
Kind: types.SpanKindRoot,
StartedAt: now,
}, attempt, nil
}
func (t *spanTracker) LatestAttempt(ctx context.Context, knowledgeID string) int {
n, err := t.repo.LatestAttempt(ctx, knowledgeID)
if err != nil {
logger.Warnf(ctx, "[SpanTracker] LatestAttempt failed kid=%s: %v", knowledgeID, err)
return 0
}
return n
}
func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attempt int, stage string, input types.JSONMap) *Span {
if knowledgeID == "" || stage == "" {
return nil
}
// Find root span — we need its span_id as parent for stage rows.
rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt)
if err != nil {
logger.Warnf(ctx, "[SpanTracker] BeginStage list failed kid=%s attempt=%d: %v",
knowledgeID, attempt, err)
return nil
}
var rootID string
for i := range rows {
if rows[i].Kind == types.SpanKindRoot {
rootID = rows[i].SpanID
break
}
}
if rootID == "" {
// Pipeline started before tracker was wired (legacy data,
// or the OpenAttempt repo write failed). Synthesize a
// rootless stage so we still record SOMETHING.
logger.Warnf(ctx, "[SpanTracker] BeginStage: no root for kid=%s attempt=%d, recording rootless",
knowledgeID, attempt)
}
now := time.Now()
id := newSpanID()
row := &types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: id,
ParentSpanID: rootID,
Name: stage,
Kind: types.SpanKindStage,
Status: types.SpanStatusRunning,
Input: input,
StartedAt: &now,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] BeginStage failed kid=%s stage=%s: %v",
knowledgeID, stage, err)
return nil
}
t.recordStart(id, now)
return &Span{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: id,
ParentSpanID: rootID,
Name: stage,
Kind: types.SpanKindStage,
StartedAt: now,
}
}
func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind string, input types.JSONMap) *Span {
if parent == nil || name == "" {
return nil
}
if kind != types.SpanKindGeneration && kind != types.SpanKindSubSpan {
kind = types.SpanKindSubSpan
}
now := time.Now()
id := newSpanID()
row := &types.KnowledgeProcessingSpan{
KnowledgeID: parent.KnowledgeID,
Attempt: parent.Attempt,
SpanID: id,
ParentSpanID: parent.SpanID,
Name: name,
Kind: kind,
Status: types.SpanStatusRunning,
Input: input,
StartedAt: &now,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] BeginSubSpan failed parent=%s name=%s: %v",
parent.SpanID, name, err)
return nil
}
t.recordStart(id, now)
return &Span{
KnowledgeID: parent.KnowledgeID,
Attempt: parent.Attempt,
SpanID: id,
ParentSpanID: parent.SpanID,
Name: name,
Kind: kind,
StartedAt: now,
}
}
func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSONMap) {
if span == nil {
return
}
now := time.Now()
dur := durationSince(t, span, now)
row := &types.KnowledgeProcessingSpan{
KnowledgeID: span.KnowledgeID,
Attempt: span.Attempt,
SpanID: span.SpanID,
ParentSpanID: span.ParentSpanID,
Name: span.Name,
Kind: span.Kind,
Status: types.SpanStatusDone,
Output: output,
StartedAt: &span.StartedAt,
FinishedAt: &now,
DurationMs: dur,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] EndSpan failed span=%s: %v", span.SpanID, err)
}
}
func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {
if span == nil {
return
}
now := time.Now()
dur := durationSince(t, span, now)
detail := ""
if errorDetail != nil {
detail = errorDetail.Error()
if len(detail) > 8192 {
detail = detail[:8192]
}
}
if len(errorMessage) > 1024 {
errorMessage = errorMessage[:1024]
}
row := &types.KnowledgeProcessingSpan{
KnowledgeID: span.KnowledgeID,
Attempt: span.Attempt,
SpanID: span.SpanID,
ParentSpanID: span.ParentSpanID,
Name: span.Name,
Kind: span.Kind,
Status: types.SpanStatusFailed,
ErrorCode: strings.TrimSpace(errorCode),
ErrorMessage: errorMessage,
ErrorDetail: detail,
StartedAt: &span.StartedAt,
FinishedAt: &now,
DurationMs: dur,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] FailSpan failed span=%s: %v", span.SpanID, err)
}
// Cascade: anything downstream of this span gets cancelled. The
// reason string is what the UI surfaces under each cancelled
// child's tooltip — keep it short and human.
reason := "upstream " + span.Name + " failed"
if errorCode != "" {
reason = reason + " (" + errorCode + ")"
}
if _, err := t.repo.CancelDescendants(ctx, span.KnowledgeID, span.Attempt, span.SpanID, reason); err != nil {
logger.Warnf(ctx, "[SpanTracker] cancel descendants failed span=%s: %v", span.SpanID, err)
}
// For STAGE failures, also cascade to dependent stages declared
// in StageDependencies (those are siblings, not descendants).
if span.Kind == types.SpanKindStage {
t.cascadeDependentStages(ctx, span, reason)
}
}
func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) {
if span == nil {
return
}
now := time.Now()
row := &types.KnowledgeProcessingSpan{
KnowledgeID: span.KnowledgeID,
Attempt: span.Attempt,
SpanID: span.SpanID,
ParentSpanID: span.ParentSpanID,
Name: span.Name,
Kind: span.Kind,
Status: types.SpanStatusSkipped,
ErrorMessage: reason,
StartedAt: &span.StartedAt,
FinishedAt: &now,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] SkipSpan failed span=%s: %v", span.SpanID, err)
}
}
func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span {
rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt)
if err != nil {
logger.Warnf(ctx, "[SpanTracker] LookupStage list failed kid=%s attempt=%d: %v",
knowledgeID, attempt, err)
return nil
}
for i := range rows {
r := rows[i]
if r.Kind != types.SpanKindStage || r.Name != stage {
continue
}
started := time.Time{}
if r.StartedAt != nil {
started = *r.StartedAt
}
return &Span{
KnowledgeID: r.KnowledgeID,
Attempt: r.Attempt,
SpanID: r.SpanID,
ParentSpanID: r.ParentSpanID,
Name: r.Name,
Kind: r.Kind,
StartedAt: started,
}
}
return nil
}
// cascadeDependentStages flips downstream STAGE rows to "cancelled" using
// types.StageDependencies. Without this, a Chunking failure leaves
// Embedding / Multimodal as "pending" forever, even though they cannot
// possibly run.
func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *Span, reason string) {
rows, err := t.repo.ListByAttempt(ctx, failedStage.KnowledgeID, failedStage.Attempt)
if err != nil {
return
}
dependents := stagesDependingOn(failedStage.Name)
if len(dependents) == 0 {
return
}
now := time.Now()
for _, row := range rows {
if row.Kind != types.SpanKindStage {
continue
}
if row.Status != types.SpanStatusPending && row.Status != types.SpanStatusRunning {
continue
}
if !contains(dependents, row.Name) {
continue
}
updated := row // copy
updated.Status = types.SpanStatusCancelled
updated.ErrorCode = "UPSTREAM_FAILED"
updated.ErrorMessage = reason
updated.FinishedAt = &now
if err := t.repo.Upsert(ctx, &updated); err != nil {
logger.Warnf(ctx, "[SpanTracker] cascade dependent stage %s: %v", row.Name, err)
}
}
}
// stagesDependingOn returns the transitive closure of stages that have
// `stage` as an upstream dependency (direct or indirect). Computed by
// reverse-walking StageDependencies; the result is bounded to 5 since
// AllStages has five members, so a naive O(N²) walk is fine.
func stagesDependingOn(stage string) []string {
var out []string
seen := map[string]bool{}
frontier := []string{stage}
for len(frontier) > 0 {
var next []string
for _, candidate := range types.AllStages {
if seen[candidate] {
continue
}
deps := types.StageDependencies[candidate]
for _, d := range deps {
if contains(frontier, d) {
seen[candidate] = true
out = append(out, candidate)
next = append(next, candidate)
break
}
}
}
frontier = next
}
return out
}
func contains(haystack []string, needle string) bool {
for _, h := range haystack {
if h == needle {
return true
}
}
return false
}
// durationSince computes elapsed ms preferring the in-process cache;
// falls back to the *Span's StartedAt for cross-process callers.
func durationSince(t *spanTracker, span *Span, now time.Time) int64 {
if start, ok := t.takeStart(span.SpanID); ok {
return now.Sub(start).Milliseconds()
}
if !span.StartedAt.IsZero() {
return now.Sub(span.StartedAt).Milliseconds()
}
return 0
}
// noopSpanTracker collapses every method to a no-op for tests/lite.
type noopSpanTracker struct{}
func (noopSpanTracker) OpenAttempt(_ context.Context, _, _ string) (*Span, int, error) {
return nil, 0, nil
}
func (noopSpanTracker) LatestAttempt(_ context.Context, _ string) int { return 0 }
func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string, _ types.JSONMap) *Span {
return nil
}
func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {
return nil
}
func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}
func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}
func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }