refactor(knowledge): replace flat stage table with langfuse-style span tree

Addresses review feedback that the PR② design had four shortcomings:

  1. The pipeline is a DAG, not a sequence — Embedding and Multimodal
     are independent of each other, both downstream of Chunking, both
     upstream of PostProcess. The flat (knowledge_id, stage) table
     couldn't represent that, so a Chunking failure left dependents
     stranded as "pending" forever instead of being marked as
     impossible-to-run.

  2. No history across attempts. A reparse erased the previous run's
     status before the new run started, leaving operators with no way
     to investigate "why did this fail twice?".

  3. Stages had only status + duration. Operators want to know how big
     the work was — pages parsed, chunks created, tokens embedded, VLM
     calls made — to distinguish "slow because the file is huge" from
     "slow because the docreader is wedged".

  4. Multimodal fans out N image tasks; Embedding fans out M batches;
     PostProcess fans out into Summary/Question/Wiki/Graph. Each unit
     is interesting on its own (Langfuse already captures this for
     LLM calls). The flat model couldn't express it.

The redesign mirrors Langfuse's trace/span/generation hierarchy:

  * Migration 000053 supersedes 000052: knowledge_processing_spans
    table with (knowledge_id, attempt, span_id) primary key, plus
    parent_span_id, kind ∈ {root, stage, subspan, generation},
    status ∈ {pending,running,done,failed,skipped,cancelled}, and
    JSONB input/output/metadata fields.

  * SpanTracker (replacing StageTracker) exposes OpenAttempt /
    BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan /
    LookupStage. Cross-process workers (image_multimodal) get the
    parent's attempt + span via payload + LookupStage so subspans
    attach correctly.

  * StageDependencies declares the DAG; FailSpan now cascades —
    descendants of the failed span and dependent stages are flipped
    to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear
    blast radius instead of orphan spinners.

  * Reparse now calls OpenAttempt up front so the timeline reflects
    "new attempt, all pending" instead of letting the previous run's
    status linger until the worker picks up the task.

  * Image_multimodal records each image as a generation subspan with
    its own success/failure on the parent attempt's multimodal stage.
    The finalize-on-last-attempt counter logic is preserved unchanged.

  * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns
    a tree shape with synthesized pending placeholders so the
    frontend always renders five timeline segments. ?attempt=N
    enables history navigation.
This commit is contained in:
wizardchen
2026-05-27 14:05:47 +08:00
committed by lyingbug
parent 04f56f9cda
commit c9941a6688
17 changed files with 1369 additions and 438 deletions
+155 -60
View File
@@ -34,7 +34,7 @@ type KnowledgeHandler struct {
kbShareService interfaces.KBShareService
agentShareService interfaces.AgentShareService
asynqClient interfaces.TaskEnqueuer
stageRepo repository.KnowledgeStageRepository
spanRepo repository.KnowledgeSpanRepository
}
// NewKnowledgeHandler creates a new knowledge handler instance
@@ -44,7 +44,7 @@ func NewKnowledgeHandler(
kbShareService interfaces.KBShareService,
agentShareService interfaces.AgentShareService,
asynqClient interfaces.TaskEnqueuer,
stageRepo repository.KnowledgeStageRepository,
spanRepo repository.KnowledgeSpanRepository,
) *KnowledgeHandler {
return &KnowledgeHandler{
kgService: kgService,
@@ -52,7 +52,7 @@ func NewKnowledgeHandler(
kbShareService: kbShareService,
agentShareService: agentShareService,
asynqClient: asynqClient,
stageRepo: stageRepo,
spanRepo: spanRepo,
}
}
@@ -549,20 +549,22 @@ func (h *KnowledgeHandler) GetKnowledge(c *gin.Context) {
})
}
// GetKnowledgeStages godoc
// @Summary 获取知识文档解析阶段进度
// @Description 返回该知识在解析流水线(docreader/chunking/embedding/multimodal/postprocess)的每段状态、耗时与错误码。前端用于渲染时间线。
// GetKnowledgeSpans godoc
// @Summary 获取知识文档解析的 Span 树(含历史尝试)
// @Description 返回该知识在解析流水线的 trace tree(root → stage → subspan):每段状态、耗时、input/output、错误码、langfuse_trace_id。支持 ?attempt=N 查看历史尝试;不传则返回最新尝试。前端用于渲染时间线 + 多模态/embedding 子节点 + 一键跳转 Langfuse。
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/knowledge/{id}/stages [get]
// @Param id path string true "知识ID"
// @Param attempt query int false "指定尝试号;省略=最新"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/knowledge/{id}/spans [get]
//
// Always returns AllProcessingStages segments — missing rows are
// Always returns the canonical 5-stage timeline; missing stage rows are
// synthesized as "pending" so the frontend timeline always renders five
// segments and doesn't have to know about backend persistence races.
func (h *KnowledgeHandler) GetKnowledgeStages(c *gin.Context) {
// segments. Subspans (multimodal.image[i], generation.*) ride along under
// each stage as children when present.
func (h *KnowledgeHandler) GetKnowledgeSpans(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
@@ -577,63 +579,60 @@ func (h *KnowledgeHandler) GetKnowledgeStages(c *gin.Context) {
return
}
rows := []types.KnowledgeProcessingStage{}
if h.stageRepo != nil {
rows, err = h.stageRepo.ListByKnowledge(ctx, knowledge.ID)
if err != nil {
// A repo error doesn't break the response — we still
// return the placeholder timeline so the UI renders.
logger.Warnf(ctx, "stages list failed for %s: %v", knowledge.ID, err)
rows = nil
// Pick attempt: explicit ?attempt=N wins; otherwise pull the
// latest attempt from the spans table. Lite-mode / fresh installs
// with zero rows fall through to attempt=0, in which case we
// return a placeholder tree (5 pending stages, no root, no
// children) so the UI still renders.
requestedAttempt := 0
if v := strings.TrimSpace(c.Query("attempt")); v != "" {
if n, perr := strconv.Atoi(v); perr == nil && n > 0 {
requestedAttempt = n
}
}
// Index by stage so synthesizing missing rows is O(N).
byStage := make(map[string]types.KnowledgeProcessingStage, len(rows))
for _, r := range rows {
byStage[r.Stage] = r
rows := []types.KnowledgeProcessingSpan{}
currentAttempt := 0
if h.spanRepo != nil {
if requestedAttempt == 0 {
latest, lerr := h.spanRepo.LatestAttempt(ctx, knowledge.ID)
if lerr != nil {
logger.Warnf(ctx, "spans LatestAttempt failed for %s: %v", knowledge.ID, lerr)
} else {
currentAttempt = latest
}
} else {
currentAttempt = requestedAttempt
}
if currentAttempt > 0 {
rows, err = h.spanRepo.ListByAttempt(ctx, knowledge.ID, currentAttempt)
if err != nil {
logger.Warnf(ctx, "spans ListByAttempt failed kid=%s attempt=%d: %v",
knowledge.ID, currentAttempt, err)
rows = nil
}
}
}
now := time.Now()
out := make([]types.KnowledgeProcessingStage, 0, len(types.AllProcessingStages))
currentStage := ""
var lastFailure *types.KnowledgeProcessingStage
for _, name := range types.AllProcessingStages {
if r, ok := byStage[name]; ok {
out = append(out, r)
if r.Status == types.ProcessingStageRunning && currentStage == "" {
currentStage = r.Stage
}
if r.Status == types.ProcessingStageFailed {
cp := r
lastFailure = &cp
}
continue
}
// Synthesize a pending placeholder. CreatedAt set to now so
// the JSON response is well-formed; clients should treat
// timestamps on placeholders as "not meaningful".
out = append(out, types.KnowledgeProcessingStage{
KnowledgeID: knowledge.ID,
Stage: name,
Status: types.ProcessingStagePending,
CreatedAt: now,
UpdatedAt: now,
})
}
// Build tree: index by SpanID, then attach to parents. Stages
// missing from the DB are synthesized as "pending" placeholders
// under a synthetic (or real, if present) root so the timeline
// always renders five segments.
tree, currentStageName, lastErr := buildSpanTree(knowledge.ID, currentAttempt, rows)
resp := gin.H{
"knowledge_id": knowledge.ID,
"parse_status": knowledge.ParseStatus,
"current_stage": currentStage,
"stages": out,
"knowledge_id": knowledge.ID,
"parse_status": knowledge.ParseStatus,
"current_attempt": currentAttempt,
"current_stage": currentStageName,
"trace": tree,
}
if lastFailure != nil {
if lastErr != nil {
resp["last_error"] = gin.H{
"stage": lastFailure.Stage,
"code": lastFailure.ErrorCode,
"message": lastFailure.ErrorMessage,
"finished_at": lastFailure.FinishedAt,
"stage": lastErr.Name,
"code": lastErr.ErrorCode,
"message": lastErr.ErrorMessage,
"finished_at": lastErr.FinishedAt,
}
}
c.JSON(http.StatusOK, gin.H{
@@ -642,6 +641,102 @@ func (h *KnowledgeHandler) GetKnowledgeStages(c *gin.Context) {
})
}
// buildSpanTree assembles a flat list of span rows into a parent-child
// tree rooted at the (knowledge, attempt)'s root span. Missing canonical
// stages are filled in with pending placeholders so the UI always renders
// the five timeline segments. Returns the root, the current_stage name
// (the running stage if any), and the most recent failed span if one
// exists.
func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProcessingSpan) (
root *types.SpanTreeNode, currentStage string, lastFailure *types.KnowledgeProcessingSpan,
) {
now := time.Now()
// Build node lookup, identify root.
nodes := make(map[string]*types.SpanTreeNode, len(rows))
var rootRow *types.KnowledgeProcessingSpan
stageRowByName := map[string]*types.KnowledgeProcessingSpan{}
for i := range rows {
r := rows[i]
nodes[r.SpanID] = &types.SpanTreeNode{KnowledgeProcessingSpan: r}
if r.Kind == types.SpanKindRoot && rootRow == nil {
cp := r
rootRow = &cp
}
if r.Kind == types.SpanKindStage {
cp := r
stageRowByName[r.Name] = &cp
}
if r.Status == types.SpanStatusRunning && r.Kind == types.SpanKindStage && currentStage == "" {
currentStage = r.Name
}
if r.Status == types.SpanStatusFailed {
cp := r
lastFailure = &cp
}
}
// Synthesize root if no rows came back so the API contract stays
// stable (frontend always expects a `trace` object).
if rootRow == nil {
root = &types.SpanTreeNode{KnowledgeProcessingSpan: types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: "",
Name: "knowledge_processing",
Kind: types.SpanKindRoot,
Status: types.SpanStatusPending,
CreatedAt: now,
UpdatedAt: now,
}}
} else {
root = nodes[rootRow.SpanID]
}
// Synthesize missing stage rows as "pending" children of root so
// the timeline always shows 5 segments.
for _, name := range types.AllStages {
if _, ok := stageRowByName[name]; ok {
continue
}
placeholder := types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
Name: name,
Kind: types.SpanKindStage,
Status: types.SpanStatusPending,
CreatedAt: now,
UpdatedAt: now,
}
nodes[name+"_placeholder_"+knowledgeID] = &types.SpanTreeNode{KnowledgeProcessingSpan: placeholder}
// Attach placeholder under root by giving it the root's
// span_id as parent — done after the link pass below.
}
// Link real children to their parents. Placeholders we just
// added have no real parent; we attach them to root in a
// second pass after the linking is done.
for _, n := range nodes {
if n == root {
continue
}
if n.ParentSpanID == "" {
// Real top-level row with no parent and not the root
// itself — attach to root so it doesn't dangle.
root.Children = append(root.Children, n)
continue
}
parent, ok := nodes[n.ParentSpanID]
if !ok {
// Unknown parent (orphan); attach to root.
root.Children = append(root.Children, n)
continue
}
parent.Children = append(parent.Children, n)
}
return root, currentStage, lastFailure
}
// ListKnowledge godoc
// @Summary 获取知识列表
// @Description 获取知识库下的知识列表,支持分页和筛选