Files
Enterprise-Knowledge-Base/internal/application/repository/knowledge_span_repo.go
T
wizardchen c9941a6688 refactor(knowledge): replace flat stage table with langfuse-style span tree
Addresses review feedback that the PR② design had four shortcomings:

  1. The pipeline is a DAG, not a sequence — Embedding and Multimodal
     are independent of each other, both downstream of Chunking, both
     upstream of PostProcess. The flat (knowledge_id, stage) table
     couldn't represent that, so a Chunking failure left dependents
     stranded as "pending" forever instead of being marked as
     impossible-to-run.

  2. No history across attempts. A reparse erased the previous run's
     status before the new run started, leaving operators with no way
     to investigate "why did this fail twice?".

  3. Stages had only status + duration. Operators want to know how big
     the work was — pages parsed, chunks created, tokens embedded, VLM
     calls made — to distinguish "slow because the file is huge" from
     "slow because the docreader is wedged".

  4. Multimodal fans out N image tasks; Embedding fans out M batches;
     PostProcess fans out into Summary/Question/Wiki/Graph. Each unit
     is interesting on its own (Langfuse already captures this for
     LLM calls). The flat model couldn't express it.

The redesign mirrors Langfuse's trace/span/generation hierarchy:

  * Migration 000053 supersedes 000052: knowledge_processing_spans
    table with (knowledge_id, attempt, span_id) primary key, plus
    parent_span_id, kind ∈ {root, stage, subspan, generation},
    status ∈ {pending,running,done,failed,skipped,cancelled}, and
    JSONB input/output/metadata fields.

  * SpanTracker (replacing StageTracker) exposes OpenAttempt /
    BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan /
    LookupStage. Cross-process workers (image_multimodal) get the
    parent's attempt + span via payload + LookupStage so subspans
    attach correctly.

  * StageDependencies declares the DAG; FailSpan now cascades —
    descendants of the failed span and dependent stages are flipped
    to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear
    blast radius instead of orphan spinners.

  * Reparse now calls OpenAttempt up front so the timeline reflects
    "new attempt, all pending" instead of letting the previous run's
    status linger until the worker picks up the task.

  * Image_multimodal records each image as a generation subspan with
    its own success/failure on the parent attempt's multimodal stage.
    The finalize-on-last-attempt counter logic is preserved unchanged.

  * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns
    a tree shape with synthesized pending placeholders so the
    frontend always renders five timeline segments. ?attempt=N
    enables history navigation.
2026-05-28 15:14:45 +08:00

176 lines
6.3 KiB
Go

package repository
import (
"context"
"errors"
"github.com/Tencent/WeKnora/internal/types"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// KnowledgeSpanRepository persists the per-attempt span tree used by the
// processing pipeline. Operations are deliberately narrow:
//
// - Upsert covers Begin/End/Fail/Skip — every state transition routes
// through the same write so the row stays internally consistent.
// - NextAttempt allocates a new attempt for re-parses without touching
// historical rows. Old attempts stay queryable for post-mortem.
// - ListByAttempt is the only read path; the handler builds the tree
// in memory rather than recursing through the DB.
type KnowledgeSpanRepository interface {
Upsert(ctx context.Context, row *types.KnowledgeProcessingSpan) error
NextAttempt(ctx context.Context, knowledgeID string) (int, error)
LatestAttempt(ctx context.Context, knowledgeID string) (int, error)
ListByAttempt(ctx context.Context, knowledgeID string, attempt int) ([]types.KnowledgeProcessingSpan, error)
GetSpan(ctx context.Context, knowledgeID string, attempt int, spanID string) (*types.KnowledgeProcessingSpan, error)
// CancelDescendants marks every descendant of a parent span as
// "cancelled" with the given reason. Used by the tracker to
// cascade an upstream failure across a stage's downstream subtree
// without iterating in Go memory.
CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error)
}
type knowledgeSpanRepository struct {
db *gorm.DB
}
// NewKnowledgeSpanRepository wires the GORM-backed implementation.
func NewKnowledgeSpanRepository(db *gorm.DB) KnowledgeSpanRepository {
return &knowledgeSpanRepository{db: db}
}
func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.KnowledgeProcessingSpan) error {
if row == nil || row.KnowledgeID == "" || row.SpanID == "" {
return errors.New("knowledgeSpanRepository.Upsert: knowledge_id and span_id required")
}
if row.Attempt == 0 {
row.Attempt = 1
}
// We let GORM populate created_at/updated_at via the autoCreate /
// autoUpdate tags. ON CONFLICT updates only the fields that may
// transition between calls — name/kind/parent are immutable once
// set so we don't list them in DoUpdates (saves a few bytes per
// write, and any mismatch indicates a programming error).
return r.db.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "knowledge_id"},
{Name: "attempt"},
{Name: "span_id"},
},
DoUpdates: clause.AssignmentColumns([]string{
"status",
"input",
"output",
"metadata",
"error_code",
"error_message",
"error_detail",
"started_at",
"finished_at",
"duration_ms",
"updated_at",
}),
}).Create(row).Error
}
func (r *knowledgeSpanRepository) NextAttempt(ctx context.Context, knowledgeID string) (int, error) {
var max int
err := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("knowledge_id = ?", knowledgeID).
Select("COALESCE(MAX(attempt), 0)").
Row().Scan(&max)
if err != nil {
return 0, err
}
return max + 1, nil
}
func (r *knowledgeSpanRepository) LatestAttempt(ctx context.Context, knowledgeID string) (int, error) {
var max int
err := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("knowledge_id = ?", knowledgeID).
Select("COALESCE(MAX(attempt), 0)").
Row().Scan(&max)
return max, err
}
func (r *knowledgeSpanRepository) ListByAttempt(ctx context.Context, knowledgeID string, attempt int) ([]types.KnowledgeProcessingSpan, error) {
if knowledgeID == "" {
return nil, nil
}
var rows []types.KnowledgeProcessingSpan
q := r.db.WithContext(ctx).Where("knowledge_id = ?", knowledgeID)
if attempt > 0 {
q = q.Where("attempt = ?", attempt)
}
// id ASC keeps the natural insertion order — useful for stable
// rendering of fan-out subspans (e.g. multimodal.image[0..N] in
// the order they were enqueued).
err := q.Order("id ASC").Find(&rows).Error
return rows, err
}
func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID string, attempt int, spanID string) (*types.KnowledgeProcessingSpan, error) {
var row types.KnowledgeProcessingSpan
err := r.db.WithContext(ctx).
Where("knowledge_id = ? AND attempt = ? AND span_id = ?", knowledgeID, attempt, spanID).
Take(&row).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &row, nil
}
// CancelDescendants performs an iterative SQL walk: each level we update
// every row whose parent_span_id is in the previous level's span_id set,
// flipping pending/running rows to cancelled. We bail when a level adds
// zero rows (fixed point reached) or after a generous depth bound.
//
// Postgres-specific WITH RECURSIVE would be denser but harder to test on
// the SQLite Lite backend. The iterative path stays portable.
func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) {
frontier := []string{parentSpanID}
var totalAffected int64
for depth := 0; depth < 16 && len(frontier) > 0; depth++ {
var nextFrontier []string
// Find children of every span currently on the frontier
// that are still in a non-terminal state — terminal rows
// (done/failed/skipped/cancelled) are left as-is so the UI
// can still see their original outcome.
var children []types.KnowledgeProcessingSpan
err := r.db.WithContext(ctx).
Where("knowledge_id = ? AND attempt = ? AND parent_span_id IN ? AND status IN ?",
knowledgeID, attempt, frontier,
[]string{types.SpanStatusPending, types.SpanStatusRunning}).
Find(&children).Error
if err != nil {
return totalAffected, err
}
if len(children) == 0 {
break
}
ids := make([]string, 0, len(children))
for _, c := range children {
ids = append(ids, c.SpanID)
nextFrontier = append(nextFrontier, c.SpanID)
}
res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("knowledge_id = ? AND attempt = ? AND span_id IN ?", knowledgeID, attempt, ids).
Updates(map[string]any{
"status": types.SpanStatusCancelled,
"error_code": "UPSTREAM_FAILED",
"error_message": reason,
})
if res.Error != nil {
return totalAffected, res.Error
}
totalAffected += res.RowsAffected
frontier = nextFrontier
}
return totalAffected, nil
}