fix(knowledge): address span-tree review feedback
- Collapse migrations 000052 (flat stages) + 000053 (span tree) into a single 000052_knowledge_processing_spans migration; the flat stages table never escaped this branch and the create-then-drop sequence had no value. - BeginStage: detect an existing (kid, attempt, stage) row before inserting and reuse its span_id with reset state, so re-entry from asynq retries or adjacent code paths no longer produces duplicate timeline segments. - FailSpan: when sibling-stage cascade flips a dependent stage to cancelled, also CancelDescendants on its subtree so already-running subspans (embedding.batch[i] etc.) cannot remain as orphan running rows under a cancelled parent. - Dead-letter callback: replace two sequential UpdateKnowledgeColumn writes with a single UpdateKnowledgeColumns map update so we cannot end up with parse_status=failed and stale error_message (or vice versa) when one of the two writes fails. - touchKnowledgeHeartbeat: skip subspan/generation transitions; only root and stage transitions poke knowledge.updated_at. Spans-table MAX(updated_at) already covers subspan progress for housekeeping, so this avoids 2*N+ UPDATE bursts on the same hot row when a multimodal stage fans out to many images. - Add regression tests for the BeginStage idempotency contract and the cascade-into-subspans behaviour.
This commit is contained in:
@@ -293,6 +293,21 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateKnowledgeColumns writes multiple columns in a single UPDATE so callers
|
||||
// that flip related fields together (parse_status + error_message after
|
||||
// dead-letter, for example) cannot leave the row half-updated when the second
|
||||
// write fails.
|
||||
func (r *knowledgeRepository) UpdateKnowledgeColumns(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
values map[string]interface{},
|
||||
) error {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
|
||||
}
|
||||
|
||||
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base
|
||||
func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
|
||||
ctx context.Context,
|
||||
|
||||
@@ -132,13 +132,33 @@ func NewSpanTracker(repo repository.KnowledgeSpanRepository, db *gorm.DB) SpanTr
|
||||
// one indexed UPDATE per transition (≤ a few dozen per knowledge), which
|
||||
// is dwarfed by the work the stages themselves do.
|
||||
//
|
||||
// touchKnowledgeHeartbeat advances knowledge.updated_at to the current
|
||||
// wall-clock so the housekeeping sweep treats this row as actively
|
||||
// progressing. Called on root / stage span transitions only — subspan and
|
||||
// generation transitions skip this side-channel because:
|
||||
//
|
||||
// - The spans table itself is updated on every transition, and the
|
||||
// housekeeping sweep already reads MAX(spans.updated_at) per
|
||||
// knowledge, so subspan progress is observable without poking the
|
||||
// parent row.
|
||||
// - A multimodal stage with N images would produce 2*N+ extra UPDATEs
|
||||
// on the same hot row (Begin+End per image plus retries), which we
|
||||
// observed contributing to row-level contention under bursty
|
||||
// uploads.
|
||||
//
|
||||
// Best-effort. We deliberately do NOT bump status here: the parse_status
|
||||
// column remains under the pipeline's control. Only the timestamp gets
|
||||
// nudged, which is exactly what housekeeping reads.
|
||||
func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID string) {
|
||||
// nudged, which is exactly what housekeeping reads as the fallback.
|
||||
func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID, kind string) {
|
||||
if t.db == nil || knowledgeID == "" {
|
||||
return
|
||||
}
|
||||
// Subspan / generation transitions are observable through the spans
|
||||
// table directly; skip the parent-row UPDATE to avoid write
|
||||
// amplification on fan-out workloads.
|
||||
if kind != types.SpanKindRoot && kind != types.SpanKindStage {
|
||||
return
|
||||
}
|
||||
if err := t.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("id = ?", knowledgeID).
|
||||
Update("updated_at", time.Now()).Error; err != nil {
|
||||
@@ -199,7 +219,7 @@ func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTrac
|
||||
return nil, attempt, err
|
||||
}
|
||||
t.recordStart(rootID, now)
|
||||
t.touchKnowledgeHeartbeat(ctx, knowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindRoot)
|
||||
return &Span{
|
||||
KnowledgeID: knowledgeID,
|
||||
Attempt: attempt,
|
||||
@@ -223,18 +243,30 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
|
||||
if knowledgeID == "" || stage == "" {
|
||||
return nil
|
||||
}
|
||||
// Find root span — we need its span_id as parent for stage rows.
|
||||
// Find root span — we need its span_id as parent for stage rows. We
|
||||
// also reuse this scan to detect an existing row for the same stage
|
||||
// name in this attempt: re-entry (asynq retry, double-call from
|
||||
// adjacent code paths) MUST NOT create a second row, otherwise the
|
||||
// timeline shows two segments for the same stage and LookupStage
|
||||
// becomes ambiguous.
|
||||
rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] BeginStage list failed kid=%s attempt=%d: %v",
|
||||
knowledgeID, attempt, err)
|
||||
return nil
|
||||
}
|
||||
var rootID string
|
||||
var (
|
||||
rootID string
|
||||
existing *types.KnowledgeProcessingSpan
|
||||
)
|
||||
for i := range rows {
|
||||
if rows[i].Kind == types.SpanKindRoot {
|
||||
rootID = rows[i].SpanID
|
||||
break
|
||||
r := rows[i]
|
||||
if r.Kind == types.SpanKindRoot && rootID == "" {
|
||||
rootID = r.SpanID
|
||||
}
|
||||
if r.Kind == types.SpanKindStage && r.Name == stage {
|
||||
cp := r
|
||||
existing = &cp
|
||||
}
|
||||
}
|
||||
if rootID == "" {
|
||||
@@ -245,6 +277,44 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
|
||||
knowledgeID, attempt)
|
||||
}
|
||||
now := time.Now()
|
||||
// Re-entry path: keep the original span_id so any subspan that
|
||||
// already references it stays attached. Reset state to running and
|
||||
// refresh started_at; clear terminal-only fields so the row reads
|
||||
// cleanly as "running again". Output/error fields go through
|
||||
// Upsert's DoUpdates list, which only writes the columns we set —
|
||||
// any nil JSONMap / empty string explicitly clears the column.
|
||||
if existing != nil {
|
||||
row := &types.KnowledgeProcessingSpan{
|
||||
KnowledgeID: existing.KnowledgeID,
|
||||
Attempt: existing.Attempt,
|
||||
SpanID: existing.SpanID,
|
||||
ParentSpanID: existing.ParentSpanID,
|
||||
Name: existing.Name,
|
||||
Kind: existing.Kind,
|
||||
Status: types.SpanStatusRunning,
|
||||
Input: input,
|
||||
Output: nil,
|
||||
StartedAt: &now,
|
||||
FinishedAt: nil,
|
||||
DurationMs: 0,
|
||||
}
|
||||
if err := t.repo.Upsert(ctx, row); err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] BeginStage re-enter failed kid=%s stage=%s: %v",
|
||||
knowledgeID, stage, err)
|
||||
return nil
|
||||
}
|
||||
t.recordStart(existing.SpanID, now)
|
||||
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage)
|
||||
return &Span{
|
||||
KnowledgeID: existing.KnowledgeID,
|
||||
Attempt: existing.Attempt,
|
||||
SpanID: existing.SpanID,
|
||||
ParentSpanID: existing.ParentSpanID,
|
||||
Name: existing.Name,
|
||||
Kind: existing.Kind,
|
||||
StartedAt: now,
|
||||
}
|
||||
}
|
||||
id := newSpanID()
|
||||
row := &types.KnowledgeProcessingSpan{
|
||||
KnowledgeID: knowledgeID,
|
||||
@@ -263,7 +333,7 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
|
||||
return nil
|
||||
}
|
||||
t.recordStart(id, now)
|
||||
t.touchKnowledgeHeartbeat(ctx, knowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage)
|
||||
return &Span{
|
||||
KnowledgeID: knowledgeID,
|
||||
Attempt: attempt,
|
||||
@@ -301,7 +371,7 @@ func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind
|
||||
return nil
|
||||
}
|
||||
t.recordStart(id, now)
|
||||
t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID, kind)
|
||||
return &Span{
|
||||
KnowledgeID: parent.KnowledgeID,
|
||||
Attempt: parent.Attempt,
|
||||
@@ -335,7 +405,7 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON
|
||||
if err := t.repo.Upsert(ctx, row); err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] EndSpan failed span=%s: %v", span.SpanID, err)
|
||||
}
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
|
||||
}
|
||||
|
||||
func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {
|
||||
@@ -387,7 +457,7 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error
|
||||
if span.Kind == types.SpanKindStage {
|
||||
t.cascadeDependentStages(ctx, span, reason)
|
||||
}
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
|
||||
}
|
||||
|
||||
func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) {
|
||||
@@ -410,7 +480,7 @@ func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) {
|
||||
if err := t.repo.Upsert(ctx, row); err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] SkipSpan failed span=%s: %v", span.SpanID, err)
|
||||
}
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
|
||||
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
|
||||
}
|
||||
|
||||
func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span {
|
||||
@@ -445,7 +515,11 @@ func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attem
|
||||
// cascadeDependentStages flips downstream STAGE rows to "cancelled" using
|
||||
// types.StageDependencies. Without this, a Chunking failure leaves
|
||||
// Embedding / Multimodal as "pending" forever, even though they cannot
|
||||
// possibly run.
|
||||
// possibly run. After flipping a dependent stage we ALSO cascade-cancel
|
||||
// any subspan/generation that already attached to it (e.g. an embedding
|
||||
// batch that started before the chunking failure was observed) — without
|
||||
// this second walk those subspans would remain in pending/running and
|
||||
// surface as orphan spinners under a cancelled parent.
|
||||
func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *Span, reason string) {
|
||||
rows, err := t.repo.ListByAttempt(ctx, failedStage.KnowledgeID, failedStage.Attempt)
|
||||
if err != nil {
|
||||
@@ -473,6 +547,13 @@ func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *S
|
||||
updated.FinishedAt = &now
|
||||
if err := t.repo.Upsert(ctx, &updated); err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] cascade dependent stage %s: %v", row.Name, err)
|
||||
continue
|
||||
}
|
||||
// Recurse into the cascaded stage's own subtree so any
|
||||
// in-flight subspan/generation is cancelled too. The
|
||||
// repo-level walk is iterative and cheap (small fan-out).
|
||||
if _, err := t.repo.CancelDescendants(ctx, row.KnowledgeID, row.Attempt, row.SpanID, reason); err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] cascade descendants of dependent %s: %v", row.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,3 +194,80 @@ func TestSpanTracker_BeginSubSpan_HangsUnderParent(t *testing.T) {
|
||||
assert.Equal(t, types.SpanKindGeneration, rows[0].Kind)
|
||||
assert.Equal(t, parent.SpanID, rows[0].ParentSpanID, "subspan must reference parent stage's span_id")
|
||||
}
|
||||
|
||||
// TestSpanTracker_BeginStage_ReentryIsIdempotent guarantees that a second
|
||||
// BeginStage call for the same (kid, attempt, stage) reuses the existing
|
||||
// span row instead of inserting a duplicate. Without this, an asynq retry
|
||||
// or any code path that begins a stage twice would produce two timeline
|
||||
// segments for the same stage, and LookupStage would resolve to whichever
|
||||
// row sorts first — both regressions the original implementation had.
|
||||
func TestSpanTracker_BeginStage_ReentryIsIdempotent(t *testing.T) {
|
||||
tracker, db := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
first := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 1})
|
||||
require.NotNil(t, first)
|
||||
// Simulate an intermediate End so the row is in a terminal state when
|
||||
// the re-entry happens (mirrors retry-after-failure ordering).
|
||||
tracker.FailSpan(ctx, first, "TEST", "first failure", errors.New("boom"))
|
||||
|
||||
second := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 2})
|
||||
require.NotNil(t, second)
|
||||
assert.Equal(t, first.SpanID, second.SpanID,
|
||||
"re-entry must reuse the existing stage span_id")
|
||||
|
||||
type row struct {
|
||||
SpanID, Status string
|
||||
}
|
||||
var rows []row
|
||||
require.NoError(t, db.Table("knowledge_processing_spans").
|
||||
Select("span_id, status").
|
||||
Where("knowledge_id = ? AND attempt = ? AND name = ?", "kid", attempt, types.StageDocReader).
|
||||
Find(&rows).Error)
|
||||
require.Len(t, rows, 1, "exactly one row per (knowledge, attempt, stage)")
|
||||
assert.Equal(t, types.SpanStatusRunning, rows[0].Status,
|
||||
"row must transition back to running after re-entry")
|
||||
}
|
||||
|
||||
// TestSpanTracker_FailSpan_CascadesDependentSubspans verifies that when a
|
||||
// chunking failure flips Embedding to "cancelled" (sibling cascade),
|
||||
// embedding's already-running subspan (e.g. embedding.batch[0]) is ALSO
|
||||
// cancelled. Without this, the UI rendered a cancelled stage with an
|
||||
// orphan running batch hanging underneath.
|
||||
func TestSpanTracker_FailSpan_CascadesDependentSubspans(t *testing.T) {
|
||||
tracker, db := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
chunking := tracker.BeginStage(ctx, "kid", attempt, types.StageChunking, nil)
|
||||
embedding := tracker.BeginStage(ctx, "kid", attempt, types.StageEmbedding, nil)
|
||||
require.NotNil(t, embedding)
|
||||
// Subspan attached to the dependent (sibling) stage that's about to
|
||||
// be cascade-cancelled.
|
||||
batch := tracker.BeginSubSpan(ctx, embedding, "embedding.batch[0]", types.SpanKindGeneration, nil)
|
||||
require.NotNil(t, batch)
|
||||
|
||||
tracker.FailSpan(ctx, chunking, "CHUNKING_FAILED", "synthetic", errors.New("boom"))
|
||||
|
||||
type row struct {
|
||||
Name, Status string
|
||||
}
|
||||
var rows []row
|
||||
require.NoError(t, db.Table("knowledge_processing_spans").
|
||||
Select("name, status").
|
||||
Where("knowledge_id = ?", "kid").
|
||||
Find(&rows).Error)
|
||||
statusBy := map[string]string{}
|
||||
for _, r := range rows {
|
||||
statusBy[r.Name] = r.Status
|
||||
}
|
||||
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageEmbedding],
|
||||
"dependent stage cascades to cancelled")
|
||||
assert.Equal(t, types.SpanStatusCancelled, statusBy["embedding.batch[0]"],
|
||||
"subspan under the cascaded stage must also be cancelled")
|
||||
}
|
||||
|
||||
@@ -293,13 +293,16 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDead
|
||||
if len(errMsg) > 8192 {
|
||||
errMsg = errMsg[:8192]
|
||||
}
|
||||
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err)
|
||||
// Single UPDATE so we never end up with parse_status=failed but
|
||||
// stale error_message (or vice versa) when the second write
|
||||
// fails.
|
||||
if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": errMsg,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err)
|
||||
}
|
||||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,6 +210,10 @@ type KnowledgeRepository interface {
|
||||
// AminusB returns the difference set of A and B.
|
||||
AminusB(ctx context.Context, Atenant uint64, A string, Btenant uint64, B string) ([]string, error)
|
||||
UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error
|
||||
// UpdateKnowledgeColumns updates multiple columns of a knowledge row in a single
|
||||
// statement so callers that flip several related fields (e.g. parse_status +
|
||||
// error_message) cannot leave the row in a half-updated state.
|
||||
UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error
|
||||
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base.
|
||||
CountKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error)
|
||||
// CountKnowledgeByStatus counts the number of knowledge items with the specified parse status.
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration: 000052_knowledge_processing_spans (rollback)
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_spans'; END $$;
|
||||
|
||||
DROP INDEX IF EXISTS idx_kpspan_parent;
|
||||
DROP INDEX IF EXISTS idx_kpspan_status_started;
|
||||
DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt;
|
||||
DROP TABLE IF EXISTS knowledge_processing_spans;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$;
|
||||
+25
-40
@@ -1,50 +1,35 @@
|
||||
-- Migration: 000053_knowledge_processing_spans
|
||||
-- Replace the flat 5-stage tracking from migration 000052 with a tree-shaped
|
||||
-- span model inspired by Langfuse traces.
|
||||
-- Migration: 000052_knowledge_processing_spans
|
||||
-- Per-(knowledge, attempt) span tree for the document parsing pipeline,
|
||||
-- inspired by Langfuse's trace / span / generation hierarchy.
|
||||
--
|
||||
-- Why redesign?
|
||||
-- Background: knowledge.parse_status is a 4-state field (pending /
|
||||
-- processing / completed / failed). When users hit "stuck in processing",
|
||||
-- they have no way to tell which stage is actually running — DocReader,
|
||||
-- chunking, embedding, multimodal OCR/VLM, or the final post-process
|
||||
-- handoff. Operators face the same problem.
|
||||
--
|
||||
-- The flat (knowledge_id, stage) model from 000052 had four shortcomings the
|
||||
-- "stuck parsing" UX requires us to solve:
|
||||
-- This table persists per-stage progress (and finer-grained subspans) so:
|
||||
-- 1. The frontend can render a five-segment timeline showing where
|
||||
-- each document is in the pipeline, with collapsible per-image /
|
||||
-- per-batch subspans underneath.
|
||||
-- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT,
|
||||
-- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized
|
||||
-- remediation text.
|
||||
-- 3. Reparse history is preserved across attempts — operators can
|
||||
-- navigate ?attempt=N to post-mortem "why did it fail twice?".
|
||||
-- 4. Cascade-cancel rules use the parent_span_id tree to flip
|
||||
-- dependent subspans/stages to "cancelled" when an upstream
|
||||
-- span fails, so the UI shows a clear blast radius instead of
|
||||
-- orphan spinners.
|
||||
--
|
||||
-- 1. Reparse semantics — a re-trigger of parsing on the same knowledge
|
||||
-- reset the existing rows, erasing the previous attempt's history.
|
||||
-- Operators investigating "why did it fail twice?" had no record of
|
||||
-- attempt 1 once attempt 2 began.
|
||||
--
|
||||
-- 2. No DAG — Embedding and Multimodal both depend on Chunking but are
|
||||
-- independent of each other. The flat model couldn't represent that,
|
||||
-- so a Chunking failure couldn't auto-cascade "cancelled" to its
|
||||
-- downstream stages and the UI had to guess.
|
||||
--
|
||||
-- 3. No subspans — a Multimodal stage produces N parallel image tasks;
|
||||
-- Embedding produces M batches; PostProcess fans out into Summary /
|
||||
-- Question / Wiki / Graph. Those finer-grained units have their own
|
||||
-- success / failure / duration that operators want to see (and that
|
||||
-- Langfuse generations already capture for the LLM-call subset).
|
||||
--
|
||||
-- 4. No input/output — duration alone hides the WHY of a slow run. With
|
||||
-- input ({pages:48, images:5}) and output ({tokens:5840, dim:1024})
|
||||
-- JSON, the timeline answers "how big was the work" without a log
|
||||
-- dive.
|
||||
--
|
||||
-- The new schema mirrors Langfuse's trace/span/generation hierarchy:
|
||||
-- Schema mirrors Langfuse's vocabulary:
|
||||
-- * one ROOT span per (knowledge_id, attempt) acting as the trace
|
||||
-- * STAGE spans (docreader/chunking/embedding/multimodal/postprocess)
|
||||
-- are children of root
|
||||
-- * SUBSPANs (multimodal.image[i], embedding.batch[i], postprocess.spawn.X)
|
||||
-- hang off their stage. The kind="generation" subset corresponds 1:1 to
|
||||
-- a Langfuse generation; metadata.langfuse_trace_id stitches them.
|
||||
--
|
||||
-- We keep the OLD 000052 table untouched on rollback so a `migrate down`
|
||||
-- doesn't lose either schema. The forward path drops it because no UI
|
||||
-- callers use it (the API was only added in the same branch and is being
|
||||
-- replaced as part of this migration).
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000053] Replacing knowledge_processing_stages with knowledge_processing_spans'; END $$;
|
||||
|
||||
DROP INDEX IF EXISTS idx_kps_status_started;
|
||||
DROP INDEX IF EXISTS idx_kps_knowledge;
|
||||
DROP TABLE IF EXISTS knowledge_processing_stages;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_spans'; END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
@@ -76,7 +61,7 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_knowledge_attempt
|
||||
ON knowledge_processing_spans (knowledge_id, attempt);
|
||||
|
||||
-- Operator query: "find spans stuck in running too long". Used by ad-hoc
|
||||
-- diagnostics and a future housekeeping sweep.
|
||||
-- diagnostics and the housekeeping sweep.
|
||||
CREATE INDEX IF NOT EXISTS idx_kpspan_status_started
|
||||
ON knowledge_processing_spans (status, started_at);
|
||||
|
||||
@@ -87,4 +72,4 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_parent
|
||||
ON knowledge_processing_spans (parent_span_id)
|
||||
WHERE parent_span_id IS NOT NULL;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000053] knowledge_processing_spans table ready'; END $$;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_spans table ready'; END $$;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- Migration: 000052_knowledge_processing_stages (rollback)
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_stages'; END $$;
|
||||
|
||||
DROP INDEX IF EXISTS idx_kps_status_started;
|
||||
DROP INDEX IF EXISTS idx_kps_knowledge;
|
||||
DROP TABLE IF EXISTS knowledge_processing_stages;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$;
|
||||
@@ -1,61 +0,0 @@
|
||||
-- Migration: 000052_knowledge_processing_stages
|
||||
-- Per-knowledge stage progress for the document parsing pipeline.
|
||||
--
|
||||
-- Background: knowledge.parse_status is a 4-state field (pending /
|
||||
-- processing / completed / failed). When users hit "stuck in processing",
|
||||
-- they have no way to tell which stage is actually running — DocReader,
|
||||
-- chunking, embedding, multimodal OCR/VLM, or the final post-process
|
||||
-- handoff. Operators face the same problem: a 500MB PDF stuck in
|
||||
-- "processing" for 90 minutes could be legitimate OCR work or a frozen
|
||||
-- docreader call, and the only signal is the absence of progress.
|
||||
--
|
||||
-- This table persists per-stage progress so:
|
||||
-- 1. The frontend can render a five-segment timeline showing where
|
||||
-- each document is in the pipeline (PR③ work).
|
||||
-- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT,
|
||||
-- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized
|
||||
-- remediation text.
|
||||
-- 3. Operators can SQL-query "all rows stuck on stage=embedding for
|
||||
-- >30 min" without log-scraping.
|
||||
--
|
||||
-- Schema decisions:
|
||||
-- - One row per (knowledge_id, stage). Retry attempts BUMP attempt
|
||||
-- counter and reset status; we don't keep historical attempt rows
|
||||
-- because the timeline UI only needs the latest state per stage.
|
||||
-- - error_message holds the user-facing summary (≤1KB);
|
||||
-- error_detail is the full stack/payload (≤8KB) gated behind
|
||||
-- admin views in the UI.
|
||||
-- - Sweeps that recover stuck knowledge will leave these rows
|
||||
-- intact so operators can post-mortem the failure.
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_stages'; END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_processing_stages (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
knowledge_id VARCHAR(64) NOT NULL,
|
||||
stage VARCHAR(32) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
finished_at TIMESTAMP WITH TIME ZONE,
|
||||
duration_ms BIGINT,
|
||||
error_code VARCHAR(64),
|
||||
error_message TEXT,
|
||||
error_detail TEXT,
|
||||
attempt INT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage)
|
||||
);
|
||||
|
||||
-- Primary read path: "give me every stage row for this knowledge_id".
|
||||
-- The unique constraint already covers point lookups by (knowledge_id,
|
||||
-- stage); this index lets us fetch all 5 rows for a knowledge with one
|
||||
-- index range scan, which is the API surface the frontend timeline calls.
|
||||
CREATE INDEX IF NOT EXISTS idx_kps_knowledge
|
||||
ON knowledge_processing_stages (knowledge_id);
|
||||
|
||||
-- Operator query: "find stages stuck in running too long". Used by the
|
||||
-- housekeeping cron in a future iteration and by ad-hoc SQL diagnostics.
|
||||
CREATE INDEX IF NOT EXISTS idx_kps_status_started
|
||||
ON knowledge_processing_stages (status, started_at);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_stages table ready'; END $$;
|
||||
@@ -1,33 +0,0 @@
|
||||
-- Migration: 000053_knowledge_processing_spans (rollback)
|
||||
-- Restores the 000052 flat-stage table so the system is operational on a
|
||||
-- back-rev. We do NOT migrate data back — the spans table carries strict
|
||||
-- supersets of the stages table's information, so a forward-and-back round
|
||||
-- trip silently drops the new tree shape (acceptable; rollback is a rare
|
||||
-- emergency lever).
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] Dropping spans, restoring stages table'; END $$;
|
||||
|
||||
DROP INDEX IF EXISTS idx_kpspan_parent;
|
||||
DROP INDEX IF EXISTS idx_kpspan_status_started;
|
||||
DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt;
|
||||
DROP TABLE IF EXISTS knowledge_processing_spans;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_processing_stages (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
knowledge_id VARCHAR(64) NOT NULL,
|
||||
stage VARCHAR(32) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
finished_at TIMESTAMP WITH TIME ZONE,
|
||||
duration_ms BIGINT,
|
||||
error_code VARCHAR(64),
|
||||
error_message TEXT,
|
||||
error_detail TEXT,
|
||||
attempt INT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_kps_knowledge ON knowledge_processing_stages (knowledge_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kps_status_started ON knowledge_processing_stages (status, started_at);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] complete'; END $$;
|
||||
Reference in New Issue
Block a user