From c82b098f44e584022118bf233b803cb96d9abbe6 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 27 May 2026 15:31:32 +0800 Subject: [PATCH] fix(knowledge): address span-tree review feedback - Collapse migrations 000052 (flat stages) + 000053 (span tree) into a single 000052_knowledge_processing_spans migration; the flat stages table never escaped this branch and the create-then-drop sequence had no value. - BeginStage: detect an existing (kid, attempt, stage) row before inserting and reuse its span_id with reset state, so re-entry from asynq retries or adjacent code paths no longer produces duplicate timeline segments. - FailSpan: when sibling-stage cascade flips a dependent stage to cancelled, also CancelDescendants on its subtree so already-running subspans (embedding.batch[i] etc.) cannot remain as orphan running rows under a cancelled parent. - Dead-letter callback: replace two sequential UpdateKnowledgeColumn writes with a single UpdateKnowledgeColumns map update so we cannot end up with parse_status=failed and stale error_message (or vice versa) when one of the two writes fails. - touchKnowledgeHeartbeat: skip subspan/generation transitions; only root and stage transitions poke knowledge.updated_at. Spans-table MAX(updated_at) already covers subspan progress for housekeeping, so this avoids 2*N+ UPDATE bursts on the same hot row when a multimodal stage fans out to many images. - Add regression tests for the BeginStage idempotency contract and the cascade-into-subspans behaviour. --- internal/application/repository/knowledge.go | 15 +++ .../service/knowledge_span_tracker.go | 109 +++++++++++++++--- .../service/knowledge_span_tracker_test.go | 77 +++++++++++++ internal/router/task.go | 13 ++- internal/types/interfaces/knowledge.go | 4 + ...000052_knowledge_processing_spans.down.sql | 9 ++ ... 000052_knowledge_processing_spans.up.sql} | 65 ++++------- ...00052_knowledge_processing_stages.down.sql | 8 -- .../000052_knowledge_processing_stages.up.sql | 61 ---------- ...000053_knowledge_processing_spans.down.sql | 33 ------ 10 files changed, 233 insertions(+), 161 deletions(-) create mode 100644 migrations/versioned/000052_knowledge_processing_spans.down.sql rename migrations/versioned/{000053_knowledge_processing_spans.up.sql => 000052_knowledge_processing_spans.up.sql} (52%) delete mode 100644 migrations/versioned/000052_knowledge_processing_stages.down.sql delete mode 100644 migrations/versioned/000052_knowledge_processing_stages.up.sql delete mode 100644 migrations/versioned/000053_knowledge_processing_spans.down.sql diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 54db9c1f..4dbe0bc8 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -293,6 +293,21 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn( return err } +// UpdateKnowledgeColumns writes multiple columns in a single UPDATE so callers +// that flip related fields together (parse_status + error_message after +// dead-letter, for example) cannot leave the row half-updated when the second +// write fails. +func (r *knowledgeRepository) UpdateKnowledgeColumns( + ctx context.Context, + id string, + values map[string]interface{}, +) error { + if len(values) == 0 { + return nil + } + return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error +} + // CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID( ctx context.Context, diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go index d1e4fabe..eb05e25b 100644 --- a/internal/application/service/knowledge_span_tracker.go +++ b/internal/application/service/knowledge_span_tracker.go @@ -132,13 +132,33 @@ func NewSpanTracker(repo repository.KnowledgeSpanRepository, db *gorm.DB) SpanTr // one indexed UPDATE per transition (≤ a few dozen per knowledge), which // is dwarfed by the work the stages themselves do. // +// touchKnowledgeHeartbeat advances knowledge.updated_at to the current +// wall-clock so the housekeeping sweep treats this row as actively +// progressing. Called on root / stage span transitions only — subspan and +// generation transitions skip this side-channel because: +// +// - The spans table itself is updated on every transition, and the +// housekeeping sweep already reads MAX(spans.updated_at) per +// knowledge, so subspan progress is observable without poking the +// parent row. +// - A multimodal stage with N images would produce 2*N+ extra UPDATEs +// on the same hot row (Begin+End per image plus retries), which we +// observed contributing to row-level contention under bursty +// uploads. +// // Best-effort. We deliberately do NOT bump status here: the parse_status // column remains under the pipeline's control. Only the timestamp gets -// nudged, which is exactly what housekeeping reads. -func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID string) { +// nudged, which is exactly what housekeeping reads as the fallback. +func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID, kind string) { if t.db == nil || knowledgeID == "" { return } + // Subspan / generation transitions are observable through the spans + // table directly; skip the parent-row UPDATE to avoid write + // amplification on fan-out workloads. + if kind != types.SpanKindRoot && kind != types.SpanKindStage { + return + } if err := t.db.WithContext(ctx).Model(&types.Knowledge{}). Where("id = ?", knowledgeID). Update("updated_at", time.Now()).Error; err != nil { @@ -199,7 +219,7 @@ func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTrac return nil, attempt, err } t.recordStart(rootID, now) - t.touchKnowledgeHeartbeat(ctx, knowledgeID) + t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindRoot) return &Span{ KnowledgeID: knowledgeID, Attempt: attempt, @@ -223,18 +243,30 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp if knowledgeID == "" || stage == "" { return nil } - // Find root span — we need its span_id as parent for stage rows. + // Find root span — we need its span_id as parent for stage rows. We + // also reuse this scan to detect an existing row for the same stage + // name in this attempt: re-entry (asynq retry, double-call from + // adjacent code paths) MUST NOT create a second row, otherwise the + // timeline shows two segments for the same stage and LookupStage + // becomes ambiguous. rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt) if err != nil { logger.Warnf(ctx, "[SpanTracker] BeginStage list failed kid=%s attempt=%d: %v", knowledgeID, attempt, err) return nil } - var rootID string + var ( + rootID string + existing *types.KnowledgeProcessingSpan + ) for i := range rows { - if rows[i].Kind == types.SpanKindRoot { - rootID = rows[i].SpanID - break + r := rows[i] + if r.Kind == types.SpanKindRoot && rootID == "" { + rootID = r.SpanID + } + if r.Kind == types.SpanKindStage && r.Name == stage { + cp := r + existing = &cp } } if rootID == "" { @@ -245,6 +277,44 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp knowledgeID, attempt) } now := time.Now() + // Re-entry path: keep the original span_id so any subspan that + // already references it stays attached. Reset state to running and + // refresh started_at; clear terminal-only fields so the row reads + // cleanly as "running again". Output/error fields go through + // Upsert's DoUpdates list, which only writes the columns we set — + // any nil JSONMap / empty string explicitly clears the column. + if existing != nil { + row := &types.KnowledgeProcessingSpan{ + KnowledgeID: existing.KnowledgeID, + Attempt: existing.Attempt, + SpanID: existing.SpanID, + ParentSpanID: existing.ParentSpanID, + Name: existing.Name, + Kind: existing.Kind, + Status: types.SpanStatusRunning, + Input: input, + Output: nil, + StartedAt: &now, + FinishedAt: nil, + DurationMs: 0, + } + if err := t.repo.Upsert(ctx, row); err != nil { + logger.Warnf(ctx, "[SpanTracker] BeginStage re-enter failed kid=%s stage=%s: %v", + knowledgeID, stage, err) + return nil + } + t.recordStart(existing.SpanID, now) + t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage) + return &Span{ + KnowledgeID: existing.KnowledgeID, + Attempt: existing.Attempt, + SpanID: existing.SpanID, + ParentSpanID: existing.ParentSpanID, + Name: existing.Name, + Kind: existing.Kind, + StartedAt: now, + } + } id := newSpanID() row := &types.KnowledgeProcessingSpan{ KnowledgeID: knowledgeID, @@ -263,7 +333,7 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp return nil } t.recordStart(id, now) - t.touchKnowledgeHeartbeat(ctx, knowledgeID) + t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage) return &Span{ KnowledgeID: knowledgeID, Attempt: attempt, @@ -301,7 +371,7 @@ func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind return nil } t.recordStart(id, now) - t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID) + t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID, kind) return &Span{ KnowledgeID: parent.KnowledgeID, Attempt: parent.Attempt, @@ -335,7 +405,7 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON if err := t.repo.Upsert(ctx, row); err != nil { logger.Warnf(ctx, "[SpanTracker] EndSpan failed span=%s: %v", span.SpanID, err) } - t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID) + t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind) } func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) { @@ -387,7 +457,7 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error if span.Kind == types.SpanKindStage { t.cascadeDependentStages(ctx, span, reason) } - t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID) + t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind) } func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) { @@ -410,7 +480,7 @@ func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) { if err := t.repo.Upsert(ctx, row); err != nil { logger.Warnf(ctx, "[SpanTracker] SkipSpan failed span=%s: %v", span.SpanID, err) } - t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID) + t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind) } func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span { @@ -445,7 +515,11 @@ func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attem // cascadeDependentStages flips downstream STAGE rows to "cancelled" using // types.StageDependencies. Without this, a Chunking failure leaves // Embedding / Multimodal as "pending" forever, even though they cannot -// possibly run. +// possibly run. After flipping a dependent stage we ALSO cascade-cancel +// any subspan/generation that already attached to it (e.g. an embedding +// batch that started before the chunking failure was observed) — without +// this second walk those subspans would remain in pending/running and +// surface as orphan spinners under a cancelled parent. func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *Span, reason string) { rows, err := t.repo.ListByAttempt(ctx, failedStage.KnowledgeID, failedStage.Attempt) if err != nil { @@ -473,6 +547,13 @@ func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *S updated.FinishedAt = &now if err := t.repo.Upsert(ctx, &updated); err != nil { logger.Warnf(ctx, "[SpanTracker] cascade dependent stage %s: %v", row.Name, err) + continue + } + // Recurse into the cascaded stage's own subtree so any + // in-flight subspan/generation is cancelled too. The + // repo-level walk is iterative and cheap (small fan-out). + if _, err := t.repo.CancelDescendants(ctx, row.KnowledgeID, row.Attempt, row.SpanID, reason); err != nil { + logger.Warnf(ctx, "[SpanTracker] cascade descendants of dependent %s: %v", row.Name, err) } } } diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go index 2667ad85..f1ef3cd4 100644 --- a/internal/application/service/knowledge_span_tracker_test.go +++ b/internal/application/service/knowledge_span_tracker_test.go @@ -194,3 +194,80 @@ func TestSpanTracker_BeginSubSpan_HangsUnderParent(t *testing.T) { assert.Equal(t, types.SpanKindGeneration, rows[0].Kind) assert.Equal(t, parent.SpanID, rows[0].ParentSpanID, "subspan must reference parent stage's span_id") } + +// TestSpanTracker_BeginStage_ReentryIsIdempotent guarantees that a second +// BeginStage call for the same (kid, attempt, stage) reuses the existing +// span row instead of inserting a duplicate. Without this, an asynq retry +// or any code path that begins a stage twice would produce two timeline +// segments for the same stage, and LookupStage would resolve to whichever +// row sorts first — both regressions the original implementation had. +func TestSpanTracker_BeginStage_ReentryIsIdempotent(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := context.Background() + + _, attempt, err := tracker.OpenAttempt(ctx, "kid", "") + require.NoError(t, err) + + first := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 1}) + require.NotNil(t, first) + // Simulate an intermediate End so the row is in a terminal state when + // the re-entry happens (mirrors retry-after-failure ordering). + tracker.FailSpan(ctx, first, "TEST", "first failure", errors.New("boom")) + + second := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 2}) + require.NotNil(t, second) + assert.Equal(t, first.SpanID, second.SpanID, + "re-entry must reuse the existing stage span_id") + + type row struct { + SpanID, Status string + } + var rows []row + require.NoError(t, db.Table("knowledge_processing_spans"). + Select("span_id, status"). + Where("knowledge_id = ? AND attempt = ? AND name = ?", "kid", attempt, types.StageDocReader). + Find(&rows).Error) + require.Len(t, rows, 1, "exactly one row per (knowledge, attempt, stage)") + assert.Equal(t, types.SpanStatusRunning, rows[0].Status, + "row must transition back to running after re-entry") +} + +// TestSpanTracker_FailSpan_CascadesDependentSubspans verifies that when a +// chunking failure flips Embedding to "cancelled" (sibling cascade), +// embedding's already-running subspan (e.g. embedding.batch[0]) is ALSO +// cancelled. Without this, the UI rendered a cancelled stage with an +// orphan running batch hanging underneath. +func TestSpanTracker_FailSpan_CascadesDependentSubspans(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := context.Background() + + _, attempt, err := tracker.OpenAttempt(ctx, "kid", "") + require.NoError(t, err) + + chunking := tracker.BeginStage(ctx, "kid", attempt, types.StageChunking, nil) + embedding := tracker.BeginStage(ctx, "kid", attempt, types.StageEmbedding, nil) + require.NotNil(t, embedding) + // Subspan attached to the dependent (sibling) stage that's about to + // be cascade-cancelled. + batch := tracker.BeginSubSpan(ctx, embedding, "embedding.batch[0]", types.SpanKindGeneration, nil) + require.NotNil(t, batch) + + tracker.FailSpan(ctx, chunking, "CHUNKING_FAILED", "synthetic", errors.New("boom")) + + type row struct { + Name, Status string + } + var rows []row + require.NoError(t, db.Table("knowledge_processing_spans"). + Select("name, status"). + Where("knowledge_id = ?", "kid"). + Find(&rows).Error) + statusBy := map[string]string{} + for _, r := range rows { + statusBy[r.Name] = r.Status + } + assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageEmbedding], + "dependent stage cascades to cancelled") + assert.Equal(t, types.SpanStatusCancelled, statusBy["embedding.batch[0]"], + "subspan under the cascaded stage must also be cancelled") +} diff --git a/internal/router/task.go b/internal/router/task.go index 72a3cb1d..ce1fb1eb 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -293,13 +293,16 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDead if len(errMsg) > 8192 { errMsg = errMsg[:8192] } - if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil { - logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err) + // Single UPDATE so we never end up with parse_status=failed but + // stale error_message (or vice versa) when the second write + // fails. + if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": errMsg, + }); err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err) return } - if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil { - logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err) - } logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type()) } } diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index 3a2b2f56..9a1c416b 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -210,6 +210,10 @@ type KnowledgeRepository interface { // AminusB returns the difference set of A and B. AminusB(ctx context.Context, Atenant uint64, A string, Btenant uint64, B string) ([]string, error) UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error + // UpdateKnowledgeColumns updates multiple columns of a knowledge row in a single + // statement so callers that flip several related fields (e.g. parse_status + + // error_message) cannot leave the row in a half-updated state. + UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error // CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base. CountKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error) // CountKnowledgeByStatus counts the number of knowledge items with the specified parse status. diff --git a/migrations/versioned/000052_knowledge_processing_spans.down.sql b/migrations/versioned/000052_knowledge_processing_spans.down.sql new file mode 100644 index 00000000..9163ddbc --- /dev/null +++ b/migrations/versioned/000052_knowledge_processing_spans.down.sql @@ -0,0 +1,9 @@ +-- Migration: 000052_knowledge_processing_spans (rollback) +DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_spans'; END $$; + +DROP INDEX IF EXISTS idx_kpspan_parent; +DROP INDEX IF EXISTS idx_kpspan_status_started; +DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt; +DROP TABLE IF EXISTS knowledge_processing_spans; + +DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$; diff --git a/migrations/versioned/000053_knowledge_processing_spans.up.sql b/migrations/versioned/000052_knowledge_processing_spans.up.sql similarity index 52% rename from migrations/versioned/000053_knowledge_processing_spans.up.sql rename to migrations/versioned/000052_knowledge_processing_spans.up.sql index 52550ebf..532b715b 100644 --- a/migrations/versioned/000053_knowledge_processing_spans.up.sql +++ b/migrations/versioned/000052_knowledge_processing_spans.up.sql @@ -1,50 +1,35 @@ --- Migration: 000053_knowledge_processing_spans --- Replace the flat 5-stage tracking from migration 000052 with a tree-shaped --- span model inspired by Langfuse traces. +-- Migration: 000052_knowledge_processing_spans +-- Per-(knowledge, attempt) span tree for the document parsing pipeline, +-- inspired by Langfuse's trace / span / generation hierarchy. -- --- Why redesign? +-- Background: knowledge.parse_status is a 4-state field (pending / +-- processing / completed / failed). When users hit "stuck in processing", +-- they have no way to tell which stage is actually running — DocReader, +-- chunking, embedding, multimodal OCR/VLM, or the final post-process +-- handoff. Operators face the same problem. -- --- The flat (knowledge_id, stage) model from 000052 had four shortcomings the --- "stuck parsing" UX requires us to solve: +-- This table persists per-stage progress (and finer-grained subspans) so: +-- 1. The frontend can render a five-segment timeline showing where +-- each document is in the pipeline, with collapsible per-image / +-- per-batch subspans underneath. +-- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT, +-- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized +-- remediation text. +-- 3. Reparse history is preserved across attempts — operators can +-- navigate ?attempt=N to post-mortem "why did it fail twice?". +-- 4. Cascade-cancel rules use the parent_span_id tree to flip +-- dependent subspans/stages to "cancelled" when an upstream +-- span fails, so the UI shows a clear blast radius instead of +-- orphan spinners. -- --- 1. Reparse semantics — a re-trigger of parsing on the same knowledge --- reset the existing rows, erasing the previous attempt's history. --- Operators investigating "why did it fail twice?" had no record of --- attempt 1 once attempt 2 began. --- --- 2. No DAG — Embedding and Multimodal both depend on Chunking but are --- independent of each other. The flat model couldn't represent that, --- so a Chunking failure couldn't auto-cascade "cancelled" to its --- downstream stages and the UI had to guess. --- --- 3. No subspans — a Multimodal stage produces N parallel image tasks; --- Embedding produces M batches; PostProcess fans out into Summary / --- Question / Wiki / Graph. Those finer-grained units have their own --- success / failure / duration that operators want to see (and that --- Langfuse generations already capture for the LLM-call subset). --- --- 4. No input/output — duration alone hides the WHY of a slow run. With --- input ({pages:48, images:5}) and output ({tokens:5840, dim:1024}) --- JSON, the timeline answers "how big was the work" without a log --- dive. --- --- The new schema mirrors Langfuse's trace/span/generation hierarchy: +-- Schema mirrors Langfuse's vocabulary: -- * one ROOT span per (knowledge_id, attempt) acting as the trace -- * STAGE spans (docreader/chunking/embedding/multimodal/postprocess) -- are children of root -- * SUBSPANs (multimodal.image[i], embedding.batch[i], postprocess.spawn.X) -- hang off their stage. The kind="generation" subset corresponds 1:1 to -- a Langfuse generation; metadata.langfuse_trace_id stitches them. --- --- We keep the OLD 000052 table untouched on rollback so a `migrate down` --- doesn't lose either schema. The forward path drops it because no UI --- callers use it (the API was only added in the same branch and is being --- replaced as part of this migration). -DO $$ BEGIN RAISE NOTICE '[Migration 000053] Replacing knowledge_processing_stages with knowledge_processing_spans'; END $$; - -DROP INDEX IF EXISTS idx_kps_status_started; -DROP INDEX IF EXISTS idx_kps_knowledge; -DROP TABLE IF EXISTS knowledge_processing_stages; +DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_spans'; END $$; CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( id BIGSERIAL PRIMARY KEY, @@ -76,7 +61,7 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_knowledge_attempt ON knowledge_processing_spans (knowledge_id, attempt); -- Operator query: "find spans stuck in running too long". Used by ad-hoc --- diagnostics and a future housekeeping sweep. +-- diagnostics and the housekeeping sweep. CREATE INDEX IF NOT EXISTS idx_kpspan_status_started ON knowledge_processing_spans (status, started_at); @@ -87,4 +72,4 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_parent ON knowledge_processing_spans (parent_span_id) WHERE parent_span_id IS NOT NULL; -DO $$ BEGIN RAISE NOTICE '[Migration 000053] knowledge_processing_spans table ready'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_spans table ready'; END $$; diff --git a/migrations/versioned/000052_knowledge_processing_stages.down.sql b/migrations/versioned/000052_knowledge_processing_stages.down.sql deleted file mode 100644 index 9d1cf0e2..00000000 --- a/migrations/versioned/000052_knowledge_processing_stages.down.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Migration: 000052_knowledge_processing_stages (rollback) -DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_stages'; END $$; - -DROP INDEX IF EXISTS idx_kps_status_started; -DROP INDEX IF EXISTS idx_kps_knowledge; -DROP TABLE IF EXISTS knowledge_processing_stages; - -DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$; diff --git a/migrations/versioned/000052_knowledge_processing_stages.up.sql b/migrations/versioned/000052_knowledge_processing_stages.up.sql deleted file mode 100644 index 4e2ce4d8..00000000 --- a/migrations/versioned/000052_knowledge_processing_stages.up.sql +++ /dev/null @@ -1,61 +0,0 @@ --- Migration: 000052_knowledge_processing_stages --- Per-knowledge stage progress for the document parsing pipeline. --- --- Background: knowledge.parse_status is a 4-state field (pending / --- processing / completed / failed). When users hit "stuck in processing", --- they have no way to tell which stage is actually running — DocReader, --- chunking, embedding, multimodal OCR/VLM, or the final post-process --- handoff. Operators face the same problem: a 500MB PDF stuck in --- "processing" for 90 minutes could be legitimate OCR work or a frozen --- docreader call, and the only signal is the absence of progress. --- --- This table persists per-stage progress so: --- 1. The frontend can render a five-segment timeline showing where --- each document is in the pipeline (PR③ work). --- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT, --- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized --- remediation text. --- 3. Operators can SQL-query "all rows stuck on stage=embedding for --- >30 min" without log-scraping. --- --- Schema decisions: --- - One row per (knowledge_id, stage). Retry attempts BUMP attempt --- counter and reset status; we don't keep historical attempt rows --- because the timeline UI only needs the latest state per stage. --- - error_message holds the user-facing summary (≤1KB); --- error_detail is the full stack/payload (≤8KB) gated behind --- admin views in the UI. --- - Sweeps that recover stuck knowledge will leave these rows --- intact so operators can post-mortem the failure. -DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_stages'; END $$; - -CREATE TABLE IF NOT EXISTS knowledge_processing_stages ( - id BIGSERIAL PRIMARY KEY, - knowledge_id VARCHAR(64) NOT NULL, - stage VARCHAR(32) NOT NULL, - status VARCHAR(16) NOT NULL, - started_at TIMESTAMP WITH TIME ZONE, - finished_at TIMESTAMP WITH TIME ZONE, - duration_ms BIGINT, - error_code VARCHAR(64), - error_message TEXT, - error_detail TEXT, - attempt INT NOT NULL DEFAULT 1, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage) -); - --- Primary read path: "give me every stage row for this knowledge_id". --- The unique constraint already covers point lookups by (knowledge_id, --- stage); this index lets us fetch all 5 rows for a knowledge with one --- index range scan, which is the API surface the frontend timeline calls. -CREATE INDEX IF NOT EXISTS idx_kps_knowledge - ON knowledge_processing_stages (knowledge_id); - --- Operator query: "find stages stuck in running too long". Used by the --- housekeeping cron in a future iteration and by ad-hoc SQL diagnostics. -CREATE INDEX IF NOT EXISTS idx_kps_status_started - ON knowledge_processing_stages (status, started_at); - -DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_stages table ready'; END $$; diff --git a/migrations/versioned/000053_knowledge_processing_spans.down.sql b/migrations/versioned/000053_knowledge_processing_spans.down.sql deleted file mode 100644 index c3239cb1..00000000 --- a/migrations/versioned/000053_knowledge_processing_spans.down.sql +++ /dev/null @@ -1,33 +0,0 @@ --- Migration: 000053_knowledge_processing_spans (rollback) --- Restores the 000052 flat-stage table so the system is operational on a --- back-rev. We do NOT migrate data back — the spans table carries strict --- supersets of the stages table's information, so a forward-and-back round --- trip silently drops the new tree shape (acceptable; rollback is a rare --- emergency lever). -DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] Dropping spans, restoring stages table'; END $$; - -DROP INDEX IF EXISTS idx_kpspan_parent; -DROP INDEX IF EXISTS idx_kpspan_status_started; -DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt; -DROP TABLE IF EXISTS knowledge_processing_spans; - -CREATE TABLE IF NOT EXISTS knowledge_processing_stages ( - id BIGSERIAL PRIMARY KEY, - knowledge_id VARCHAR(64) NOT NULL, - stage VARCHAR(32) NOT NULL, - status VARCHAR(16) NOT NULL, - started_at TIMESTAMP WITH TIME ZONE, - finished_at TIMESTAMP WITH TIME ZONE, - duration_ms BIGINT, - error_code VARCHAR(64), - error_message TEXT, - error_detail TEXT, - attempt INT NOT NULL DEFAULT 1, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage) -); -CREATE INDEX IF NOT EXISTS idx_kps_knowledge ON knowledge_processing_stages (knowledge_id); -CREATE INDEX IF NOT EXISTS idx_kps_status_started ON knowledge_processing_stages (status, started_at); - -DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] complete'; END $$;