fix(knowledge): address span-tree review feedback

- Collapse migrations 000052 (flat stages) + 000053 (span tree) into a
  single 000052_knowledge_processing_spans migration; the flat stages
  table never escaped this branch and the create-then-drop sequence had
  no value.
- BeginStage: detect an existing (kid, attempt, stage) row before
  inserting and reuse its span_id with reset state, so re-entry from
  asynq retries or adjacent code paths no longer produces duplicate
  timeline segments.
- FailSpan: when sibling-stage cascade flips a dependent stage to
  cancelled, also CancelDescendants on its subtree so already-running
  subspans (embedding.batch[i] etc.) cannot remain as orphan running
  rows under a cancelled parent.
- Dead-letter callback: replace two sequential UpdateKnowledgeColumn
  writes with a single UpdateKnowledgeColumns map update so we cannot
  end up with parse_status=failed and stale error_message (or vice
  versa) when one of the two writes fails.
- touchKnowledgeHeartbeat: skip subspan/generation transitions; only
  root and stage transitions poke knowledge.updated_at. Spans-table
  MAX(updated_at) already covers subspan progress for housekeeping, so
  this avoids 2*N+ UPDATE bursts on the same hot row when a multimodal
  stage fans out to many images.
- Add regression tests for the BeginStage idempotency contract and the
  cascade-into-subspans behaviour.
This commit is contained in:
wizardchen
2026-05-27 15:31:32 +08:00
committed by lyingbug
parent 414749681f
commit c82b098f44
10 changed files with 233 additions and 161 deletions
@@ -293,6 +293,21 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(
return err
}
// UpdateKnowledgeColumns writes multiple columns in a single UPDATE so callers
// that flip related fields together (parse_status + error_message after
// dead-letter, for example) cannot leave the row half-updated when the second
// write fails.
func (r *knowledgeRepository) UpdateKnowledgeColumns(
ctx context.Context,
id string,
values map[string]interface{},
) error {
if len(values) == 0 {
return nil
}
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
}
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base
func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
ctx context.Context,
@@ -132,13 +132,33 @@ func NewSpanTracker(repo repository.KnowledgeSpanRepository, db *gorm.DB) SpanTr
// one indexed UPDATE per transition (≤ a few dozen per knowledge), which
// is dwarfed by the work the stages themselves do.
//
// touchKnowledgeHeartbeat advances knowledge.updated_at to the current
// wall-clock so the housekeeping sweep treats this row as actively
// progressing. Called on root / stage span transitions only — subspan and
// generation transitions skip this side-channel because:
//
// - The spans table itself is updated on every transition, and the
// housekeeping sweep already reads MAX(spans.updated_at) per
// knowledge, so subspan progress is observable without poking the
// parent row.
// - A multimodal stage with N images would produce 2*N+ extra UPDATEs
// on the same hot row (Begin+End per image plus retries), which we
// observed contributing to row-level contention under bursty
// uploads.
//
// Best-effort. We deliberately do NOT bump status here: the parse_status
// column remains under the pipeline's control. Only the timestamp gets
// nudged, which is exactly what housekeeping reads.
func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID string) {
// nudged, which is exactly what housekeeping reads as the fallback.
func (t *spanTracker) touchKnowledgeHeartbeat(ctx context.Context, knowledgeID, kind string) {
if t.db == nil || knowledgeID == "" {
return
}
// Subspan / generation transitions are observable through the spans
// table directly; skip the parent-row UPDATE to avoid write
// amplification on fan-out workloads.
if kind != types.SpanKindRoot && kind != types.SpanKindStage {
return
}
if err := t.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("id = ?", knowledgeID).
Update("updated_at", time.Now()).Error; err != nil {
@@ -199,7 +219,7 @@ func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTrac
return nil, attempt, err
}
t.recordStart(rootID, now)
t.touchKnowledgeHeartbeat(ctx, knowledgeID)
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindRoot)
return &Span{
KnowledgeID: knowledgeID,
Attempt: attempt,
@@ -223,18 +243,30 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
if knowledgeID == "" || stage == "" {
return nil
}
// Find root span — we need its span_id as parent for stage rows.
// Find root span — we need its span_id as parent for stage rows. We
// also reuse this scan to detect an existing row for the same stage
// name in this attempt: re-entry (asynq retry, double-call from
// adjacent code paths) MUST NOT create a second row, otherwise the
// timeline shows two segments for the same stage and LookupStage
// becomes ambiguous.
rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt)
if err != nil {
logger.Warnf(ctx, "[SpanTracker] BeginStage list failed kid=%s attempt=%d: %v",
knowledgeID, attempt, err)
return nil
}
var rootID string
var (
rootID string
existing *types.KnowledgeProcessingSpan
)
for i := range rows {
if rows[i].Kind == types.SpanKindRoot {
rootID = rows[i].SpanID
break
r := rows[i]
if r.Kind == types.SpanKindRoot && rootID == "" {
rootID = r.SpanID
}
if r.Kind == types.SpanKindStage && r.Name == stage {
cp := r
existing = &cp
}
}
if rootID == "" {
@@ -245,6 +277,44 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
knowledgeID, attempt)
}
now := time.Now()
// Re-entry path: keep the original span_id so any subspan that
// already references it stays attached. Reset state to running and
// refresh started_at; clear terminal-only fields so the row reads
// cleanly as "running again". Output/error fields go through
// Upsert's DoUpdates list, which only writes the columns we set —
// any nil JSONMap / empty string explicitly clears the column.
if existing != nil {
row := &types.KnowledgeProcessingSpan{
KnowledgeID: existing.KnowledgeID,
Attempt: existing.Attempt,
SpanID: existing.SpanID,
ParentSpanID: existing.ParentSpanID,
Name: existing.Name,
Kind: existing.Kind,
Status: types.SpanStatusRunning,
Input: input,
Output: nil,
StartedAt: &now,
FinishedAt: nil,
DurationMs: 0,
}
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] BeginStage re-enter failed kid=%s stage=%s: %v",
knowledgeID, stage, err)
return nil
}
t.recordStart(existing.SpanID, now)
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage)
return &Span{
KnowledgeID: existing.KnowledgeID,
Attempt: existing.Attempt,
SpanID: existing.SpanID,
ParentSpanID: existing.ParentSpanID,
Name: existing.Name,
Kind: existing.Kind,
StartedAt: now,
}
}
id := newSpanID()
row := &types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
@@ -263,7 +333,7 @@ func (t *spanTracker) BeginStage(ctx context.Context, knowledgeID string, attemp
return nil
}
t.recordStart(id, now)
t.touchKnowledgeHeartbeat(ctx, knowledgeID)
t.touchKnowledgeHeartbeat(ctx, knowledgeID, types.SpanKindStage)
return &Span{
KnowledgeID: knowledgeID,
Attempt: attempt,
@@ -301,7 +371,7 @@ func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind
return nil
}
t.recordStart(id, now)
t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID)
t.touchKnowledgeHeartbeat(ctx, parent.KnowledgeID, kind)
return &Span{
KnowledgeID: parent.KnowledgeID,
Attempt: parent.Attempt,
@@ -335,7 +405,7 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] EndSpan failed span=%s: %v", span.SpanID, err)
}
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
}
func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {
@@ -387,7 +457,7 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error
if span.Kind == types.SpanKindStage {
t.cascadeDependentStages(ctx, span, reason)
}
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
}
func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) {
@@ -410,7 +480,7 @@ func (t *spanTracker) SkipSpan(ctx context.Context, span *Span, reason string) {
if err := t.repo.Upsert(ctx, row); err != nil {
logger.Warnf(ctx, "[SpanTracker] SkipSpan failed span=%s: %v", span.SpanID, err)
}
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID)
t.touchKnowledgeHeartbeat(ctx, span.KnowledgeID, span.Kind)
}
func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attempt int, stage string) *Span {
@@ -445,7 +515,11 @@ func (t *spanTracker) LookupStage(ctx context.Context, knowledgeID string, attem
// cascadeDependentStages flips downstream STAGE rows to "cancelled" using
// types.StageDependencies. Without this, a Chunking failure leaves
// Embedding / Multimodal as "pending" forever, even though they cannot
// possibly run.
// possibly run. After flipping a dependent stage we ALSO cascade-cancel
// any subspan/generation that already attached to it (e.g. an embedding
// batch that started before the chunking failure was observed) — without
// this second walk those subspans would remain in pending/running and
// surface as orphan spinners under a cancelled parent.
func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *Span, reason string) {
rows, err := t.repo.ListByAttempt(ctx, failedStage.KnowledgeID, failedStage.Attempt)
if err != nil {
@@ -473,6 +547,13 @@ func (t *spanTracker) cascadeDependentStages(ctx context.Context, failedStage *S
updated.FinishedAt = &now
if err := t.repo.Upsert(ctx, &updated); err != nil {
logger.Warnf(ctx, "[SpanTracker] cascade dependent stage %s: %v", row.Name, err)
continue
}
// Recurse into the cascaded stage's own subtree so any
// in-flight subspan/generation is cancelled too. The
// repo-level walk is iterative and cheap (small fan-out).
if _, err := t.repo.CancelDescendants(ctx, row.KnowledgeID, row.Attempt, row.SpanID, reason); err != nil {
logger.Warnf(ctx, "[SpanTracker] cascade descendants of dependent %s: %v", row.Name, err)
}
}
}
@@ -194,3 +194,80 @@ func TestSpanTracker_BeginSubSpan_HangsUnderParent(t *testing.T) {
assert.Equal(t, types.SpanKindGeneration, rows[0].Kind)
assert.Equal(t, parent.SpanID, rows[0].ParentSpanID, "subspan must reference parent stage's span_id")
}
// TestSpanTracker_BeginStage_ReentryIsIdempotent guarantees that a second
// BeginStage call for the same (kid, attempt, stage) reuses the existing
// span row instead of inserting a duplicate. Without this, an asynq retry
// or any code path that begins a stage twice would produce two timeline
// segments for the same stage, and LookupStage would resolve to whichever
// row sorts first — both regressions the original implementation had.
func TestSpanTracker_BeginStage_ReentryIsIdempotent(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
first := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 1})
require.NotNil(t, first)
// Simulate an intermediate End so the row is in a terminal state when
// the re-entry happens (mirrors retry-after-failure ordering).
tracker.FailSpan(ctx, first, "TEST", "first failure", errors.New("boom"))
second := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 2})
require.NotNil(t, second)
assert.Equal(t, first.SpanID, second.SpanID,
"re-entry must reuse the existing stage span_id")
type row struct {
SpanID, Status string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("span_id, status").
Where("knowledge_id = ? AND attempt = ? AND name = ?", "kid", attempt, types.StageDocReader).
Find(&rows).Error)
require.Len(t, rows, 1, "exactly one row per (knowledge, attempt, stage)")
assert.Equal(t, types.SpanStatusRunning, rows[0].Status,
"row must transition back to running after re-entry")
}
// TestSpanTracker_FailSpan_CascadesDependentSubspans verifies that when a
// chunking failure flips Embedding to "cancelled" (sibling cascade),
// embedding's already-running subspan (e.g. embedding.batch[0]) is ALSO
// cancelled. Without this, the UI rendered a cancelled stage with an
// orphan running batch hanging underneath.
func TestSpanTracker_FailSpan_CascadesDependentSubspans(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
chunking := tracker.BeginStage(ctx, "kid", attempt, types.StageChunking, nil)
embedding := tracker.BeginStage(ctx, "kid", attempt, types.StageEmbedding, nil)
require.NotNil(t, embedding)
// Subspan attached to the dependent (sibling) stage that's about to
// be cascade-cancelled.
batch := tracker.BeginSubSpan(ctx, embedding, "embedding.batch[0]", types.SpanKindGeneration, nil)
require.NotNil(t, batch)
tracker.FailSpan(ctx, chunking, "CHUNKING_FAILED", "synthetic", errors.New("boom"))
type row struct {
Name, Status string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("name, status").
Where("knowledge_id = ?", "kid").
Find(&rows).Error)
statusBy := map[string]string{}
for _, r := range rows {
statusBy[r.Name] = r.Status
}
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageEmbedding],
"dependent stage cascades to cancelled")
assert.Equal(t, types.SpanStatusCancelled, statusBy["embedding.batch[0]"],
"subspan under the cascaded stage must also be cancelled")
}
+8 -5
View File
@@ -293,13 +293,16 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDead
if len(errMsg) > 8192 {
errMsg = errMsg[:8192]
}
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err)
// Single UPDATE so we never end up with parse_status=failed but
// stale error_message (or vice versa) when the second write
// fails.
if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": errMsg,
}); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err)
return
}
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err)
}
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
}
}
+4
View File
@@ -210,6 +210,10 @@ type KnowledgeRepository interface {
// AminusB returns the difference set of A and B.
AminusB(ctx context.Context, Atenant uint64, A string, Btenant uint64, B string) ([]string, error)
UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error
// UpdateKnowledgeColumns updates multiple columns of a knowledge row in a single
// statement so callers that flip several related fields (e.g. parse_status +
// error_message) cannot leave the row in a half-updated state.
UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base.
CountKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error)
// CountKnowledgeByStatus counts the number of knowledge items with the specified parse status.
@@ -0,0 +1,9 @@
-- Migration: 000052_knowledge_processing_spans (rollback)
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_spans'; END $$;
DROP INDEX IF EXISTS idx_kpspan_parent;
DROP INDEX IF EXISTS idx_kpspan_status_started;
DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt;
DROP TABLE IF EXISTS knowledge_processing_spans;
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$;
@@ -1,50 +1,35 @@
-- Migration: 000053_knowledge_processing_spans
-- Replace the flat 5-stage tracking from migration 000052 with a tree-shaped
-- span model inspired by Langfuse traces.
-- Migration: 000052_knowledge_processing_spans
-- Per-(knowledge, attempt) span tree for the document parsing pipeline,
-- inspired by Langfuse's trace / span / generation hierarchy.
--
-- Why redesign?
-- Background: knowledge.parse_status is a 4-state field (pending /
-- processing / completed / failed). When users hit "stuck in processing",
-- they have no way to tell which stage is actually running — DocReader,
-- chunking, embedding, multimodal OCR/VLM, or the final post-process
-- handoff. Operators face the same problem.
--
-- The flat (knowledge_id, stage) model from 000052 had four shortcomings the
-- "stuck parsing" UX requires us to solve:
-- This table persists per-stage progress (and finer-grained subspans) so:
-- 1. The frontend can render a five-segment timeline showing where
-- each document is in the pipeline, with collapsible per-image /
-- per-batch subspans underneath.
-- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT,
-- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized
-- remediation text.
-- 3. Reparse history is preserved across attempts — operators can
-- navigate ?attempt=N to post-mortem "why did it fail twice?".
-- 4. Cascade-cancel rules use the parent_span_id tree to flip
-- dependent subspans/stages to "cancelled" when an upstream
-- span fails, so the UI shows a clear blast radius instead of
-- orphan spinners.
--
-- 1. Reparse semantics — a re-trigger of parsing on the same knowledge
-- reset the existing rows, erasing the previous attempt's history.
-- Operators investigating "why did it fail twice?" had no record of
-- attempt 1 once attempt 2 began.
--
-- 2. No DAG — Embedding and Multimodal both depend on Chunking but are
-- independent of each other. The flat model couldn't represent that,
-- so a Chunking failure couldn't auto-cascade "cancelled" to its
-- downstream stages and the UI had to guess.
--
-- 3. No subspans — a Multimodal stage produces N parallel image tasks;
-- Embedding produces M batches; PostProcess fans out into Summary /
-- Question / Wiki / Graph. Those finer-grained units have their own
-- success / failure / duration that operators want to see (and that
-- Langfuse generations already capture for the LLM-call subset).
--
-- 4. No input/output — duration alone hides the WHY of a slow run. With
-- input ({pages:48, images:5}) and output ({tokens:5840, dim:1024})
-- JSON, the timeline answers "how big was the work" without a log
-- dive.
--
-- The new schema mirrors Langfuse's trace/span/generation hierarchy:
-- Schema mirrors Langfuse's vocabulary:
-- * one ROOT span per (knowledge_id, attempt) acting as the trace
-- * STAGE spans (docreader/chunking/embedding/multimodal/postprocess)
-- are children of root
-- * SUBSPANs (multimodal.image[i], embedding.batch[i], postprocess.spawn.X)
-- hang off their stage. The kind="generation" subset corresponds 1:1 to
-- a Langfuse generation; metadata.langfuse_trace_id stitches them.
--
-- We keep the OLD 000052 table untouched on rollback so a `migrate down`
-- doesn't lose either schema. The forward path drops it because no UI
-- callers use it (the API was only added in the same branch and is being
-- replaced as part of this migration).
DO $$ BEGIN RAISE NOTICE '[Migration 000053] Replacing knowledge_processing_stages with knowledge_processing_spans'; END $$;
DROP INDEX IF EXISTS idx_kps_status_started;
DROP INDEX IF EXISTS idx_kps_knowledge;
DROP TABLE IF EXISTS knowledge_processing_stages;
DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_spans'; END $$;
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
id BIGSERIAL PRIMARY KEY,
@@ -76,7 +61,7 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_knowledge_attempt
ON knowledge_processing_spans (knowledge_id, attempt);
-- Operator query: "find spans stuck in running too long". Used by ad-hoc
-- diagnostics and a future housekeeping sweep.
-- diagnostics and the housekeeping sweep.
CREATE INDEX IF NOT EXISTS idx_kpspan_status_started
ON knowledge_processing_spans (status, started_at);
@@ -87,4 +72,4 @@ CREATE INDEX IF NOT EXISTS idx_kpspan_parent
ON knowledge_processing_spans (parent_span_id)
WHERE parent_span_id IS NOT NULL;
DO $$ BEGIN RAISE NOTICE '[Migration 000053] knowledge_processing_spans table ready'; END $$;
DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_spans table ready'; END $$;
@@ -1,8 +0,0 @@
-- Migration: 000052_knowledge_processing_stages (rollback)
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] Dropping table: knowledge_processing_stages'; END $$;
DROP INDEX IF EXISTS idx_kps_status_started;
DROP INDEX IF EXISTS idx_kps_knowledge;
DROP TABLE IF EXISTS knowledge_processing_stages;
DO $$ BEGIN RAISE NOTICE '[Migration 000052 rollback] complete'; END $$;
@@ -1,61 +0,0 @@
-- Migration: 000052_knowledge_processing_stages
-- Per-knowledge stage progress for the document parsing pipeline.
--
-- Background: knowledge.parse_status is a 4-state field (pending /
-- processing / completed / failed). When users hit "stuck in processing",
-- they have no way to tell which stage is actually running — DocReader,
-- chunking, embedding, multimodal OCR/VLM, or the final post-process
-- handoff. Operators face the same problem: a 500MB PDF stuck in
-- "processing" for 90 minutes could be legitimate OCR work or a frozen
-- docreader call, and the only signal is the absence of progress.
--
-- This table persists per-stage progress so:
-- 1. The frontend can render a five-segment timeline showing where
-- each document is in the pipeline (PR③ work).
-- 2. Failures carry a stable error_code (DOCREADER_TIMEOUT,
-- EMBEDDING_RATE_LIMIT, ...) the UI can map to localized
-- remediation text.
-- 3. Operators can SQL-query "all rows stuck on stage=embedding for
-- >30 min" without log-scraping.
--
-- Schema decisions:
-- - One row per (knowledge_id, stage). Retry attempts BUMP attempt
-- counter and reset status; we don't keep historical attempt rows
-- because the timeline UI only needs the latest state per stage.
-- - error_message holds the user-facing summary (≤1KB);
-- error_detail is the full stack/payload (≤8KB) gated behind
-- admin views in the UI.
-- - Sweeps that recover stuck knowledge will leave these rows
-- intact so operators can post-mortem the failure.
DO $$ BEGIN RAISE NOTICE '[Migration 000052] Creating table: knowledge_processing_stages'; END $$;
CREATE TABLE IF NOT EXISTS knowledge_processing_stages (
id BIGSERIAL PRIMARY KEY,
knowledge_id VARCHAR(64) NOT NULL,
stage VARCHAR(32) NOT NULL,
status VARCHAR(16) NOT NULL,
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
duration_ms BIGINT,
error_code VARCHAR(64),
error_message TEXT,
error_detail TEXT,
attempt INT NOT NULL DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage)
);
-- Primary read path: "give me every stage row for this knowledge_id".
-- The unique constraint already covers point lookups by (knowledge_id,
-- stage); this index lets us fetch all 5 rows for a knowledge with one
-- index range scan, which is the API surface the frontend timeline calls.
CREATE INDEX IF NOT EXISTS idx_kps_knowledge
ON knowledge_processing_stages (knowledge_id);
-- Operator query: "find stages stuck in running too long". Used by the
-- housekeeping cron in a future iteration and by ad-hoc SQL diagnostics.
CREATE INDEX IF NOT EXISTS idx_kps_status_started
ON knowledge_processing_stages (status, started_at);
DO $$ BEGIN RAISE NOTICE '[Migration 000052] knowledge_processing_stages table ready'; END $$;
@@ -1,33 +0,0 @@
-- Migration: 000053_knowledge_processing_spans (rollback)
-- Restores the 000052 flat-stage table so the system is operational on a
-- back-rev. We do NOT migrate data back — the spans table carries strict
-- supersets of the stages table's information, so a forward-and-back round
-- trip silently drops the new tree shape (acceptable; rollback is a rare
-- emergency lever).
DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] Dropping spans, restoring stages table'; END $$;
DROP INDEX IF EXISTS idx_kpspan_parent;
DROP INDEX IF EXISTS idx_kpspan_status_started;
DROP INDEX IF EXISTS idx_kpspan_knowledge_attempt;
DROP TABLE IF EXISTS knowledge_processing_spans;
CREATE TABLE IF NOT EXISTS knowledge_processing_stages (
id BIGSERIAL PRIMARY KEY,
knowledge_id VARCHAR(64) NOT NULL,
stage VARCHAR(32) NOT NULL,
status VARCHAR(16) NOT NULL,
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
duration_ms BIGINT,
error_code VARCHAR(64),
error_message TEXT,
error_detail TEXT,
attempt INT NOT NULL DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_kps_knowledge_stage UNIQUE (knowledge_id, stage)
);
CREATE INDEX IF NOT EXISTS idx_kps_knowledge ON knowledge_processing_stages (knowledge_id);
CREATE INDEX IF NOT EXISTS idx_kps_status_started ON knowledge_processing_stages (status, started_at);
DO $$ BEGIN RAISE NOTICE '[Migration 000053 rollback] complete'; END $$;