Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_span_tracker_test.go
T
wizardchen c82b098f44 fix(knowledge): address span-tree review feedback
- Collapse migrations 000052 (flat stages) + 000053 (span tree) into a
  single 000052_knowledge_processing_spans migration; the flat stages
  table never escaped this branch and the create-then-drop sequence had
  no value.
- BeginStage: detect an existing (kid, attempt, stage) row before
  inserting and reuse its span_id with reset state, so re-entry from
  asynq retries or adjacent code paths no longer produces duplicate
  timeline segments.
- FailSpan: when sibling-stage cascade flips a dependent stage to
  cancelled, also CancelDescendants on its subtree so already-running
  subspans (embedding.batch[i] etc.) cannot remain as orphan running
  rows under a cancelled parent.
- Dead-letter callback: replace two sequential UpdateKnowledgeColumn
  writes with a single UpdateKnowledgeColumns map update so we cannot
  end up with parse_status=failed and stale error_message (or vice
  versa) when one of the two writes fails.
- touchKnowledgeHeartbeat: skip subspan/generation transitions; only
  root and stage transitions poke knowledge.updated_at. Spans-table
  MAX(updated_at) already covers subspan progress for housekeeping, so
  this avoids 2*N+ UPDATE bursts on the same hot row when a multimodal
  stage fans out to many images.
- Add regression tests for the BeginStage idempotency contract and the
  cascade-into-subspans behaviour.
2026-05-28 15:14:45 +08:00

274 lines
11 KiB
Go

package service
import (
"context"
"errors"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// span tracker tests use a real GORM-backed repo against an in-memory
// SQLite DB. We do this instead of a stub repo because the cascade /
// LookupStage logic interacts non-trivially with the persistence layer
// (UPSERT, MAX(attempt), parent IN ...) — a stub would let regressions
// in those queries slip through.
//
// We DDL-define the spans table inline (same content as the repo test's
// spansTestDDL — kept duplicated rather than exported because a service
// test crossing into the repository test file's identifiers couples the
// two too tightly).
const spanTrackerTestDDL = `
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
knowledge_id VARCHAR(64) NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
span_id VARCHAR(64) NOT NULL,
parent_span_id VARCHAR(64),
name VARCHAR(64) NOT NULL,
kind VARCHAR(16) NOT NULL,
status VARCHAR(16) NOT NULL,
input TEXT,
output TEXT,
metadata TEXT,
error_code VARCHAR(64),
error_message TEXT,
error_detail TEXT,
started_at DATETIME,
finished_at DATETIME,
duration_ms BIGINT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE (knowledge_id, attempt, span_id)
);
`
func setupSpanTrackerTest(t *testing.T) (SpanTracker, *gorm.DB) {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(spanTrackerTestDDL).Error)
// Pass nil for the heartbeat db: these tests don't exercise
// heartbeat side-effects (those are covered in the housekeeping
// suite). Keeping it nil also avoids needing the knowledges
// table just to validate span behaviour.
repo := repository.NewKnowledgeSpanRepository(db)
return NewSpanTracker(repo, nil), db
}
// TestSpanTracker_OpenAttempt_AllocatesFreshNumbers covers the contract
// that drives reparse history: each OpenAttempt must hand out a strictly
// increasing attempt number per knowledge, and previous attempts'
// rows must remain queryable (via a separate ?attempt=N navigation).
func TestSpanTracker_OpenAttempt_AllocatesFreshNumbers(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
root1, n1, err := tracker.OpenAttempt(ctx, "kid", "trace-1")
require.NoError(t, err)
require.NotNil(t, root1)
assert.Equal(t, 1, n1)
root2, n2, err := tracker.OpenAttempt(ctx, "kid", "trace-2")
require.NoError(t, err)
require.NotNil(t, root2)
assert.Equal(t, 2, n2, "second OpenAttempt must allocate attempt 2")
assert.NotEqual(t, root1.SpanID, root2.SpanID, "each attempt has its own root span ID")
// Both roots must persist — a reparse must NOT erase the previous
// attempt's history.
var count int64
require.NoError(t, db.Table("knowledge_processing_spans").
Where("knowledge_id = ? AND kind = 'root'", "kid").
Count(&count).Error)
assert.Equal(t, int64(2), count, "previous attempt's root must remain after reparse")
}
// TestSpanTracker_FailSpan_CascadesDownstream verifies that failing a
// stage flips its dependent stages to "cancelled" so the UI shows a
// clear blast radius instead of orphan spinners. This is the central
// guarantee of the DAG model — without it, a Chunking failure leaves
// Embedding/Multimodal/PostProcess as pending forever.
func TestSpanTracker_FailSpan_CascadesDownstream(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
require.Equal(t, 1, attempt)
// Begin every stage so the cascade has something to cancel.
docreader := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, nil)
tracker.EndSpan(ctx, docreader, nil)
chunking := tracker.BeginStage(ctx, "kid", attempt, types.StageChunking, nil)
embedding := tracker.BeginStage(ctx, "kid", attempt, types.StageEmbedding, nil)
multimodal := tracker.BeginStage(ctx, "kid", attempt, types.StageMultimodal, nil)
postprocess := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, nil)
// Fail Chunking. Embedding/Multimodal/PostProcess must cascade.
tracker.FailSpan(ctx, chunking, "CHUNKING_FAILED", "synthetic", errors.New("boom"))
statusBy := map[string]string{}
type row struct {
Name, Status string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("name, status").
Where("knowledge_id = ? AND attempt = ?", "kid", attempt).
Find(&rows).Error)
for _, r := range rows {
statusBy[r.Name] = r.Status
}
assert.Equal(t, types.SpanStatusDone, statusBy[types.StageDocReader], "upstream stage stays done")
assert.Equal(t, types.SpanStatusFailed, statusBy[types.StageChunking], "the failed stage itself stays failed")
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageEmbedding], "direct dependent must cascade")
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageMultimodal], "sibling dependent must cascade")
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StagePostProcess], "transitive dependent must cascade")
// Quiet the unused-variable check: embedding / multimodal /
// postprocess pointers were used to seed the table; their state
// is now in statusBy. Linter still wants them "consumed".
_ = embedding
_ = multimodal
_ = postprocess
}
// TestSpanTracker_LookupStage_FindsAcrossProcesses simulates the
// cross-process bridge an asynq worker uses: the upstream pipeline
// creates the multimodal stage span, then a separate worker process
// must locate it by (kid, attempt, name) to attach its image subspan.
func TestSpanTracker_LookupStage_FindsAcrossProcesses(t *testing.T) {
tracker, _ := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
mm := tracker.BeginStage(ctx, "kid", attempt, types.StageMultimodal, nil)
require.NotNil(t, mm)
// Pretend we're a different process — the in-memory `starts`
// cache is the same map here, but the cross-process semantics
// don't depend on it; LookupStage hits the DB.
found := tracker.LookupStage(ctx, "kid", attempt, types.StageMultimodal)
require.NotNil(t, found)
assert.Equal(t, mm.SpanID, found.SpanID, "LookupStage must return the same span row")
// A different stage must not be confused with multimodal.
other := tracker.LookupStage(ctx, "kid", attempt, types.StageEmbedding)
assert.Nil(t, other, "LookupStage(missing) must return nil")
}
// TestSpanTracker_BeginSubSpan_HangsUnderParent confirms multimodal /
// embedding fan-out subspans reference the parent stage's span_id —
// the structural invariant the buildSpanTree handler walks.
func TestSpanTracker_BeginSubSpan_HangsUnderParent(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
parent := tracker.BeginStage(ctx, "kid", attempt, types.StageMultimodal, nil)
require.NotNil(t, parent)
sub := tracker.BeginSubSpan(ctx, parent, "multimodal.image[0]", types.SpanKindGeneration, types.JSONMap{
"image_url": "x",
})
require.NotNil(t, sub)
type row struct {
Name, Kind, ParentSpanID string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("name, kind, parent_span_id").
Where("knowledge_id = ? AND name = ?", "kid", "multimodal.image[0]").
Find(&rows).Error)
require.Len(t, rows, 1)
assert.Equal(t, types.SpanKindGeneration, rows[0].Kind)
assert.Equal(t, parent.SpanID, rows[0].ParentSpanID, "subspan must reference parent stage's span_id")
}
// TestSpanTracker_BeginStage_ReentryIsIdempotent guarantees that a second
// BeginStage call for the same (kid, attempt, stage) reuses the existing
// span row instead of inserting a duplicate. Without this, an asynq retry
// or any code path that begins a stage twice would produce two timeline
// segments for the same stage, and LookupStage would resolve to whichever
// row sorts first — both regressions the original implementation had.
func TestSpanTracker_BeginStage_ReentryIsIdempotent(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
first := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 1})
require.NotNil(t, first)
// Simulate an intermediate End so the row is in a terminal state when
// the re-entry happens (mirrors retry-after-failure ordering).
tracker.FailSpan(ctx, first, "TEST", "first failure", errors.New("boom"))
second := tracker.BeginStage(ctx, "kid", attempt, types.StageDocReader, types.JSONMap{"pages": 2})
require.NotNil(t, second)
assert.Equal(t, first.SpanID, second.SpanID,
"re-entry must reuse the existing stage span_id")
type row struct {
SpanID, Status string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("span_id, status").
Where("knowledge_id = ? AND attempt = ? AND name = ?", "kid", attempt, types.StageDocReader).
Find(&rows).Error)
require.Len(t, rows, 1, "exactly one row per (knowledge, attempt, stage)")
assert.Equal(t, types.SpanStatusRunning, rows[0].Status,
"row must transition back to running after re-entry")
}
// TestSpanTracker_FailSpan_CascadesDependentSubspans verifies that when a
// chunking failure flips Embedding to "cancelled" (sibling cascade),
// embedding's already-running subspan (e.g. embedding.batch[0]) is ALSO
// cancelled. Without this, the UI rendered a cancelled stage with an
// orphan running batch hanging underneath.
func TestSpanTracker_FailSpan_CascadesDependentSubspans(t *testing.T) {
tracker, db := setupSpanTrackerTest(t)
ctx := context.Background()
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
require.NoError(t, err)
chunking := tracker.BeginStage(ctx, "kid", attempt, types.StageChunking, nil)
embedding := tracker.BeginStage(ctx, "kid", attempt, types.StageEmbedding, nil)
require.NotNil(t, embedding)
// Subspan attached to the dependent (sibling) stage that's about to
// be cascade-cancelled.
batch := tracker.BeginSubSpan(ctx, embedding, "embedding.batch[0]", types.SpanKindGeneration, nil)
require.NotNil(t, batch)
tracker.FailSpan(ctx, chunking, "CHUNKING_FAILED", "synthetic", errors.New("boom"))
type row struct {
Name, Status string
}
var rows []row
require.NoError(t, db.Table("knowledge_processing_spans").
Select("name, status").
Where("knowledge_id = ?", "kid").
Find(&rows).Error)
statusBy := map[string]string{}
for _, r := range rows {
statusBy[r.Name] = r.Status
}
assert.Equal(t, types.SpanStatusCancelled, statusBy[types.StageEmbedding],
"dependent stage cascades to cancelled")
assert.Equal(t, types.SpanStatusCancelled, statusBy["embedding.batch[0]"],
"subspan under the cascaded stage must also be cancelled")
}