diff --git a/internal/application/repository/knowledge_span_repo_test.go b/internal/application/repository/knowledge_span_repo_test.go index 3b067567..341a3d4f 100644 --- a/internal/application/repository/knowledge_span_repo_test.go +++ b/internal/application/repository/knowledge_span_repo_test.go @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( attempt INTEGER NOT NULL DEFAULT 1, span_id VARCHAR(64) NOT NULL, parent_span_id VARCHAR(64), - name VARCHAR(64) NOT NULL, + name VARCHAR(255) NOT NULL, kind VARCHAR(16) NOT NULL, status VARCHAR(16) NOT NULL, input TEXT, diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index beb3ae4a..368b565a 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( attempt INTEGER NOT NULL DEFAULT 1, span_id VARCHAR(64) NOT NULL, parent_span_id VARCHAR(64), - name VARCHAR(64) NOT NULL, + name VARCHAR(255) NOT NULL, kind VARCHAR(16) NOT NULL, status VARCHAR(16) NOT NULL, input TEXT, diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go index e6ae8042..ba380533 100644 --- a/internal/application/service/knowledge_span_tracker.go +++ b/internal/application/service/knowledge_span_tracker.go @@ -26,6 +26,8 @@ package service import ( "context" + "crypto/sha256" + "fmt" "strings" "sync" "time" @@ -37,6 +39,26 @@ import ( "gorm.io/gorm" ) +// maxSpanNameLen matches knowledge_processing_spans.name (varchar(255)). +const maxSpanNameLen = 255 + +// fitSpanName ensures a span name fits the DB column. Wiki ingest builds +// names like postprocess.wiki.page[] which can exceed 64 chars when +// the slug is a long romanized entity name; when truncated an 8-hex hash +// suffix keeps concurrent subspans distinct. +func fitSpanName(name string) string { + if len(name) <= maxSpanNameLen { + return name + } + sum := sha256.Sum256([]byte(name)) + suffix := fmt.Sprintf("~%x", sum[:4]) + keep := maxSpanNameLen - len(suffix) + if keep < 1 { + return suffix[:maxSpanNameLen] + } + return name[:keep] + suffix +} + // Span is the in-memory handle the pipeline holds while a stage / subspan // is executing. It carries enough context for End/Fail/Skip to write back // without re-querying the DB. Returned (and required) from every Begin*. @@ -373,6 +395,7 @@ func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind if parent == nil || name == "" { return nil } + name = fitSpanName(name) if kind != types.SpanKindGeneration && kind != types.SpanKindSubSpan { kind = types.SpanKindSubSpan } diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go index 1073e4ff..fe63a302 100644 --- a/internal/application/service/knowledge_span_tracker_test.go +++ b/internal/application/service/knowledge_span_tracker_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strings" "testing" "github.com/Tencent/WeKnora/internal/application/repository" @@ -31,7 +32,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( attempt INTEGER NOT NULL DEFAULT 1, span_id VARCHAR(64) NOT NULL, parent_span_id VARCHAR(64), - name VARCHAR(64) NOT NULL, + name VARCHAR(255) NOT NULL, kind VARCHAR(16) NOT NULL, status VARCHAR(16) NOT NULL, input TEXT, @@ -166,6 +167,57 @@ func TestSpanTracker_LookupStage_FindsAcrossProcesses(t *testing.T) { assert.Nil(t, other, "LookupStage(missing) must return nil") } +func TestFitSpanName(t *testing.T) { + short := "postprocess.wiki.extract" + if got := fitSpanName(short); got != short { + t.Fatalf("short name should pass through, got %q", got) + } + + // Use a synthetic overlong wiki span name (> varchar(255)). + long := "postprocess.wiki.page[concept/" + strings.Repeat("a", 280) + "]" + got := fitSpanName(long) + if len(got) > maxSpanNameLen { + t.Fatalf("fitted name len=%d, want <= %d: %q", len(got), maxSpanNameLen, got) + } + if got == long { + t.Fatalf("expected truncation, got unchanged %q", got) + } + if fitSpanName(long) != got { + t.Fatal("fitSpanName must be deterministic") + } + + other := "postprocess.wiki.page[concept/" + strings.Repeat("b", 280) + "]" + if fitSpanName(other) == got { + t.Fatalf("different long names must not collapse to the same fitted name") + } +} + +// TestSpanTracker_BeginSubSpan_LongWikiPageName verifies wiki ingest's +// postprocess.wiki.page[] subspans persist even when the slug pushes +// the name past varchar(255). +func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) { + tracker, db := setupSpanTrackerTest(t) + ctx := context.Background() + + _, attempt, err := tracker.OpenAttempt(ctx, "kid", "") + require.NoError(t, err) + parent := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, nil) + require.NotNil(t, parent) + + rawName := "postprocess.wiki.page[concept/" + strings.Repeat("x", 280) + "]" + sub := tracker.BeginSubSpan(ctx, parent, rawName, types.SpanKindSubSpan, types.JSONMap{ + "slug": "concept/" + strings.Repeat("x", 280), + }) + require.NotNil(t, sub) + require.LessOrEqual(t, len(sub.Name), maxSpanNameLen) + + var count int64 + require.NoError(t, db.Table("knowledge_processing_spans"). + Where("knowledge_id = ? AND name = ?", "kid", sub.Name). + Count(&count).Error) + require.Equal(t, int64(1), count) +} + // TestSpanTracker_BeginSubSpan_HangsUnderParent confirms multimodal / // embedding fan-out subspans reference the parent stage's span_id — // the structural invariant the buildSpanTree handler walks. diff --git a/internal/types/knowledge_span.go b/internal/types/knowledge_span.go index 2ed043a7..9d121c00 100644 --- a/internal/types/knowledge_span.go +++ b/internal/types/knowledge_span.go @@ -86,7 +86,7 @@ type KnowledgeProcessingSpan struct { Attempt int `gorm:"column:attempt" json:"attempt"` SpanID string `gorm:"column:span_id;size:64" json:"span_id"` ParentSpanID string `gorm:"column:parent_span_id;size:64" json:"parent_span_id,omitempty"` - Name string `gorm:"column:name;size:64" json:"name"` + Name string `gorm:"column:name;size:255" json:"name"` Kind string `gorm:"column:kind;size:16" json:"kind"` Status string `gorm:"column:status;size:16" json:"status"` Input JSONMap `gorm:"column:input;type:jsonb" json:"input,omitempty"` diff --git a/migrations/versioned/000066_expand_knowledge_span_name.down.sql b/migrations/versioned/000066_expand_knowledge_span_name.down.sql new file mode 100644 index 00000000..b5d5f6e4 --- /dev/null +++ b/migrations/versioned/000066_expand_knowledge_span_name.down.sql @@ -0,0 +1,6 @@ +DO $$ BEGIN RAISE NOTICE '[Migration 000066 down] Reverting knowledge_processing_spans.name to VARCHAR(64)...'; END $$; + +ALTER TABLE knowledge_processing_spans + ALTER COLUMN name TYPE VARCHAR(64) USING LEFT(name, 64); + +DO $$ BEGIN RAISE NOTICE '[Migration 000066 down] knowledge_processing_spans.name reverted'; END $$; diff --git a/migrations/versioned/000066_expand_knowledge_span_name.up.sql b/migrations/versioned/000066_expand_knowledge_span_name.up.sql new file mode 100644 index 00000000..de927033 --- /dev/null +++ b/migrations/versioned/000066_expand_knowledge_span_name.up.sql @@ -0,0 +1,6 @@ +DO $$ BEGIN RAISE NOTICE '[Migration 000066] Expanding knowledge_processing_spans.name to VARCHAR(255)...'; END $$; + +ALTER TABLE knowledge_processing_spans + ALTER COLUMN name TYPE VARCHAR(255); + +DO $$ BEGIN RAISE NOTICE '[Migration 000066] knowledge_processing_spans.name expanded'; END $$;