fix(spans): expand knowledge span name column and truncate long wiki names
Wiki ingest builds postprocess.wiki.page[<slug>] subspan names that can exceed varchar(64); widen the column to 255 and hash-suffix truncation so long slugs still persist distinctly. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
span_id VARCHAR(64) NOT NULL,
|
||||
parent_span_id VARCHAR(64),
|
||||
name VARCHAR(64) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
input TEXT,
|
||||
|
||||
@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
span_id VARCHAR(64) NOT NULL,
|
||||
parent_span_id VARCHAR(64),
|
||||
name VARCHAR(64) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
input TEXT,
|
||||
|
||||
@@ -26,6 +26,8 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -37,6 +39,26 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// maxSpanNameLen matches knowledge_processing_spans.name (varchar(255)).
|
||||
const maxSpanNameLen = 255
|
||||
|
||||
// fitSpanName ensures a span name fits the DB column. Wiki ingest builds
|
||||
// names like postprocess.wiki.page[<slug>] which can exceed 64 chars when
|
||||
// the slug is a long romanized entity name; when truncated an 8-hex hash
|
||||
// suffix keeps concurrent subspans distinct.
|
||||
func fitSpanName(name string) string {
|
||||
if len(name) <= maxSpanNameLen {
|
||||
return name
|
||||
}
|
||||
sum := sha256.Sum256([]byte(name))
|
||||
suffix := fmt.Sprintf("~%x", sum[:4])
|
||||
keep := maxSpanNameLen - len(suffix)
|
||||
if keep < 1 {
|
||||
return suffix[:maxSpanNameLen]
|
||||
}
|
||||
return name[:keep] + suffix
|
||||
}
|
||||
|
||||
// Span is the in-memory handle the pipeline holds while a stage / subspan
|
||||
// is executing. It carries enough context for End/Fail/Skip to write back
|
||||
// without re-querying the DB. Returned (and required) from every Begin*.
|
||||
@@ -373,6 +395,7 @@ func (t *spanTracker) BeginSubSpan(ctx context.Context, parent *Span, name, kind
|
||||
if parent == nil || name == "" {
|
||||
return nil
|
||||
}
|
||||
name = fitSpanName(name)
|
||||
if kind != types.SpanKindGeneration && kind != types.SpanKindSubSpan {
|
||||
kind = types.SpanKindSubSpan
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
@@ -31,7 +32,7 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
span_id VARCHAR(64) NOT NULL,
|
||||
parent_span_id VARCHAR(64),
|
||||
name VARCHAR(64) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
input TEXT,
|
||||
@@ -166,6 +167,57 @@ func TestSpanTracker_LookupStage_FindsAcrossProcesses(t *testing.T) {
|
||||
assert.Nil(t, other, "LookupStage(missing) must return nil")
|
||||
}
|
||||
|
||||
func TestFitSpanName(t *testing.T) {
|
||||
short := "postprocess.wiki.extract"
|
||||
if got := fitSpanName(short); got != short {
|
||||
t.Fatalf("short name should pass through, got %q", got)
|
||||
}
|
||||
|
||||
// Use a synthetic overlong wiki span name (> varchar(255)).
|
||||
long := "postprocess.wiki.page[concept/" + strings.Repeat("a", 280) + "]"
|
||||
got := fitSpanName(long)
|
||||
if len(got) > maxSpanNameLen {
|
||||
t.Fatalf("fitted name len=%d, want <= %d: %q", len(got), maxSpanNameLen, got)
|
||||
}
|
||||
if got == long {
|
||||
t.Fatalf("expected truncation, got unchanged %q", got)
|
||||
}
|
||||
if fitSpanName(long) != got {
|
||||
t.Fatal("fitSpanName must be deterministic")
|
||||
}
|
||||
|
||||
other := "postprocess.wiki.page[concept/" + strings.Repeat("b", 280) + "]"
|
||||
if fitSpanName(other) == got {
|
||||
t.Fatalf("different long names must not collapse to the same fitted name")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpanTracker_BeginSubSpan_LongWikiPageName verifies wiki ingest's
|
||||
// postprocess.wiki.page[<slug>] subspans persist even when the slug pushes
|
||||
// the name past varchar(255).
|
||||
func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) {
|
||||
tracker, db := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
|
||||
require.NoError(t, err)
|
||||
parent := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, nil)
|
||||
require.NotNil(t, parent)
|
||||
|
||||
rawName := "postprocess.wiki.page[concept/" + strings.Repeat("x", 280) + "]"
|
||||
sub := tracker.BeginSubSpan(ctx, parent, rawName, types.SpanKindSubSpan, types.JSONMap{
|
||||
"slug": "concept/" + strings.Repeat("x", 280),
|
||||
})
|
||||
require.NotNil(t, sub)
|
||||
require.LessOrEqual(t, len(sub.Name), maxSpanNameLen)
|
||||
|
||||
var count int64
|
||||
require.NoError(t, db.Table("knowledge_processing_spans").
|
||||
Where("knowledge_id = ? AND name = ?", "kid", sub.Name).
|
||||
Count(&count).Error)
|
||||
require.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
// TestSpanTracker_BeginSubSpan_HangsUnderParent confirms multimodal /
|
||||
// embedding fan-out subspans reference the parent stage's span_id —
|
||||
// the structural invariant the buildSpanTree handler walks.
|
||||
|
||||
@@ -86,7 +86,7 @@ type KnowledgeProcessingSpan struct {
|
||||
Attempt int `gorm:"column:attempt" json:"attempt"`
|
||||
SpanID string `gorm:"column:span_id;size:64" json:"span_id"`
|
||||
ParentSpanID string `gorm:"column:parent_span_id;size:64" json:"parent_span_id,omitempty"`
|
||||
Name string `gorm:"column:name;size:64" json:"name"`
|
||||
Name string `gorm:"column:name;size:255" json:"name"`
|
||||
Kind string `gorm:"column:kind;size:16" json:"kind"`
|
||||
Status string `gorm:"column:status;size:16" json:"status"`
|
||||
Input JSONMap `gorm:"column:input;type:jsonb" json:"input,omitempty"`
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000066 down] Reverting knowledge_processing_spans.name to VARCHAR(64)...'; END $$;
|
||||
|
||||
ALTER TABLE knowledge_processing_spans
|
||||
ALTER COLUMN name TYPE VARCHAR(64) USING LEFT(name, 64);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000066 down] knowledge_processing_spans.name reverted'; END $$;
|
||||
@@ -0,0 +1,6 @@
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000066] Expanding knowledge_processing_spans.name to VARCHAR(255)...'; END $$;
|
||||
|
||||
ALTER TABLE knowledge_processing_spans
|
||||
ALTER COLUMN name TYPE VARCHAR(255);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000066] knowledge_processing_spans.name expanded'; END $$;
|
||||
Reference in New Issue
Block a user