fix(spans): make fitSpanName rune-aware and normalize lookups
Follow-up to the knowledge span name column expansion: truncate span names by rune count to match PostgreSQL VARCHAR(255), apply the same normalization in LookupSpanByName, and add regression tests for mid-length wiki slugs and CJK truncation boundaries.
This commit is contained in:
@@ -45,18 +45,25 @@ const maxSpanNameLen = 255
|
||||
// fitSpanName ensures a span name fits the DB column. Wiki ingest builds
|
||||
// names like postprocess.wiki.page[<slug>] which can exceed 64 chars when
|
||||
// the slug is a long romanized entity name; when truncated an 8-hex hash
|
||||
// suffix keeps concurrent subspans distinct.
|
||||
// suffix keeps concurrent subspans distinct. Truncation is rune-aware to
|
||||
// match PostgreSQL VARCHAR(255) character semantics and avoid splitting
|
||||
// multi-byte UTF-8 sequences.
|
||||
func fitSpanName(name string) string {
|
||||
if len(name) <= maxSpanNameLen {
|
||||
runes := []rune(name)
|
||||
if len(runes) <= maxSpanNameLen {
|
||||
return name
|
||||
}
|
||||
sum := sha256.Sum256([]byte(name))
|
||||
suffix := fmt.Sprintf("~%x", sum[:4])
|
||||
keep := maxSpanNameLen - len(suffix)
|
||||
suffixRunes := []rune(suffix)
|
||||
keep := maxSpanNameLen - len(suffixRunes)
|
||||
if keep < 1 {
|
||||
return suffix[:maxSpanNameLen]
|
||||
if len(suffixRunes) > maxSpanNameLen {
|
||||
return string(suffixRunes[:maxSpanNameLen])
|
||||
}
|
||||
return suffix
|
||||
}
|
||||
return name[:keep] + suffix
|
||||
return string(runes[:keep]) + suffix
|
||||
}
|
||||
|
||||
// Span is the in-memory handle the pipeline holds while a stage / subspan
|
||||
@@ -581,6 +588,7 @@ func (t *spanTracker) LookupSpanByName(ctx context.Context, knowledgeID string,
|
||||
if name == "" || knowledgeID == "" || attempt <= 0 {
|
||||
return nil
|
||||
}
|
||||
name = fitSpanName(name)
|
||||
rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[SpanTracker] LookupSpanByName list failed kid=%s attempt=%d: %v",
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -173,11 +174,21 @@ func TestFitSpanName(t *testing.T) {
|
||||
t.Fatalf("short name should pass through, got %q", got)
|
||||
}
|
||||
|
||||
// Regression: names in the 65–223 char window failed under VARCHAR(64)
|
||||
// but must pass through unchanged at VARCHAR(255).
|
||||
mid := "postprocess.wiki.page[concept/" + strings.Repeat("a", 100) + "]"
|
||||
if got := fitSpanName(mid); got != mid {
|
||||
t.Fatalf("mid-length wiki page name should pass through, got %q", got)
|
||||
}
|
||||
|
||||
// Use a synthetic overlong wiki span name (> varchar(255)).
|
||||
long := "postprocess.wiki.page[concept/" + strings.Repeat("a", 280) + "]"
|
||||
got := fitSpanName(long)
|
||||
if len(got) > maxSpanNameLen {
|
||||
t.Fatalf("fitted name len=%d, want <= %d: %q", len(got), maxSpanNameLen, got)
|
||||
if utf8.RuneCountInString(got) > maxSpanNameLen {
|
||||
t.Fatalf("fitted name runes=%d, want <= %d: %q", utf8.RuneCountInString(got), maxSpanNameLen, got)
|
||||
}
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("fitted name must be valid UTF-8: %q", got)
|
||||
}
|
||||
if got == long {
|
||||
t.Fatalf("expected truncation, got unchanged %q", got)
|
||||
@@ -190,6 +201,16 @@ func TestFitSpanName(t *testing.T) {
|
||||
if fitSpanName(other) == got {
|
||||
t.Fatalf("different long names must not collapse to the same fitted name")
|
||||
}
|
||||
|
||||
// CJK slugs must truncate on rune boundaries, not byte boundaries.
|
||||
cjkLong := "postprocess.wiki.page[" + strings.Repeat("中", 260) + "]"
|
||||
cjkGot := fitSpanName(cjkLong)
|
||||
if utf8.RuneCountInString(cjkGot) > maxSpanNameLen {
|
||||
t.Fatalf("CJK fitted name runes=%d, want <= %d", utf8.RuneCountInString(cjkGot), maxSpanNameLen)
|
||||
}
|
||||
if !utf8.ValidString(cjkGot) {
|
||||
t.Fatalf("CJK fitted name must be valid UTF-8: %q", cjkGot)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpanTracker_BeginSubSpan_LongWikiPageName verifies wiki ingest's
|
||||
@@ -209,7 +230,7 @@ func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) {
|
||||
"slug": "concept/" + strings.Repeat("x", 280),
|
||||
})
|
||||
require.NotNil(t, sub)
|
||||
require.LessOrEqual(t, len(sub.Name), maxSpanNameLen)
|
||||
require.LessOrEqual(t, utf8.RuneCountInString(sub.Name), maxSpanNameLen)
|
||||
|
||||
var count int64
|
||||
require.NoError(t, db.Table("knowledge_processing_spans").
|
||||
@@ -218,6 +239,27 @@ func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) {
|
||||
require.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
// TestSpanTracker_LookupSpanByName_FitsLongName verifies cross-process
|
||||
// callers can look up a wiki page subspan using the raw overlong name.
|
||||
func TestSpanTracker_LookupSpanByName_FitsLongName(t *testing.T) {
|
||||
tracker, _ := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "")
|
||||
require.NoError(t, err)
|
||||
parent := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, nil)
|
||||
require.NotNil(t, parent)
|
||||
|
||||
rawName := "postprocess.wiki.page[concept/" + strings.Repeat("y", 280) + "]"
|
||||
created := tracker.BeginSubSpan(ctx, parent, rawName, types.SpanKindSubSpan, nil)
|
||||
require.NotNil(t, created)
|
||||
|
||||
found := tracker.LookupSpanByName(ctx, "kid", attempt, rawName)
|
||||
require.NotNil(t, found, "LookupSpanByName must normalize the raw name")
|
||||
assert.Equal(t, created.SpanID, found.SpanID)
|
||||
assert.Equal(t, created.Name, found.Name)
|
||||
}
|
||||
|
||||
// TestSpanTracker_BeginSubSpan_HangsUnderParent confirms multimodal /
|
||||
// embedding fan-out subspans reference the parent stage's span_id —
|
||||
// the structural invariant the buildSpanTree handler walks.
|
||||
|
||||
Reference in New Issue
Block a user