From c8793b001d6cf5fd5d05bd90e8540edd97fff085 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 13 Jul 2026 10:37:37 +0800 Subject: [PATCH] fix(spans): make fitSpanName rune-aware and normalize lookups Follow-up to the knowledge span name column expansion: truncate span names by rune count to match PostgreSQL VARCHAR(255), apply the same normalization in LookupSpanByName, and add regression tests for mid-length wiki slugs and CJK truncation boundaries. --- .../service/knowledge_span_tracker.go | 18 +++++-- .../service/knowledge_span_tracker_test.go | 48 +++++++++++++++++-- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go index ba380533..e4228b7c 100644 --- a/internal/application/service/knowledge_span_tracker.go +++ b/internal/application/service/knowledge_span_tracker.go @@ -45,18 +45,25 @@ const maxSpanNameLen = 255 // fitSpanName ensures a span name fits the DB column. Wiki ingest builds // names like postprocess.wiki.page[] which can exceed 64 chars when // the slug is a long romanized entity name; when truncated an 8-hex hash -// suffix keeps concurrent subspans distinct. +// suffix keeps concurrent subspans distinct. Truncation is rune-aware to +// match PostgreSQL VARCHAR(255) character semantics and avoid splitting +// multi-byte UTF-8 sequences. func fitSpanName(name string) string { - if len(name) <= maxSpanNameLen { + runes := []rune(name) + if len(runes) <= maxSpanNameLen { return name } sum := sha256.Sum256([]byte(name)) suffix := fmt.Sprintf("~%x", sum[:4]) - keep := maxSpanNameLen - len(suffix) + suffixRunes := []rune(suffix) + keep := maxSpanNameLen - len(suffixRunes) if keep < 1 { - return suffix[:maxSpanNameLen] + if len(suffixRunes) > maxSpanNameLen { + return string(suffixRunes[:maxSpanNameLen]) + } + return suffix } - return name[:keep] + suffix + return string(runes[:keep]) + suffix } // Span is the in-memory handle the pipeline holds while a stage / subspan @@ -581,6 +588,7 @@ func (t *spanTracker) LookupSpanByName(ctx context.Context, knowledgeID string, if name == "" || knowledgeID == "" || attempt <= 0 { return nil } + name = fitSpanName(name) rows, err := t.repo.ListByAttempt(ctx, knowledgeID, attempt) if err != nil { logger.Warnf(ctx, "[SpanTracker] LookupSpanByName list failed kid=%s attempt=%d: %v", diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go index fe63a302..3b1fbf79 100644 --- a/internal/application/service/knowledge_span_tracker_test.go +++ b/internal/application/service/knowledge_span_tracker_test.go @@ -6,6 +6,7 @@ import ( "errors" "strings" "testing" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/types" @@ -173,11 +174,21 @@ func TestFitSpanName(t *testing.T) { t.Fatalf("short name should pass through, got %q", got) } + // Regression: names in the 65–223 char window failed under VARCHAR(64) + // but must pass through unchanged at VARCHAR(255). + mid := "postprocess.wiki.page[concept/" + strings.Repeat("a", 100) + "]" + if got := fitSpanName(mid); got != mid { + t.Fatalf("mid-length wiki page name should pass through, got %q", got) + } + // Use a synthetic overlong wiki span name (> varchar(255)). long := "postprocess.wiki.page[concept/" + strings.Repeat("a", 280) + "]" got := fitSpanName(long) - if len(got) > maxSpanNameLen { - t.Fatalf("fitted name len=%d, want <= %d: %q", len(got), maxSpanNameLen, got) + if utf8.RuneCountInString(got) > maxSpanNameLen { + t.Fatalf("fitted name runes=%d, want <= %d: %q", utf8.RuneCountInString(got), maxSpanNameLen, got) + } + if !utf8.ValidString(got) { + t.Fatalf("fitted name must be valid UTF-8: %q", got) } if got == long { t.Fatalf("expected truncation, got unchanged %q", got) @@ -190,6 +201,16 @@ func TestFitSpanName(t *testing.T) { if fitSpanName(other) == got { t.Fatalf("different long names must not collapse to the same fitted name") } + + // CJK slugs must truncate on rune boundaries, not byte boundaries. + cjkLong := "postprocess.wiki.page[" + strings.Repeat("中", 260) + "]" + cjkGot := fitSpanName(cjkLong) + if utf8.RuneCountInString(cjkGot) > maxSpanNameLen { + t.Fatalf("CJK fitted name runes=%d, want <= %d", utf8.RuneCountInString(cjkGot), maxSpanNameLen) + } + if !utf8.ValidString(cjkGot) { + t.Fatalf("CJK fitted name must be valid UTF-8: %q", cjkGot) + } } // TestSpanTracker_BeginSubSpan_LongWikiPageName verifies wiki ingest's @@ -209,7 +230,7 @@ func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) { "slug": "concept/" + strings.Repeat("x", 280), }) require.NotNil(t, sub) - require.LessOrEqual(t, len(sub.Name), maxSpanNameLen) + require.LessOrEqual(t, utf8.RuneCountInString(sub.Name), maxSpanNameLen) var count int64 require.NoError(t, db.Table("knowledge_processing_spans"). @@ -218,6 +239,27 @@ func TestSpanTracker_BeginSubSpan_LongWikiPageName(t *testing.T) { require.Equal(t, int64(1), count) } +// TestSpanTracker_LookupSpanByName_FitsLongName verifies cross-process +// callers can look up a wiki page subspan using the raw overlong name. +func TestSpanTracker_LookupSpanByName_FitsLongName(t *testing.T) { + tracker, _ := setupSpanTrackerTest(t) + ctx := context.Background() + + _, attempt, err := tracker.OpenAttempt(ctx, "kid", "") + require.NoError(t, err) + parent := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, nil) + require.NotNil(t, parent) + + rawName := "postprocess.wiki.page[concept/" + strings.Repeat("y", 280) + "]" + created := tracker.BeginSubSpan(ctx, parent, rawName, types.SpanKindSubSpan, nil) + require.NotNil(t, created) + + found := tracker.LookupSpanByName(ctx, "kid", attempt, rawName) + require.NotNil(t, found, "LookupSpanByName must normalize the raw name") + assert.Equal(t, created.SpanID, found.SpanID) + assert.Equal(t, created.Name, found.Name) +} + // TestSpanTracker_BeginSubSpan_HangsUnderParent confirms multimodal / // embedding fan-out subspans reference the parent stage's span_id — // the structural invariant the buildSpanTree handler walks.