diff --git a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue index 1a86c15e..51ce2c6f 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue @@ -161,7 +161,8 @@ interface Props { const props = defineProps() const { t } = useI18n() -const MAX_CHARS = 256 * 1024 +// Mirrors handler.previewMaxChars on the backend. Keep in sync. +const MAX_CHARS = 64 * 1024 const open = ref(false) const sample = ref('') @@ -181,15 +182,18 @@ const runPreview = async () => { result.value = null expandedChunks.value = new Set() try { + // Send all fields explicitly (including empty / 0 / []) so the + // preview faithfully reflects what would happen on save. Mirrors + // the buildSubmitData convention in KnowledgeBaseEditorModal. const resp = await previewChunking({ text: sample.value, chunking_config: { chunk_size: props.config.chunkSize, chunk_overlap: props.config.chunkOverlap, separators: props.config.separators, - strategy: props.config.strategy || undefined, - token_limit: props.config.tokenLimit || undefined, - languages: props.config.languages?.length ? props.config.languages : undefined + strategy: props.config.strategy ?? '', + token_limit: props.config.tokenLimit ?? 0, + languages: props.config.languages ?? [] } }) if (!resp.success) { diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go index 213a0a70..609e5951 100644 --- a/internal/handler/chunker_debug.go +++ b/internal/handler/chunker_debug.go @@ -8,17 +8,28 @@ import ( "context" "math" "net/http" + "strings" "time" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/infrastructure/chunker" "github.com/Tencent/WeKnora/internal/logger" "github.com/gin-gonic/gin" ) -// previewMaxChars caps the input text size so callers can't tie up the -// server with arbitrarily large payloads. Chosen to stay well below the -// 256 KB byte limit even for ASCII (256 KB / ~1 byte/rune). -const previewMaxChars = 256 * 1024 +// previewMaxChars caps the input text size so a single preview request +// cannot tie up the splitter for long. Chosen to bound worst-case CPU +// well under the previewTimeout: at 64k runes even the heaviest tier +// chain finishes in well under a second on commodity hardware. +// +// SECURITY NOTE: the splitter is CPU-bound and does NOT accept a +// context.Context. When previewTimeout fires the handler returns to the +// caller, but the worker goroutine keeps running until the splitter +// finishes naturally. The 64k ceiling is the primary mitigation against +// goroutine pile-up under repeated authenticated requests. If the +// splitter ever gains context-awareness, the goroutine-wrapper in +// PreviewChunking should switch to it for true cancellation. +const previewMaxChars = 64 * 1024 // previewMaxChunks caps the number of chunks returned in a single preview // response so the UI doesn't choke on pathological splits. Stats are @@ -26,14 +37,15 @@ const previewMaxChars = 256 * 1024 // avg/min/max/stddev stay representative. const previewMaxChunks = 500 -// previewTimeout caps how long the splitter is allowed to run for a single -// preview call. CJK input at the 256k-rune ceiling can otherwise take -// several seconds across all four tier attempts. +// previewTimeout caps how long the handler waits for the splitter +// goroutine before returning a 504. See note above on previewMaxChars. const previewTimeout = 5 * time.Second // PreviewChunkingRequest is the body shape accepted by /chunker/preview. +// Text is checked manually below so we can return a friendlier error than +// gin's default "Field validation for 'Text' failed on the 'required' tag". type PreviewChunkingRequest struct { - Text string `json:"text" binding:"required"` + Text string `json:"text"` ChunkingConfig PreviewChunkingPayload `json:"chunking_config"` } @@ -100,7 +112,15 @@ func PreviewChunking(c *gin.Context) { return } - if len([]rune(req.Text)) > previewMaxChars { + if strings.TrimSpace(req.Text) == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "error": "text is empty — paste a sample to preview chunking", + }) + return + } + + if utf8.RuneCountInString(req.Text) > previewMaxChars { c.JSON(http.StatusRequestEntityTooLarge, gin.H{ "success": false, "error": "text exceeds preview limit", @@ -159,24 +179,32 @@ func PreviewChunking(c *gin.Context) { lang = profile.DetectedLangs[0] } + // Compute rune lengths once per chunk; reused for stats and result + // payload below. Avoids the previous triple-pass over each chunk's + // content (stats + result + ApproxTokenCount each rune-counted). + runeLens := make([]int, len(chunks)) + for i, ch := range chunks { + runeLens[i] = utf8.RuneCountInString(ch.Content) + } + // Compute stats over the FULL chunk set first so the metrics stay // representative even when we trim the response to previewMaxChunks. totalCount := len(chunks) - stats := computeChunkSizeStats(chunks, lang) + stats := computeChunkSizeStats(runeLens) if totalCount > previewMaxChunks { stats.TruncatedTo = totalCount chunks = chunks[:previewMaxChunks] + runeLens = runeLens[:previewMaxChunks] } results := make([]PreviewChunkResult, 0, len(chunks)) - for _, ch := range chunks { - runeLen := len([]rune(ch.Content)) + for i, ch := range chunks { results = append(results, PreviewChunkResult{ Seq: ch.Seq, Start: ch.Start, End: ch.End, - SizeChars: runeLen, - SizeTokensApprox: chunker.ApproxTokenCount(ch.Content, lang), + SizeChars: runeLens[i], + SizeTokensApprox: chunker.ApproxTokenCountFromRuneLen(runeLens[i], lang), ContextHeader: ch.ContextHeader, Content: ch.Content, }) @@ -193,23 +221,20 @@ func PreviewChunking(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } -// computeChunkSizeStats walks the full chunk slice once and returns the -// size distribution stats. Operates directly on chunker.Chunk so we don't -// need to materialize PreviewChunkResult before truncation. -// -// lang is forwarded to ApproxTokenCount only if callers extend the stats -// later — currently the result struct only tracks chars. -func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStats { - stats := PreviewChunkingStats{Count: len(chunks)} - if len(chunks) == 0 { +// computeChunkSizeStats summarizes count / avg / min / max / stddev from +// a pre-computed rune-length slice. Decoupling from chunker.Chunk lets +// the caller compute rune lengths once and reuse them for the response +// payload (avoids a second []rune allocation per chunk). +func computeChunkSizeStats(runeLens []int) PreviewChunkingStats { + stats := PreviewChunkingStats{Count: len(runeLens)} + if len(runeLens) == 0 { return stats } var sum, sumSq float64 minLen, maxLen := math.MaxInt32, 0 - for _, ch := range chunks { - l := len([]rune(ch.Content)) + for _, l := range runeLens { sum += float64(l) - sumSq += float64(l * l) + sumSq += float64(l) * float64(l) if l < minLen { minLen = l } @@ -217,8 +242,8 @@ func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStat maxLen = l } } - avg := sum / float64(len(chunks)) - variance := sumSq/float64(len(chunks)) - avg*avg + avg := sum / float64(len(runeLens)) + variance := sumSq/float64(len(runeLens)) - avg*avg if variance < 0 { // Float precision can push the variance slightly below zero on // near-uniform inputs; clamp so sqrt doesn't return NaN. diff --git a/internal/handler/chunker_debug_test.go b/internal/handler/chunker_debug_test.go new file mode 100644 index 00000000..0210667a --- /dev/null +++ b/internal/handler/chunker_debug_test.go @@ -0,0 +1,199 @@ +package handler + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/infrastructure/chunker" + "github.com/gin-gonic/gin" +) + +func init() { + gin.SetMode(gin.TestMode) +} + +func TestComputeChunkSizeStats_Empty(t *testing.T) { + stats := computeChunkSizeStats(nil) + if stats.Count != 0 || stats.AvgChars != 0 || stats.MaxChars != 0 { + t.Errorf("empty input should yield zero stats, got %+v", stats) + } +} + +func TestComputeChunkSizeStats_SingleChunk(t *testing.T) { + stats := computeChunkSizeStats([]int{500}) + if stats.Count != 1 { + t.Errorf("count: got %d want 1", stats.Count) + } + if stats.AvgChars != 500 || stats.MinChars != 500 || stats.MaxChars != 500 { + t.Errorf("single-chunk stats should all equal 500, got %+v", stats) + } + if stats.StddevChars != 0 { + t.Errorf("stddev for one element should be 0, got %d", stats.StddevChars) + } +} + +func TestComputeChunkSizeStats_VaryingSizes(t *testing.T) { + // 100, 200, 300, 400, 500 → avg 300, stddev ≈ 141 + stats := computeChunkSizeStats([]int{100, 200, 300, 400, 500}) + if stats.Count != 5 { + t.Errorf("count: got %d want 5", stats.Count) + } + if stats.AvgChars != 300 { + t.Errorf("avg: got %d want 300", stats.AvgChars) + } + if stats.MinChars != 100 || stats.MaxChars != 500 { + t.Errorf("min/max: got %d/%d want 100/500", stats.MinChars, stats.MaxChars) + } + if stats.StddevChars < 130 || stats.StddevChars > 150 { + t.Errorf("stddev: got %d, want ~141", stats.StddevChars) + } +} + +func TestComputeChunkSizeStats_NoVarianceUnderflow(t *testing.T) { + // All identical — variance must clamp to 0 not flip negative on + // float-precision rounding. + stats := computeChunkSizeStats([]int{1234, 1234, 1234, 1234}) + if stats.StddevChars != 0 { + t.Errorf("identical values must yield stddev=0, got %d", stats.StddevChars) + } +} + +// --- PreviewChunking httptest ------------------------------------------------- + +func newPreviewRouter() *gin.Engine { + r := gin.New() + r.POST("/chunker/preview", PreviewChunking) + return r +} + +func postPreview(t *testing.T, body any) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + r := newPreviewRouter() + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(body); err != nil { + t.Fatalf("encode body: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/chunker/preview", buf) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + var parsed map[string]any + if w.Body.Len() > 0 { + _ = json.Unmarshal(w.Body.Bytes(), &parsed) + } + return w, parsed +} + +func TestPreviewChunking_HappyPath_AutoStrategy(t *testing.T) { + body := PreviewChunkingRequest{ + Text: "# Top\nintro paragraph here.\n\n## Section A\nbody A.\n\n## Section B\nbody B.", + ChunkingConfig: PreviewChunkingPayload{ + ChunkSize: 200, + ChunkOverlap: 20, + Separators: []string{"\n\n", "\n"}, + Strategy: "auto", + }, + } + w, parsed := postPreview(t, body) + if w.Code != http.StatusOK { + t.Fatalf("status: got %d want 200; body=%s", w.Code, w.Body.String()) + } + if parsed["success"] != true { + t.Fatalf("success flag missing or false: %v", parsed) + } + data, ok := parsed["data"].(map[string]any) + if !ok { + t.Fatalf("data missing: %v", parsed) + } + if data["selected_tier"] == "" { + t.Errorf("selected_tier must be set, got %v", data["selected_tier"]) + } + if _, ok := data["chunks"].([]any); !ok { + t.Errorf("chunks must be an array, got %T", data["chunks"]) + } + stats, ok := data["stats"].(map[string]any) + if !ok { + t.Fatalf("stats must be an object, got %T", data["stats"]) + } + if c, _ := stats["count"].(float64); c <= 0 { + t.Errorf("stats.count should be > 0, got %v", stats["count"]) + } +} + +func TestPreviewChunking_RejectsEmptyText(t *testing.T) { + w, parsed := postPreview(t, PreviewChunkingRequest{Text: " \n\t "}) + if w.Code != http.StatusBadRequest { + t.Errorf("status: got %d want 400", w.Code) + } + if errStr, _ := parsed["error"].(string); !strings.Contains(errStr, "empty") { + t.Errorf("error should mention 'empty', got %q", errStr) + } +} + +func TestPreviewChunking_RejectsOversizedText(t *testing.T) { + body := PreviewChunkingRequest{Text: strings.Repeat("a", previewMaxChars+1)} + w, parsed := postPreview(t, body) + if w.Code != http.StatusRequestEntityTooLarge { + t.Errorf("status: got %d want 413", w.Code) + } + if parsed["limit"] == nil { + t.Errorf("response should include limit hint, got %v", parsed) + } +} + +func TestPreviewChunking_LegacyStrategy_NoProfile(t *testing.T) { + // Auto-strategy is the only path that produces a profile inside + // SplitWithDiagnostics. For explicit strategies the handler + // materializes one itself so the UI always sees stats. + body := PreviewChunkingRequest{ + Text: "para one.\n\npara two.\n\npara three.\n\npara four.", + ChunkingConfig: PreviewChunkingPayload{ + ChunkSize: 100, + ChunkOverlap: 10, + Separators: []string{"\n\n"}, + Strategy: "legacy", + }, + } + w, parsed := postPreview(t, body) + if w.Code != http.StatusOK { + t.Fatalf("status %d body=%s", w.Code, w.Body.String()) + } + data := parsed["data"].(map[string]any) + if data["profile"] == nil { + t.Error("profile should be materialized for explicit strategy too") + } + if string(chunker.StrategyTier(data["selected_tier"].(string))) != string(chunker.TierLegacy) { + t.Errorf("selected_tier: got %v want %s", data["selected_tier"], chunker.TierLegacy) + } +} + +func TestPreviewChunking_ChunkTruncation(t *testing.T) { + // Build text that produces > previewMaxChunks chunks. + body := PreviewChunkingRequest{ + Text: strings.Repeat("x.\n\n", previewMaxChunks+50), + ChunkingConfig: PreviewChunkingPayload{ + ChunkSize: 3, + ChunkOverlap: 0, + Separators: []string{"\n\n"}, + Strategy: "legacy", + }, + } + w, parsed := postPreview(t, body) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + data := parsed["data"].(map[string]any) + chunks := data["chunks"].([]any) + if len(chunks) > previewMaxChunks { + t.Errorf("chunks should be truncated to ≤%d, got %d", previewMaxChunks, len(chunks)) + } + stats := data["stats"].(map[string]any) + if truncated, _ := stats["truncated_to"].(float64); int(truncated) <= previewMaxChunks { + t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated) + } +} diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index aaef43bb..ba2f5fd7 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -199,8 +199,10 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, * } // runTier dispatches the splitter implementation for the given tier. -// Heading and heuristic tiers are stubbed in this scaffold and currently -// fall through to the legacy splitter — they are filled in in later phases. +// splitByHeadings / splitByHeuristics are package-level vars overridden +// from heading_splitter.go / heuristic_splitter.go via init(); recursive +// and legacy share the same SplitText path. The default branch is kept +// as defensive belt-and-suspenders for future StrategyTier additions. func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk { switch tier { case TierHeading: diff --git a/internal/infrastructure/chunker/strategy_diagnostics_test.go b/internal/infrastructure/chunker/strategy_diagnostics_test.go index d9b936bf..82bc8aa0 100644 --- a/internal/infrastructure/chunker/strategy_diagnostics_test.go +++ b/internal/infrastructure/chunker/strategy_diagnostics_test.go @@ -47,12 +47,52 @@ func TestSplitWithDiagnostics_EmptyText(t *testing.T) { } } -func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) { +// TestSplit_AndDiagnostics_AgreeOnChunks ensures Split (no diagnostics) +// and SplitWithDiagnostics produce the same chunk set for a given input. +// They run independent loops as of the post-audit refactor — this test +// is the regression wall against them drifting. +func TestSplit_AndDiagnostics_AgreeOnChunks(t *testing.T) { text := "para one.\n\npara two.\n\npara three." cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10} a := Split(text, cfg) - b, _ := SplitWithDiagnostics(text, cfg) + b, diag := SplitWithDiagnostics(text, cfg) if len(a) != len(b) { - t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b)) + t.Fatalf("chunk count disagrees: Split=%d Diagnostics=%d", len(a), len(b)) + } + for i := range a { + if a[i].Content != b[i].Content || a[i].Start != b[i].Start || a[i].End != b[i].End { + t.Errorf("chunk %d differs:\n Split: %+v\n Diag : %+v", i, a[i], b[i]) + } + } + if diag == nil { + t.Error("diagnostics must not be nil") + } +} + +// TestSplitWithDiagnostics_ProfileSetForAuto verifies that auto-strategy +// returns the DocProfile that drove tier selection — required by the +// preview endpoint to avoid double-profiling. +func TestSplitWithDiagnostics_ProfileSetForAuto(t *testing.T) { + doc := "# Top\nintro.\n\n## A\nbody A.\n\n## B\nbody B." + _, diag := SplitWithDiagnostics(doc, SplitterConfig{ChunkSize: 200, Strategy: StrategyAuto}) + if diag.Profile == nil { + t.Fatal("auto strategy must populate diag.Profile") + } + if diag.Profile.MdHeadingTotal == 0 { + t.Errorf("profile should have detected headings, got %+v", diag.Profile) + } +} + +// TestSplitWithDiagnostics_ProfileNilForExplicit verifies the inverse: +// explicit strategies bypass profiling and leave Profile nil so the +// preview handler knows to materialize one if it needs stats. +func TestSplitWithDiagnostics_ProfileNilForExplicit(t *testing.T) { + for _, strat := range []string{StrategyHeading, StrategyHeuristic, StrategyRecursive, StrategyLegacy} { + t.Run(strat, func(t *testing.T) { + _, diag := SplitWithDiagnostics("plain text", SplitterConfig{ChunkSize: 200, Strategy: strat}) + if diag.Profile != nil { + t.Errorf("strategy %q should leave Profile nil, got %+v", strat, diag.Profile) + } + }) } } diff --git a/internal/infrastructure/chunker/tokens.go b/internal/infrastructure/chunker/tokens.go index 568f2528..6c0195f4 100644 --- a/internal/infrastructure/chunker/tokens.go +++ b/internal/infrastructure/chunker/tokens.go @@ -6,7 +6,10 @@ // over-estimate token counts so that chunks stay safely under model limits. package chunker -import "unicode" +import ( + "unicode" + "unicode/utf8" +) // Language identifiers used by the token estimator and the heuristic splitter. const ( @@ -31,12 +34,23 @@ func ApproxTokenCount(s string, lang string) int { if s == "" { return 0 } + return ApproxTokenCountFromRuneLen(utf8.RuneCountInString(s), lang) +} + +// ApproxTokenCountFromRuneLen is the allocation-free variant of +// ApproxTokenCount when the caller has already computed the rune length. +// Use this in hot loops where the same content's rune count would +// otherwise be recomputed multiple times (e.g. preview endpoint emitting +// per-chunk stats). +func ApproxTokenCountFromRuneLen(runeLen int, lang string) int { + if runeLen <= 0 { + return 0 + } ratio, ok := charsPerToken[lang] if !ok { ratio = charsPerToken[LangMixed] } - runes := []rune(s) - approx := float64(len(runes)) / ratio + approx := float64(runeLen) / ratio if approx < 1 { return 1 }