Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/strategy_diagnostics_test.go
T
Claude a867b1b29d feat(handler): add chunker preview endpoint for debugging
Adds POST /api/v1/chunker/preview — a read-only endpoint that runs the
adaptive chunker on supplied text and returns the chunks plus
diagnostic info (which tier won, which were rejected and why, the
DocProfile that the selector saw). Used by the upcoming KB editor
debug panel so users can experiment with chunking parameters without
re-uploading documents.

Backend pieces:
- chunker.SplitWithDiagnostics(text, cfg) returns chunks plus a
  Diagnostics struct (SelectedTier, TierChain, Rejected). Split() now
  delegates here.
- chunker.DocProfile gets JSON tags so it serializes cleanly.
- handler.PreviewChunking caps input at 256k chars and chunks at 500
  per response, computes a size-distribution summary, and never
  touches DB or embedding APIs.

Auth: lives under the v1 group so the standard JWT middleware applies.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

59 lines
2.0 KiB
Go

package chunker
import (
"strings"
"testing"
)
func TestSplitWithDiagnostics_LegacyStrategy_ReportsLegacyTier(t *testing.T) {
// Splittable input so the validator accepts the legacy output cleanly.
text := strings.Repeat("Hello world.\n\nNext paragraph here.\n\n", 50)
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy}
chunks, diag := SplitWithDiagnostics(text, cfg)
if len(chunks) == 0 {
t.Fatal("expected chunks")
}
if diag.SelectedTier != TierLegacy {
t.Errorf("expected SelectedTier=legacy, got %s", diag.SelectedTier)
}
if len(diag.TierChain) != 1 || diag.TierChain[0] != TierLegacy {
t.Errorf("expected single-tier chain [legacy], got %v", diag.TierChain)
}
if len(diag.Rejected) != 0 {
t.Errorf("expected no rejections for splittable input, got %v", diag.Rejected)
}
}
func TestSplitWithDiagnostics_AutoOnHeadingDoc_PicksHeading(t *testing.T) {
doc := strings.Repeat("# Top\nintro paragraph here.\n\n## Section A\nbody A here.\n\n## Section B\nbody B here.\n\n## Section C\nbody C here.\n\n", 1)
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Strategy: StrategyAuto}
_, diag := SplitWithDiagnostics(doc, cfg)
if len(diag.TierChain) == 0 {
t.Fatal("expected non-empty tier chain")
}
// Heading tier should be tried first for this doc.
if diag.TierChain[0] != TierHeading {
t.Errorf("expected heading tier first, got chain %v", diag.TierChain)
}
}
func TestSplitWithDiagnostics_EmptyText(t *testing.T) {
chunks, diag := SplitWithDiagnostics("", DefaultConfig())
if chunks != nil {
t.Errorf("expected nil chunks for empty text, got %v", chunks)
}
if diag == nil {
t.Fatal("diag must never be nil")
}
}
func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) {
text := "para one.\n\npara two.\n\npara three."
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10}
a := Split(text, cfg)
b, _ := SplitWithDiagnostics(text, cfg)
if len(a) != len(b) {
t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b))
}
}