feat(handler): add chunker preview endpoint for debugging
Adds POST /api/v1/chunker/preview — a read-only endpoint that runs the adaptive chunker on supplied text and returns the chunks plus diagnostic info (which tier won, which were rejected and why, the DocProfile that the selector saw). Used by the upcoming KB editor debug panel so users can experiment with chunking parameters without re-uploading documents. Backend pieces: - chunker.SplitWithDiagnostics(text, cfg) returns chunks plus a Diagnostics struct (SelectedTier, TierChain, Rejected). Split() now delegates here. - chunker.DocProfile gets JSON tags so it serializes cleanly. - handler.PreviewChunking caps input at 256k chars and chunks at 500 per response, computes a size-distribution summary, and never touches DB or embedding APIs. Auth: lives under the v1 group so the standard JWT middleware applies. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -10,34 +10,36 @@ import (
|
||||
)
|
||||
|
||||
// DocProfile holds the document-level signals used to choose a chunking tier.
|
||||
// JSON tags allow the profile to be surfaced via the chunker preview endpoint
|
||||
// for debug UIs.
|
||||
type DocProfile struct {
|
||||
TotalChars int
|
||||
TotalLines int
|
||||
AvgLineLen float64
|
||||
StdLineLen float64
|
||||
TotalChars int `json:"total_chars"`
|
||||
TotalLines int `json:"total_lines"`
|
||||
AvgLineLen float64 `json:"avg_line_len"`
|
||||
StdLineLen float64 `json:"std_line_len"`
|
||||
|
||||
// Markdown structure
|
||||
MdHeadingCounts map[int]int // level (1..6) → count
|
||||
MdHeadingTotal int
|
||||
MdHeadingCounts map[int]int `json:"md_heading_counts"` // level (1..6) → count
|
||||
MdHeadingTotal int `json:"md_heading_total"`
|
||||
|
||||
// Heuristic indicators
|
||||
NumberedSectionCount int
|
||||
AllCapsShortLineCount int
|
||||
BlankParagraphBreaks int
|
||||
FormFeedCount int
|
||||
VisualSepCount int
|
||||
GermanChapterCount int
|
||||
EnglishChapterCount int
|
||||
ChineseChapterCount int
|
||||
RepeatedFooterCount int
|
||||
NumberedSectionCount int `json:"numbered_section_count"`
|
||||
AllCapsShortLineCount int `json:"all_caps_short_line_count"`
|
||||
BlankParagraphBreaks int `json:"blank_paragraph_breaks"`
|
||||
FormFeedCount int `json:"form_feed_count"`
|
||||
VisualSepCount int `json:"visual_sep_count"`
|
||||
GermanChapterCount int `json:"german_chapter_count"`
|
||||
EnglishChapterCount int `json:"english_chapter_count"`
|
||||
ChineseChapterCount int `json:"chinese_chapter_count"`
|
||||
RepeatedFooterCount int `json:"repeated_footer_count"`
|
||||
|
||||
// Content characteristics
|
||||
HasTables bool
|
||||
HasCode bool
|
||||
CodeRatio float64
|
||||
HasTables bool `json:"has_tables"`
|
||||
HasCode bool `json:"has_code"`
|
||||
CodeRatio float64 `json:"code_ratio"`
|
||||
|
||||
// Detected language hints (best-effort)
|
||||
DetectedLangs []string
|
||||
DetectedLangs []string `json:"detected_langs"`
|
||||
}
|
||||
|
||||
// HeadingDensity returns the share of lines that are Markdown headings.
|
||||
|
||||
@@ -27,34 +27,64 @@ const (
|
||||
// always returns a non-nil result: on tier failure the chain falls through
|
||||
// to the legacy splitter, which is the original Tier 3 implementation.
|
||||
func Split(text string, cfg SplitterConfig) []Chunk {
|
||||
chunks, _ := SplitWithDiagnostics(text, cfg)
|
||||
return chunks
|
||||
}
|
||||
|
||||
// TierRejection records why a tier was rejected by the validator and the
|
||||
// chain advanced to the next tier. Surfaced by SplitWithDiagnostics for
|
||||
// the debug/preview endpoint.
|
||||
type TierRejection struct {
|
||||
Tier StrategyTier `json:"tier"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// Diagnostics captures which tier produced the returned chunks plus the
|
||||
// chain that was attempted and any rejected tiers along the way. Useful
|
||||
// for surfacing in a debug UI; not produced by the normal Split path.
|
||||
type Diagnostics struct {
|
||||
SelectedTier StrategyTier `json:"selected_tier"`
|
||||
TierChain []StrategyTier `json:"tier_chain"`
|
||||
Rejected []TierRejection `json:"rejected"`
|
||||
}
|
||||
|
||||
// SplitWithDiagnostics is the same as Split but also returns the
|
||||
// diagnostic trace (selected tier, full chain, rejection reasons). Use
|
||||
// this for the chunker preview endpoint where the caller wants to know
|
||||
// which tier won and why others lost.
|
||||
func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) {
|
||||
diag := &Diagnostics{}
|
||||
if text == "" {
|
||||
return nil
|
||||
return nil, diag
|
||||
}
|
||||
cfg = ensureDefaults(cfg)
|
||||
|
||||
chain := resolveChain(text, cfg)
|
||||
diag.TierChain = chain
|
||||
totalChars := len([]rune(text))
|
||||
|
||||
var lastOut []Chunk
|
||||
var lastTier StrategyTier
|
||||
for i, tier := range chain {
|
||||
out := runTier(tier, text, cfg)
|
||||
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
||||
return out
|
||||
} else {
|
||||
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
v := ValidateChunks(out, totalChars, cfg.ChunkSize)
|
||||
if v.OK {
|
||||
diag.SelectedTier = tier
|
||||
return out, diag
|
||||
}
|
||||
// Remember the legacy tier's output: we'll return it as-is below if
|
||||
// every tier rejected — running SplitText again would just produce
|
||||
// the same rejected result.
|
||||
diag.Rejected = append(diag.Rejected, TierRejection{Tier: tier, Reason: v.Reason})
|
||||
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
if tier == TierLegacy && i == len(chain)-1 {
|
||||
lastOut = out
|
||||
lastTier = tier
|
||||
}
|
||||
}
|
||||
if lastOut != nil {
|
||||
return lastOut
|
||||
diag.SelectedTier = lastTier
|
||||
return lastOut, diag
|
||||
}
|
||||
// Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy).
|
||||
return SplitText(text, cfg)
|
||||
// Defensive last-ditch fallback.
|
||||
diag.SelectedTier = TierLegacy
|
||||
return SplitText(text, cfg), diag
|
||||
}
|
||||
|
||||
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitWithDiagnostics_LegacyStrategy_ReportsLegacyTier(t *testing.T) {
|
||||
// Splittable input so the validator accepts the legacy output cleanly.
|
||||
text := strings.Repeat("Hello world.\n\nNext paragraph here.\n\n", 50)
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy}
|
||||
chunks, diag := SplitWithDiagnostics(text, cfg)
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("expected chunks")
|
||||
}
|
||||
if diag.SelectedTier != TierLegacy {
|
||||
t.Errorf("expected SelectedTier=legacy, got %s", diag.SelectedTier)
|
||||
}
|
||||
if len(diag.TierChain) != 1 || diag.TierChain[0] != TierLegacy {
|
||||
t.Errorf("expected single-tier chain [legacy], got %v", diag.TierChain)
|
||||
}
|
||||
if len(diag.Rejected) != 0 {
|
||||
t.Errorf("expected no rejections for splittable input, got %v", diag.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitWithDiagnostics_AutoOnHeadingDoc_PicksHeading(t *testing.T) {
|
||||
doc := strings.Repeat("# Top\nintro paragraph here.\n\n## Section A\nbody A here.\n\n## Section B\nbody B here.\n\n## Section C\nbody C here.\n\n", 1)
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Strategy: StrategyAuto}
|
||||
_, diag := SplitWithDiagnostics(doc, cfg)
|
||||
if len(diag.TierChain) == 0 {
|
||||
t.Fatal("expected non-empty tier chain")
|
||||
}
|
||||
// Heading tier should be tried first for this doc.
|
||||
if diag.TierChain[0] != TierHeading {
|
||||
t.Errorf("expected heading tier first, got chain %v", diag.TierChain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitWithDiagnostics_EmptyText(t *testing.T) {
|
||||
chunks, diag := SplitWithDiagnostics("", DefaultConfig())
|
||||
if chunks != nil {
|
||||
t.Errorf("expected nil chunks for empty text, got %v", chunks)
|
||||
}
|
||||
if diag == nil {
|
||||
t.Fatal("diag must never be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) {
|
||||
text := "para one.\n\npara two.\n\npara three."
|
||||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10}
|
||||
a := Split(text, cfg)
|
||||
b, _ := SplitWithDiagnostics(text, cfg)
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user