feat(handler): add chunker preview endpoint for debugging

Adds POST /api/v1/chunker/preview — a read-only endpoint that runs the
adaptive chunker on supplied text and returns the chunks plus
diagnostic info (which tier won, which were rejected and why, the
DocProfile that the selector saw). Used by the upcoming KB editor
debug panel so users can experiment with chunking parameters without
re-uploading documents.

Backend pieces:
- chunker.SplitWithDiagnostics(text, cfg) returns chunks plus a
  Diagnostics struct (SelectedTier, TierChain, Rejected). Split() now
  delegates here.
- chunker.DocProfile gets JSON tags so it serializes cleanly.
- handler.PreviewChunking caps input at 256k chars and chunks at 500
  per response, computes a size-distribution summary, and never
  touches DB or embedding APIs.

Auth: lives under the v1 group so the standard JWT middleware applies.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-03 06:30:55 +00:00
committed by lyingbug
parent 46d9c396c6
commit a867b1b29d
5 changed files with 306 additions and 31 deletions
+178
View File
@@ -0,0 +1,178 @@
// Package handler — chunker_debug.go exposes a read-only preview endpoint
// that runs the adaptive chunker on supplied text without touching the DB
// or generating embeddings. Used by the KB editor's debug panel so users
// can experiment with chunking parameters before committing to a re-index.
package handler
import (
"math"
"net/http"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/gin-gonic/gin"
)
// previewMaxChars caps the input text size so callers can't tie up the
// server with arbitrarily large payloads. Chosen to stay well below the
// 256 KB byte limit even for ASCII (256 KB / ~1 byte/rune).
const previewMaxChars = 256 * 1024
// previewMaxChunks caps the number of chunks returned in a single preview
// response so the UI doesn't choke on pathological splits.
const previewMaxChunks = 500
// PreviewChunkingRequest is the body shape accepted by /chunker/preview.
type PreviewChunkingRequest struct {
Text string `json:"text" binding:"required"`
ChunkingConfig PreviewChunkingPayload `json:"chunking_config"`
}
// PreviewChunkingPayload mirrors the snake_case JSON the rest of the API
// uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig
// directly because it carries a lot of unrelated fields (parser engine
// rules, parent-child sizes, etc.) that the preview path doesn't need.
type PreviewChunkingPayload struct {
ChunkSize int `json:"chunk_size"`
ChunkOverlap int `json:"chunk_overlap"`
Separators []string `json:"separators"`
Strategy string `json:"strategy"`
TokenLimit int `json:"token_limit"`
Languages []string `json:"languages"`
}
// PreviewChunkResult describes one chunk emitted during preview.
type PreviewChunkResult struct {
Seq int `json:"seq"`
Start int `json:"start"`
End int `json:"end"`
SizeChars int `json:"size_chars"`
SizeTokensApprox int `json:"size_tokens_approx"`
ContextHeader string `json:"context_header,omitempty"`
Content string `json:"content"`
}
// PreviewChunkingStats summarizes chunk-size distribution.
type PreviewChunkingStats struct {
Count int `json:"count"`
AvgChars int `json:"avg_chars"`
MinChars int `json:"min_chars"`
MaxChars int `json:"max_chars"`
StddevChars int `json:"stddev_chars"`
TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response
}
// PreviewChunkingResponse is the body returned by /chunker/preview.
type PreviewChunkingResponse struct {
SelectedTier chunker.StrategyTier `json:"selected_tier"`
TierChain []chunker.StrategyTier `json:"tier_chain"`
Rejected []chunker.TierRejection `json:"rejected"`
Profile *chunker.DocProfile `json:"profile"`
Chunks []PreviewChunkResult `json:"chunks"`
Stats PreviewChunkingStats `json:"stats"`
}
// PreviewChunking handles POST /chunker/preview. It runs the supplied text
// through the adaptive chunker and returns the chunks plus diagnostic
// information about which tier won. Read-only: no DB writes, no embedding
// calls, no logging of the supplied text.
func PreviewChunking(c *gin.Context) {
ctx := c.Request.Context()
var req PreviewChunkingRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": "invalid request body: " + err.Error()})
return
}
if len([]rune(req.Text)) > previewMaxChars {
c.JSON(http.StatusRequestEntityTooLarge, gin.H{
"success": false,
"error": "text exceeds preview limit",
"limit": previewMaxChars,
})
return
}
cfg := chunker.SplitterConfig{
ChunkSize: req.ChunkingConfig.ChunkSize,
ChunkOverlap: req.ChunkingConfig.ChunkOverlap,
Separators: req.ChunkingConfig.Separators,
Strategy: req.ChunkingConfig.Strategy,
TokenLimit: req.ChunkingConfig.TokenLimit,
Languages: req.ChunkingConfig.Languages,
}
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
profile := chunker.ProfileDocument(req.Text)
logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks))
totalCount := len(chunks)
truncatedTo := 0
if totalCount > previewMaxChunks {
truncatedTo = totalCount
chunks = chunks[:previewMaxChunks]
}
lang := chunker.LangMixed
if len(profile.DetectedLangs) > 0 {
lang = profile.DetectedLangs[0]
}
results := make([]PreviewChunkResult, 0, len(chunks))
for _, ch := range chunks {
runeLen := len([]rune(ch.Content))
results = append(results, PreviewChunkResult{
Seq: ch.Seq,
Start: ch.Start,
End: ch.End,
SizeChars: runeLen,
SizeTokensApprox: chunker.ApproxTokenCount(ch.Content, lang),
ContextHeader: ch.ContextHeader,
Content: ch.Content,
})
}
resp := PreviewChunkingResponse{
SelectedTier: diag.SelectedTier,
TierChain: diag.TierChain,
Rejected: diag.Rejected,
Profile: profile,
Chunks: results,
Stats: chunkStats(results, truncatedTo),
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": resp})
}
// chunkStats computes count / avg / min / max / stddev over the chunks'
// SizeChars values. Returns zeroes for an empty slice.
func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats {
stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo}
if len(chunks) == 0 {
return stats
}
var sum, sumSq float64
minLen, maxLen := math.MaxInt32, 0
for _, ch := range chunks {
l := ch.SizeChars
sum += float64(l)
sumSq += float64(l * l)
if l < minLen {
minLen = l
}
if l > maxLen {
maxLen = l
}
}
avg := sum / float64(len(chunks))
variance := sumSq/float64(len(chunks)) - avg*avg
if variance < 0 {
variance = 0
}
stats.AvgChars = int(avg + 0.5)
stats.MinChars = minLen
stats.MaxChars = maxLen
stats.StddevChars = int(math.Sqrt(variance) + 0.5)
return stats
}
+21 -19
View File
@@ -10,34 +10,36 @@ import (
)
// DocProfile holds the document-level signals used to choose a chunking tier.
// JSON tags allow the profile to be surfaced via the chunker preview endpoint
// for debug UIs.
type DocProfile struct {
TotalChars int
TotalLines int
AvgLineLen float64
StdLineLen float64
TotalChars int `json:"total_chars"`
TotalLines int `json:"total_lines"`
AvgLineLen float64 `json:"avg_line_len"`
StdLineLen float64 `json:"std_line_len"`
// Markdown structure
MdHeadingCounts map[int]int // level (1..6) → count
MdHeadingTotal int
MdHeadingCounts map[int]int `json:"md_heading_counts"` // level (1..6) → count
MdHeadingTotal int `json:"md_heading_total"`
// Heuristic indicators
NumberedSectionCount int
AllCapsShortLineCount int
BlankParagraphBreaks int
FormFeedCount int
VisualSepCount int
GermanChapterCount int
EnglishChapterCount int
ChineseChapterCount int
RepeatedFooterCount int
NumberedSectionCount int `json:"numbered_section_count"`
AllCapsShortLineCount int `json:"all_caps_short_line_count"`
BlankParagraphBreaks int `json:"blank_paragraph_breaks"`
FormFeedCount int `json:"form_feed_count"`
VisualSepCount int `json:"visual_sep_count"`
GermanChapterCount int `json:"german_chapter_count"`
EnglishChapterCount int `json:"english_chapter_count"`
ChineseChapterCount int `json:"chinese_chapter_count"`
RepeatedFooterCount int `json:"repeated_footer_count"`
// Content characteristics
HasTables bool
HasCode bool
CodeRatio float64
HasTables bool `json:"has_tables"`
HasCode bool `json:"has_code"`
CodeRatio float64 `json:"code_ratio"`
// Detected language hints (best-effort)
DetectedLangs []string
DetectedLangs []string `json:"detected_langs"`
}
// HeadingDensity returns the share of lines that are Markdown headings.
+42 -12
View File
@@ -27,34 +27,64 @@ const (
// always returns a non-nil result: on tier failure the chain falls through
// to the legacy splitter, which is the original Tier 3 implementation.
func Split(text string, cfg SplitterConfig) []Chunk {
chunks, _ := SplitWithDiagnostics(text, cfg)
return chunks
}
// TierRejection records why a tier was rejected by the validator and the
// chain advanced to the next tier. Surfaced by SplitWithDiagnostics for
// the debug/preview endpoint.
type TierRejection struct {
Tier StrategyTier `json:"tier"`
Reason string `json:"reason"`
}
// Diagnostics captures which tier produced the returned chunks plus the
// chain that was attempted and any rejected tiers along the way. Useful
// for surfacing in a debug UI; not produced by the normal Split path.
type Diagnostics struct {
SelectedTier StrategyTier `json:"selected_tier"`
TierChain []StrategyTier `json:"tier_chain"`
Rejected []TierRejection `json:"rejected"`
}
// SplitWithDiagnostics is the same as Split but also returns the
// diagnostic trace (selected tier, full chain, rejection reasons). Use
// this for the chunker preview endpoint where the caller wants to know
// which tier won and why others lost.
func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) {
diag := &Diagnostics{}
if text == "" {
return nil
return nil, diag
}
cfg = ensureDefaults(cfg)
chain := resolveChain(text, cfg)
diag.TierChain = chain
totalChars := len([]rune(text))
var lastOut []Chunk
var lastTier StrategyTier
for i, tier := range chain {
out := runTier(tier, text, cfg)
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
v := ValidateChunks(out, totalChars, cfg.ChunkSize)
if v.OK {
diag.SelectedTier = tier
return out, diag
}
// Remember the legacy tier's output: we'll return it as-is below if
// every tier rejected — running SplitText again would just produce
// the same rejected result.
diag.Rejected = append(diag.Rejected, TierRejection{Tier: tier, Reason: v.Reason})
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
if tier == TierLegacy && i == len(chain)-1 {
lastOut = out
lastTier = tier
}
}
if lastOut != nil {
return lastOut
diag.SelectedTier = lastTier
return lastOut, diag
}
// Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy).
return SplitText(text, cfg)
// Defensive last-ditch fallback.
diag.SelectedTier = TierLegacy
return SplitText(text, cfg), diag
}
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
@@ -0,0 +1,58 @@
package chunker
import (
"strings"
"testing"
)
func TestSplitWithDiagnostics_LegacyStrategy_ReportsLegacyTier(t *testing.T) {
// Splittable input so the validator accepts the legacy output cleanly.
text := strings.Repeat("Hello world.\n\nNext paragraph here.\n\n", 50)
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy}
chunks, diag := SplitWithDiagnostics(text, cfg)
if len(chunks) == 0 {
t.Fatal("expected chunks")
}
if diag.SelectedTier != TierLegacy {
t.Errorf("expected SelectedTier=legacy, got %s", diag.SelectedTier)
}
if len(diag.TierChain) != 1 || diag.TierChain[0] != TierLegacy {
t.Errorf("expected single-tier chain [legacy], got %v", diag.TierChain)
}
if len(diag.Rejected) != 0 {
t.Errorf("expected no rejections for splittable input, got %v", diag.Rejected)
}
}
func TestSplitWithDiagnostics_AutoOnHeadingDoc_PicksHeading(t *testing.T) {
doc := strings.Repeat("# Top\nintro paragraph here.\n\n## Section A\nbody A here.\n\n## Section B\nbody B here.\n\n## Section C\nbody C here.\n\n", 1)
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Strategy: StrategyAuto}
_, diag := SplitWithDiagnostics(doc, cfg)
if len(diag.TierChain) == 0 {
t.Fatal("expected non-empty tier chain")
}
// Heading tier should be tried first for this doc.
if diag.TierChain[0] != TierHeading {
t.Errorf("expected heading tier first, got chain %v", diag.TierChain)
}
}
func TestSplitWithDiagnostics_EmptyText(t *testing.T) {
chunks, diag := SplitWithDiagnostics("", DefaultConfig())
if chunks != nil {
t.Errorf("expected nil chunks for empty text, got %v", chunks)
}
if diag == nil {
t.Fatal("diag must never be nil")
}
}
func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) {
text := "para one.\n\npara two.\n\npara three."
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10}
a := Split(text, cfg)
b, _ := SplitWithDiagnostics(text, cfg)
if len(a) != len(b) {
t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b))
}
}
+7
View File
@@ -161,11 +161,18 @@ func NewRouter(params RouterParams) *gin.Engine {
RegisterDataSourceRoutes(v1, params.DataSourceHandler)
RegisterWeKnoraCloudRoutes(v1, params.WeKnoraCloudHandler)
RegisterWikiPageRoutes(v1, params.WikiPageHandler)
RegisterChunkerDebugRoutes(v1)
}
return r
}
// RegisterChunkerDebugRoutes wires the read-only chunker preview endpoint
// used by the KB editor's debug panel. Stateless — uses no service deps.
func RegisterChunkerDebugRoutes(r *gin.RouterGroup) {
r.POST("/chunker/preview", handler.PreviewChunking)
}
// RegisterChunkRoutes 注册分块相关的路由
func RegisterChunkRoutes(r *gin.RouterGroup, handler *handler.ChunkHandler) {
// 分块路由组