feat(handler): add chunker preview endpoint for debugging
Adds POST /api/v1/chunker/preview — a read-only endpoint that runs the adaptive chunker on supplied text and returns the chunks plus diagnostic info (which tier won, which were rejected and why, the DocProfile that the selector saw). Used by the upcoming KB editor debug panel so users can experiment with chunking parameters without re-uploading documents. Backend pieces: - chunker.SplitWithDiagnostics(text, cfg) returns chunks plus a Diagnostics struct (SelectedTier, TierChain, Rejected). Split() now delegates here. - chunker.DocProfile gets JSON tags so it serializes cleanly. - handler.PreviewChunking caps input at 256k chars and chunks at 500 per response, computes a size-distribution summary, and never touches DB or embedding APIs. Auth: lives under the v1 group so the standard JWT middleware applies. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
// Package handler — chunker_debug.go exposes a read-only preview endpoint
|
||||
// that runs the adaptive chunker on supplied text without touching the DB
|
||||
// or generating embeddings. Used by the KB editor's debug panel so users
|
||||
// can experiment with chunking parameters before committing to a re-index.
|
||||
package handler
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// previewMaxChars caps the input text size so callers can't tie up the
|
||||
// server with arbitrarily large payloads. Chosen to stay well below the
|
||||
// 256 KB byte limit even for ASCII (256 KB / ~1 byte/rune).
|
||||
const previewMaxChars = 256 * 1024
|
||||
|
||||
// previewMaxChunks caps the number of chunks returned in a single preview
|
||||
// response so the UI doesn't choke on pathological splits.
|
||||
const previewMaxChunks = 500
|
||||
|
||||
// PreviewChunkingRequest is the body shape accepted by /chunker/preview.
|
||||
type PreviewChunkingRequest struct {
|
||||
Text string `json:"text" binding:"required"`
|
||||
ChunkingConfig PreviewChunkingPayload `json:"chunking_config"`
|
||||
}
|
||||
|
||||
// PreviewChunkingPayload mirrors the snake_case JSON the rest of the API
|
||||
// uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig
|
||||
// directly because it carries a lot of unrelated fields (parser engine
|
||||
// rules, parent-child sizes, etc.) that the preview path doesn't need.
|
||||
type PreviewChunkingPayload struct {
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
Separators []string `json:"separators"`
|
||||
Strategy string `json:"strategy"`
|
||||
TokenLimit int `json:"token_limit"`
|
||||
Languages []string `json:"languages"`
|
||||
}
|
||||
|
||||
// PreviewChunkResult describes one chunk emitted during preview.
|
||||
type PreviewChunkResult struct {
|
||||
Seq int `json:"seq"`
|
||||
Start int `json:"start"`
|
||||
End int `json:"end"`
|
||||
SizeChars int `json:"size_chars"`
|
||||
SizeTokensApprox int `json:"size_tokens_approx"`
|
||||
ContextHeader string `json:"context_header,omitempty"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// PreviewChunkingStats summarizes chunk-size distribution.
|
||||
type PreviewChunkingStats struct {
|
||||
Count int `json:"count"`
|
||||
AvgChars int `json:"avg_chars"`
|
||||
MinChars int `json:"min_chars"`
|
||||
MaxChars int `json:"max_chars"`
|
||||
StddevChars int `json:"stddev_chars"`
|
||||
TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response
|
||||
}
|
||||
|
||||
// PreviewChunkingResponse is the body returned by /chunker/preview.
|
||||
type PreviewChunkingResponse struct {
|
||||
SelectedTier chunker.StrategyTier `json:"selected_tier"`
|
||||
TierChain []chunker.StrategyTier `json:"tier_chain"`
|
||||
Rejected []chunker.TierRejection `json:"rejected"`
|
||||
Profile *chunker.DocProfile `json:"profile"`
|
||||
Chunks []PreviewChunkResult `json:"chunks"`
|
||||
Stats PreviewChunkingStats `json:"stats"`
|
||||
}
|
||||
|
||||
// PreviewChunking handles POST /chunker/preview. It runs the supplied text
|
||||
// through the adaptive chunker and returns the chunks plus diagnostic
|
||||
// information about which tier won. Read-only: no DB writes, no embedding
|
||||
// calls, no logging of the supplied text.
|
||||
func PreviewChunking(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req PreviewChunkingRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": "invalid request body: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if len([]rune(req.Text)) > previewMaxChars {
|
||||
c.JSON(http.StatusRequestEntityTooLarge, gin.H{
|
||||
"success": false,
|
||||
"error": "text exceeds preview limit",
|
||||
"limit": previewMaxChars,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
cfg := chunker.SplitterConfig{
|
||||
ChunkSize: req.ChunkingConfig.ChunkSize,
|
||||
ChunkOverlap: req.ChunkingConfig.ChunkOverlap,
|
||||
Separators: req.ChunkingConfig.Separators,
|
||||
Strategy: req.ChunkingConfig.Strategy,
|
||||
TokenLimit: req.ChunkingConfig.TokenLimit,
|
||||
Languages: req.ChunkingConfig.Languages,
|
||||
}
|
||||
|
||||
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
|
||||
profile := chunker.ProfileDocument(req.Text)
|
||||
|
||||
logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks))
|
||||
|
||||
totalCount := len(chunks)
|
||||
truncatedTo := 0
|
||||
if totalCount > previewMaxChunks {
|
||||
truncatedTo = totalCount
|
||||
chunks = chunks[:previewMaxChunks]
|
||||
}
|
||||
|
||||
lang := chunker.LangMixed
|
||||
if len(profile.DetectedLangs) > 0 {
|
||||
lang = profile.DetectedLangs[0]
|
||||
}
|
||||
|
||||
results := make([]PreviewChunkResult, 0, len(chunks))
|
||||
for _, ch := range chunks {
|
||||
runeLen := len([]rune(ch.Content))
|
||||
results = append(results, PreviewChunkResult{
|
||||
Seq: ch.Seq,
|
||||
Start: ch.Start,
|
||||
End: ch.End,
|
||||
SizeChars: runeLen,
|
||||
SizeTokensApprox: chunker.ApproxTokenCount(ch.Content, lang),
|
||||
ContextHeader: ch.ContextHeader,
|
||||
Content: ch.Content,
|
||||
})
|
||||
}
|
||||
|
||||
resp := PreviewChunkingResponse{
|
||||
SelectedTier: diag.SelectedTier,
|
||||
TierChain: diag.TierChain,
|
||||
Rejected: diag.Rejected,
|
||||
Profile: profile,
|
||||
Chunks: results,
|
||||
Stats: chunkStats(results, truncatedTo),
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": resp})
|
||||
}
|
||||
|
||||
// chunkStats computes count / avg / min / max / stddev over the chunks'
|
||||
// SizeChars values. Returns zeroes for an empty slice.
|
||||
func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats {
|
||||
stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo}
|
||||
if len(chunks) == 0 {
|
||||
return stats
|
||||
}
|
||||
var sum, sumSq float64
|
||||
minLen, maxLen := math.MaxInt32, 0
|
||||
for _, ch := range chunks {
|
||||
l := ch.SizeChars
|
||||
sum += float64(l)
|
||||
sumSq += float64(l * l)
|
||||
if l < minLen {
|
||||
minLen = l
|
||||
}
|
||||
if l > maxLen {
|
||||
maxLen = l
|
||||
}
|
||||
}
|
||||
avg := sum / float64(len(chunks))
|
||||
variance := sumSq/float64(len(chunks)) - avg*avg
|
||||
if variance < 0 {
|
||||
variance = 0
|
||||
}
|
||||
stats.AvgChars = int(avg + 0.5)
|
||||
stats.MinChars = minLen
|
||||
stats.MaxChars = maxLen
|
||||
stats.StddevChars = int(math.Sqrt(variance) + 0.5)
|
||||
return stats
|
||||
}
|
||||
@@ -10,34 +10,36 @@ import (
|
||||
)
|
||||
|
||||
// DocProfile holds the document-level signals used to choose a chunking tier.
|
||||
// JSON tags allow the profile to be surfaced via the chunker preview endpoint
|
||||
// for debug UIs.
|
||||
type DocProfile struct {
|
||||
TotalChars int
|
||||
TotalLines int
|
||||
AvgLineLen float64
|
||||
StdLineLen float64
|
||||
TotalChars int `json:"total_chars"`
|
||||
TotalLines int `json:"total_lines"`
|
||||
AvgLineLen float64 `json:"avg_line_len"`
|
||||
StdLineLen float64 `json:"std_line_len"`
|
||||
|
||||
// Markdown structure
|
||||
MdHeadingCounts map[int]int // level (1..6) → count
|
||||
MdHeadingTotal int
|
||||
MdHeadingCounts map[int]int `json:"md_heading_counts"` // level (1..6) → count
|
||||
MdHeadingTotal int `json:"md_heading_total"`
|
||||
|
||||
// Heuristic indicators
|
||||
NumberedSectionCount int
|
||||
AllCapsShortLineCount int
|
||||
BlankParagraphBreaks int
|
||||
FormFeedCount int
|
||||
VisualSepCount int
|
||||
GermanChapterCount int
|
||||
EnglishChapterCount int
|
||||
ChineseChapterCount int
|
||||
RepeatedFooterCount int
|
||||
NumberedSectionCount int `json:"numbered_section_count"`
|
||||
AllCapsShortLineCount int `json:"all_caps_short_line_count"`
|
||||
BlankParagraphBreaks int `json:"blank_paragraph_breaks"`
|
||||
FormFeedCount int `json:"form_feed_count"`
|
||||
VisualSepCount int `json:"visual_sep_count"`
|
||||
GermanChapterCount int `json:"german_chapter_count"`
|
||||
EnglishChapterCount int `json:"english_chapter_count"`
|
||||
ChineseChapterCount int `json:"chinese_chapter_count"`
|
||||
RepeatedFooterCount int `json:"repeated_footer_count"`
|
||||
|
||||
// Content characteristics
|
||||
HasTables bool
|
||||
HasCode bool
|
||||
CodeRatio float64
|
||||
HasTables bool `json:"has_tables"`
|
||||
HasCode bool `json:"has_code"`
|
||||
CodeRatio float64 `json:"code_ratio"`
|
||||
|
||||
// Detected language hints (best-effort)
|
||||
DetectedLangs []string
|
||||
DetectedLangs []string `json:"detected_langs"`
|
||||
}
|
||||
|
||||
// HeadingDensity returns the share of lines that are Markdown headings.
|
||||
|
||||
@@ -27,34 +27,64 @@ const (
|
||||
// always returns a non-nil result: on tier failure the chain falls through
|
||||
// to the legacy splitter, which is the original Tier 3 implementation.
|
||||
func Split(text string, cfg SplitterConfig) []Chunk {
|
||||
chunks, _ := SplitWithDiagnostics(text, cfg)
|
||||
return chunks
|
||||
}
|
||||
|
||||
// TierRejection records why a tier was rejected by the validator and the
|
||||
// chain advanced to the next tier. Surfaced by SplitWithDiagnostics for
|
||||
// the debug/preview endpoint.
|
||||
type TierRejection struct {
|
||||
Tier StrategyTier `json:"tier"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// Diagnostics captures which tier produced the returned chunks plus the
|
||||
// chain that was attempted and any rejected tiers along the way. Useful
|
||||
// for surfacing in a debug UI; not produced by the normal Split path.
|
||||
type Diagnostics struct {
|
||||
SelectedTier StrategyTier `json:"selected_tier"`
|
||||
TierChain []StrategyTier `json:"tier_chain"`
|
||||
Rejected []TierRejection `json:"rejected"`
|
||||
}
|
||||
|
||||
// SplitWithDiagnostics is the same as Split but also returns the
|
||||
// diagnostic trace (selected tier, full chain, rejection reasons). Use
|
||||
// this for the chunker preview endpoint where the caller wants to know
|
||||
// which tier won and why others lost.
|
||||
func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) {
|
||||
diag := &Diagnostics{}
|
||||
if text == "" {
|
||||
return nil
|
||||
return nil, diag
|
||||
}
|
||||
cfg = ensureDefaults(cfg)
|
||||
|
||||
chain := resolveChain(text, cfg)
|
||||
diag.TierChain = chain
|
||||
totalChars := len([]rune(text))
|
||||
|
||||
var lastOut []Chunk
|
||||
var lastTier StrategyTier
|
||||
for i, tier := range chain {
|
||||
out := runTier(tier, text, cfg)
|
||||
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
||||
return out
|
||||
} else {
|
||||
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
v := ValidateChunks(out, totalChars, cfg.ChunkSize)
|
||||
if v.OK {
|
||||
diag.SelectedTier = tier
|
||||
return out, diag
|
||||
}
|
||||
// Remember the legacy tier's output: we'll return it as-is below if
|
||||
// every tier rejected — running SplitText again would just produce
|
||||
// the same rejected result.
|
||||
diag.Rejected = append(diag.Rejected, TierRejection{Tier: tier, Reason: v.Reason})
|
||||
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
if tier == TierLegacy && i == len(chain)-1 {
|
||||
lastOut = out
|
||||
lastTier = tier
|
||||
}
|
||||
}
|
||||
if lastOut != nil {
|
||||
return lastOut
|
||||
diag.SelectedTier = lastTier
|
||||
return lastOut, diag
|
||||
}
|
||||
// Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy).
|
||||
return SplitText(text, cfg)
|
||||
// Defensive last-ditch fallback.
|
||||
diag.SelectedTier = TierLegacy
|
||||
return SplitText(text, cfg), diag
|
||||
}
|
||||
|
||||
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitWithDiagnostics_LegacyStrategy_ReportsLegacyTier(t *testing.T) {
|
||||
// Splittable input so the validator accepts the legacy output cleanly.
|
||||
text := strings.Repeat("Hello world.\n\nNext paragraph here.\n\n", 50)
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy}
|
||||
chunks, diag := SplitWithDiagnostics(text, cfg)
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("expected chunks")
|
||||
}
|
||||
if diag.SelectedTier != TierLegacy {
|
||||
t.Errorf("expected SelectedTier=legacy, got %s", diag.SelectedTier)
|
||||
}
|
||||
if len(diag.TierChain) != 1 || diag.TierChain[0] != TierLegacy {
|
||||
t.Errorf("expected single-tier chain [legacy], got %v", diag.TierChain)
|
||||
}
|
||||
if len(diag.Rejected) != 0 {
|
||||
t.Errorf("expected no rejections for splittable input, got %v", diag.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitWithDiagnostics_AutoOnHeadingDoc_PicksHeading(t *testing.T) {
|
||||
doc := strings.Repeat("# Top\nintro paragraph here.\n\n## Section A\nbody A here.\n\n## Section B\nbody B here.\n\n## Section C\nbody C here.\n\n", 1)
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Strategy: StrategyAuto}
|
||||
_, diag := SplitWithDiagnostics(doc, cfg)
|
||||
if len(diag.TierChain) == 0 {
|
||||
t.Fatal("expected non-empty tier chain")
|
||||
}
|
||||
// Heading tier should be tried first for this doc.
|
||||
if diag.TierChain[0] != TierHeading {
|
||||
t.Errorf("expected heading tier first, got chain %v", diag.TierChain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitWithDiagnostics_EmptyText(t *testing.T) {
|
||||
chunks, diag := SplitWithDiagnostics("", DefaultConfig())
|
||||
if chunks != nil {
|
||||
t.Errorf("expected nil chunks for empty text, got %v", chunks)
|
||||
}
|
||||
if diag == nil {
|
||||
t.Fatal("diag must never be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) {
|
||||
text := "para one.\n\npara two.\n\npara three."
|
||||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10}
|
||||
a := Split(text, cfg)
|
||||
b, _ := SplitWithDiagnostics(text, cfg)
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b))
|
||||
}
|
||||
}
|
||||
@@ -161,11 +161,18 @@ func NewRouter(params RouterParams) *gin.Engine {
|
||||
RegisterDataSourceRoutes(v1, params.DataSourceHandler)
|
||||
RegisterWeKnoraCloudRoutes(v1, params.WeKnoraCloudHandler)
|
||||
RegisterWikiPageRoutes(v1, params.WikiPageHandler)
|
||||
RegisterChunkerDebugRoutes(v1)
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// RegisterChunkerDebugRoutes wires the read-only chunker preview endpoint
|
||||
// used by the KB editor's debug panel. Stateless — uses no service deps.
|
||||
func RegisterChunkerDebugRoutes(r *gin.RouterGroup) {
|
||||
r.POST("/chunker/preview", handler.PreviewChunking)
|
||||
}
|
||||
|
||||
// RegisterChunkRoutes 注册分块相关的路由
|
||||
func RegisterChunkRoutes(r *gin.RouterGroup, handler *handler.ChunkHandler) {
|
||||
// 分块路由组
|
||||
|
||||
Reference in New Issue
Block a user