diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go new file mode 100644 index 00000000..e2ceaf47 --- /dev/null +++ b/internal/handler/chunker_debug.go @@ -0,0 +1,178 @@ +// Package handler — chunker_debug.go exposes a read-only preview endpoint +// that runs the adaptive chunker on supplied text without touching the DB +// or generating embeddings. Used by the KB editor's debug panel so users +// can experiment with chunking parameters before committing to a re-index. +package handler + +import ( + "math" + "net/http" + + "github.com/Tencent/WeKnora/internal/infrastructure/chunker" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/gin-gonic/gin" +) + +// previewMaxChars caps the input text size so callers can't tie up the +// server with arbitrarily large payloads. Chosen to stay well below the +// 256 KB byte limit even for ASCII (256 KB / ~1 byte/rune). +const previewMaxChars = 256 * 1024 + +// previewMaxChunks caps the number of chunks returned in a single preview +// response so the UI doesn't choke on pathological splits. +const previewMaxChunks = 500 + +// PreviewChunkingRequest is the body shape accepted by /chunker/preview. +type PreviewChunkingRequest struct { + Text string `json:"text" binding:"required"` + ChunkingConfig PreviewChunkingPayload `json:"chunking_config"` +} + +// PreviewChunkingPayload mirrors the snake_case JSON the rest of the API +// uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig +// directly because it carries a lot of unrelated fields (parser engine +// rules, parent-child sizes, etc.) that the preview path doesn't need. +type PreviewChunkingPayload struct { + ChunkSize int `json:"chunk_size"` + ChunkOverlap int `json:"chunk_overlap"` + Separators []string `json:"separators"` + Strategy string `json:"strategy"` + TokenLimit int `json:"token_limit"` + Languages []string `json:"languages"` +} + +// PreviewChunkResult describes one chunk emitted during preview. +type PreviewChunkResult struct { + Seq int `json:"seq"` + Start int `json:"start"` + End int `json:"end"` + SizeChars int `json:"size_chars"` + SizeTokensApprox int `json:"size_tokens_approx"` + ContextHeader string `json:"context_header,omitempty"` + Content string `json:"content"` +} + +// PreviewChunkingStats summarizes chunk-size distribution. +type PreviewChunkingStats struct { + Count int `json:"count"` + AvgChars int `json:"avg_chars"` + MinChars int `json:"min_chars"` + MaxChars int `json:"max_chars"` + StddevChars int `json:"stddev_chars"` + TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response +} + +// PreviewChunkingResponse is the body returned by /chunker/preview. +type PreviewChunkingResponse struct { + SelectedTier chunker.StrategyTier `json:"selected_tier"` + TierChain []chunker.StrategyTier `json:"tier_chain"` + Rejected []chunker.TierRejection `json:"rejected"` + Profile *chunker.DocProfile `json:"profile"` + Chunks []PreviewChunkResult `json:"chunks"` + Stats PreviewChunkingStats `json:"stats"` +} + +// PreviewChunking handles POST /chunker/preview. It runs the supplied text +// through the adaptive chunker and returns the chunks plus diagnostic +// information about which tier won. Read-only: no DB writes, no embedding +// calls, no logging of the supplied text. +func PreviewChunking(c *gin.Context) { + ctx := c.Request.Context() + + var req PreviewChunkingRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": "invalid request body: " + err.Error()}) + return + } + + if len([]rune(req.Text)) > previewMaxChars { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{ + "success": false, + "error": "text exceeds preview limit", + "limit": previewMaxChars, + }) + return + } + + cfg := chunker.SplitterConfig{ + ChunkSize: req.ChunkingConfig.ChunkSize, + ChunkOverlap: req.ChunkingConfig.ChunkOverlap, + Separators: req.ChunkingConfig.Separators, + Strategy: req.ChunkingConfig.Strategy, + TokenLimit: req.ChunkingConfig.TokenLimit, + Languages: req.ChunkingConfig.Languages, + } + + chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) + profile := chunker.ProfileDocument(req.Text) + + logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks)) + + totalCount := len(chunks) + truncatedTo := 0 + if totalCount > previewMaxChunks { + truncatedTo = totalCount + chunks = chunks[:previewMaxChunks] + } + + lang := chunker.LangMixed + if len(profile.DetectedLangs) > 0 { + lang = profile.DetectedLangs[0] + } + + results := make([]PreviewChunkResult, 0, len(chunks)) + for _, ch := range chunks { + runeLen := len([]rune(ch.Content)) + results = append(results, PreviewChunkResult{ + Seq: ch.Seq, + Start: ch.Start, + End: ch.End, + SizeChars: runeLen, + SizeTokensApprox: chunker.ApproxTokenCount(ch.Content, lang), + ContextHeader: ch.ContextHeader, + Content: ch.Content, + }) + } + + resp := PreviewChunkingResponse{ + SelectedTier: diag.SelectedTier, + TierChain: diag.TierChain, + Rejected: diag.Rejected, + Profile: profile, + Chunks: results, + Stats: chunkStats(results, truncatedTo), + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) +} + +// chunkStats computes count / avg / min / max / stddev over the chunks' +// SizeChars values. Returns zeroes for an empty slice. +func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats { + stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo} + if len(chunks) == 0 { + return stats + } + var sum, sumSq float64 + minLen, maxLen := math.MaxInt32, 0 + for _, ch := range chunks { + l := ch.SizeChars + sum += float64(l) + sumSq += float64(l * l) + if l < minLen { + minLen = l + } + if l > maxLen { + maxLen = l + } + } + avg := sum / float64(len(chunks)) + variance := sumSq/float64(len(chunks)) - avg*avg + if variance < 0 { + variance = 0 + } + stats.AvgChars = int(avg + 0.5) + stats.MinChars = minLen + stats.MaxChars = maxLen + stats.StddevChars = int(math.Sqrt(variance) + 0.5) + return stats +} diff --git a/internal/infrastructure/chunker/profiler.go b/internal/infrastructure/chunker/profiler.go index ea9476a9..4f9e0294 100644 --- a/internal/infrastructure/chunker/profiler.go +++ b/internal/infrastructure/chunker/profiler.go @@ -10,34 +10,36 @@ import ( ) // DocProfile holds the document-level signals used to choose a chunking tier. +// JSON tags allow the profile to be surfaced via the chunker preview endpoint +// for debug UIs. type DocProfile struct { - TotalChars int - TotalLines int - AvgLineLen float64 - StdLineLen float64 + TotalChars int `json:"total_chars"` + TotalLines int `json:"total_lines"` + AvgLineLen float64 `json:"avg_line_len"` + StdLineLen float64 `json:"std_line_len"` // Markdown structure - MdHeadingCounts map[int]int // level (1..6) → count - MdHeadingTotal int + MdHeadingCounts map[int]int `json:"md_heading_counts"` // level (1..6) → count + MdHeadingTotal int `json:"md_heading_total"` // Heuristic indicators - NumberedSectionCount int - AllCapsShortLineCount int - BlankParagraphBreaks int - FormFeedCount int - VisualSepCount int - GermanChapterCount int - EnglishChapterCount int - ChineseChapterCount int - RepeatedFooterCount int + NumberedSectionCount int `json:"numbered_section_count"` + AllCapsShortLineCount int `json:"all_caps_short_line_count"` + BlankParagraphBreaks int `json:"blank_paragraph_breaks"` + FormFeedCount int `json:"form_feed_count"` + VisualSepCount int `json:"visual_sep_count"` + GermanChapterCount int `json:"german_chapter_count"` + EnglishChapterCount int `json:"english_chapter_count"` + ChineseChapterCount int `json:"chinese_chapter_count"` + RepeatedFooterCount int `json:"repeated_footer_count"` // Content characteristics - HasTables bool - HasCode bool - CodeRatio float64 + HasTables bool `json:"has_tables"` + HasCode bool `json:"has_code"` + CodeRatio float64 `json:"code_ratio"` // Detected language hints (best-effort) - DetectedLangs []string + DetectedLangs []string `json:"detected_langs"` } // HeadingDensity returns the share of lines that are Markdown headings. diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index cf5e1bde..6e5bc5cf 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -27,34 +27,64 @@ const ( // always returns a non-nil result: on tier failure the chain falls through // to the legacy splitter, which is the original Tier 3 implementation. func Split(text string, cfg SplitterConfig) []Chunk { + chunks, _ := SplitWithDiagnostics(text, cfg) + return chunks +} + +// TierRejection records why a tier was rejected by the validator and the +// chain advanced to the next tier. Surfaced by SplitWithDiagnostics for +// the debug/preview endpoint. +type TierRejection struct { + Tier StrategyTier `json:"tier"` + Reason string `json:"reason"` +} + +// Diagnostics captures which tier produced the returned chunks plus the +// chain that was attempted and any rejected tiers along the way. Useful +// for surfacing in a debug UI; not produced by the normal Split path. +type Diagnostics struct { + SelectedTier StrategyTier `json:"selected_tier"` + TierChain []StrategyTier `json:"tier_chain"` + Rejected []TierRejection `json:"rejected"` +} + +// SplitWithDiagnostics is the same as Split but also returns the +// diagnostic trace (selected tier, full chain, rejection reasons). Use +// this for the chunker preview endpoint where the caller wants to know +// which tier won and why others lost. +func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) { + diag := &Diagnostics{} if text == "" { - return nil + return nil, diag } cfg = ensureDefaults(cfg) - chain := resolveChain(text, cfg) + diag.TierChain = chain totalChars := len([]rune(text)) var lastOut []Chunk + var lastTier StrategyTier for i, tier := range chain { out := runTier(tier, text, cfg) - if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { - return out - } else { - logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason) + v := ValidateChunks(out, totalChars, cfg.ChunkSize) + if v.OK { + diag.SelectedTier = tier + return out, diag } - // Remember the legacy tier's output: we'll return it as-is below if - // every tier rejected — running SplitText again would just produce - // the same rejected result. + diag.Rejected = append(diag.Rejected, TierRejection{Tier: tier, Reason: v.Reason}) + logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason) if tier == TierLegacy && i == len(chain)-1 { lastOut = out + lastTier = tier } } if lastOut != nil { - return lastOut + diag.SelectedTier = lastTier + return lastOut, diag } - // Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy). - return SplitText(text, cfg) + // Defensive last-ditch fallback. + diag.SelectedTier = TierLegacy + return SplitText(text, cfg), diag } // SplitParentChild is the strategy-aware analog of SplitTextParentChild. diff --git a/internal/infrastructure/chunker/strategy_diagnostics_test.go b/internal/infrastructure/chunker/strategy_diagnostics_test.go new file mode 100644 index 00000000..d9b936bf --- /dev/null +++ b/internal/infrastructure/chunker/strategy_diagnostics_test.go @@ -0,0 +1,58 @@ +package chunker + +import ( + "strings" + "testing" +) + +func TestSplitWithDiagnostics_LegacyStrategy_ReportsLegacyTier(t *testing.T) { + // Splittable input so the validator accepts the legacy output cleanly. + text := strings.Repeat("Hello world.\n\nNext paragraph here.\n\n", 50) + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy} + chunks, diag := SplitWithDiagnostics(text, cfg) + if len(chunks) == 0 { + t.Fatal("expected chunks") + } + if diag.SelectedTier != TierLegacy { + t.Errorf("expected SelectedTier=legacy, got %s", diag.SelectedTier) + } + if len(diag.TierChain) != 1 || diag.TierChain[0] != TierLegacy { + t.Errorf("expected single-tier chain [legacy], got %v", diag.TierChain) + } + if len(diag.Rejected) != 0 { + t.Errorf("expected no rejections for splittable input, got %v", diag.Rejected) + } +} + +func TestSplitWithDiagnostics_AutoOnHeadingDoc_PicksHeading(t *testing.T) { + doc := strings.Repeat("# Top\nintro paragraph here.\n\n## Section A\nbody A here.\n\n## Section B\nbody B here.\n\n## Section C\nbody C here.\n\n", 1) + cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Strategy: StrategyAuto} + _, diag := SplitWithDiagnostics(doc, cfg) + if len(diag.TierChain) == 0 { + t.Fatal("expected non-empty tier chain") + } + // Heading tier should be tried first for this doc. + if diag.TierChain[0] != TierHeading { + t.Errorf("expected heading tier first, got chain %v", diag.TierChain) + } +} + +func TestSplitWithDiagnostics_EmptyText(t *testing.T) { + chunks, diag := SplitWithDiagnostics("", DefaultConfig()) + if chunks != nil { + t.Errorf("expected nil chunks for empty text, got %v", chunks) + } + if diag == nil { + t.Fatal("diag must never be nil") + } +} + +func TestSplit_DelegatesToSplitWithDiagnostics(t *testing.T) { + text := "para one.\n\npara two.\n\npara three." + cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 10} + a := Split(text, cfg) + b, _ := SplitWithDiagnostics(text, cfg) + if len(a) != len(b) { + t.Errorf("Split and SplitWithDiagnostics disagree: %d vs %d", len(a), len(b)) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 7d321839..dbcd3737 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -161,11 +161,18 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterDataSourceRoutes(v1, params.DataSourceHandler) RegisterWeKnoraCloudRoutes(v1, params.WeKnoraCloudHandler) RegisterWikiPageRoutes(v1, params.WikiPageHandler) + RegisterChunkerDebugRoutes(v1) } return r } +// RegisterChunkerDebugRoutes wires the read-only chunker preview endpoint +// used by the KB editor's debug panel. Stateless — uses no service deps. +func RegisterChunkerDebugRoutes(r *gin.RouterGroup) { + r.POST("/chunker/preview", handler.PreviewChunking) +} + // RegisterChunkRoutes 注册分块相关的路由 func RegisterChunkRoutes(r *gin.RouterGroup, handler *handler.ChunkHandler) { // 分块路由组