fix(chunker): post-review fixes for preview endpoint robustness
Resolves issues from the review of be326aa..119f5e4. Each fix has a
regression test attached and overclaimed findings (parserEngineRules
defensive copy, runTier dead-code branch, init-vars architecture)
were intentionally not touched after re-evaluation.
Goroutine leak mitigation
- previewMaxChars dropped from 256k to 64k runes. The splitter does
not accept a context.Context, so when previewTimeout fires the
worker keeps running. Bounding input size keeps worst-case CPU
per request well under a second on commodity hardware. Sized so
10 concurrent timeouts don't pile up faster than they finish.
- Frontend MAX_CHARS lowered to match.
- Comment in handler explains the trade-off and points at the
follow-up: real cancellation needs the splitter to take a ctx.
Performance
- ApproxTokenCountFromRuneLen variant lets the preview handler
reuse a single rune-count per chunk for stats + size + token
estimation. Eliminates the previous triple []rune allocation per
chunk in the response loop.
- computeChunkSizeStats now takes []int (pre-computed rune lens)
instead of []chunker.Chunk; sumSq computed in float64 to avoid
the int*int overflow at l > ~46k.
Correctness / UX
- Preview panel sends strategy / token_limit / languages
unconditionally, mirroring the buildSubmitData convention so the
preview faithfully reflects what would happen on save.
- Empty-text returns a friendly 400 ("paste a sample…") instead of
gin's cryptic 'Field validation failed on the required tag'.
Tests
- TestSplit_DelegatesToSplitWithDiagnostics renamed to
TestSplit_AndDiagnostics_AgreeOnChunks (the post-audit refactor
made the original name a misnomer; the test still asserts the
right invariant under the new name).
- New TestSplitWithDiagnostics_ProfileSetForAuto and
TestSplitWithDiagnostics_ProfileNilForExplicit lock in the
profile-reuse contract that the preview endpoint depends on.
- New chunker_debug_test.go covers computeChunkSizeStats edge
cases (empty / single / varying / no-variance underflow) plus
PreviewChunking httptest scenarios (auto path, legacy strategy,
empty-text rejection, oversize rejection, chunk truncation with
full-set stats).
Doc cleanup
- runTier comment updated; the "stubbed in this scaffold" line was
obsolete since the heading and heuristic splitters shipped.
https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -8,17 +8,28 @@ import (
|
||||
"context"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// previewMaxChars caps the input text size so callers can't tie up the
|
||||
// server with arbitrarily large payloads. Chosen to stay well below the
|
||||
// 256 KB byte limit even for ASCII (256 KB / ~1 byte/rune).
|
||||
const previewMaxChars = 256 * 1024
|
||||
// previewMaxChars caps the input text size so a single preview request
|
||||
// cannot tie up the splitter for long. Chosen to bound worst-case CPU
|
||||
// well under the previewTimeout: at 64k runes even the heaviest tier
|
||||
// chain finishes in well under a second on commodity hardware.
|
||||
//
|
||||
// SECURITY NOTE: the splitter is CPU-bound and does NOT accept a
|
||||
// context.Context. When previewTimeout fires the handler returns to the
|
||||
// caller, but the worker goroutine keeps running until the splitter
|
||||
// finishes naturally. The 64k ceiling is the primary mitigation against
|
||||
// goroutine pile-up under repeated authenticated requests. If the
|
||||
// splitter ever gains context-awareness, the goroutine-wrapper in
|
||||
// PreviewChunking should switch to it for true cancellation.
|
||||
const previewMaxChars = 64 * 1024
|
||||
|
||||
// previewMaxChunks caps the number of chunks returned in a single preview
|
||||
// response so the UI doesn't choke on pathological splits. Stats are
|
||||
@@ -26,14 +37,15 @@ const previewMaxChars = 256 * 1024
|
||||
// avg/min/max/stddev stay representative.
|
||||
const previewMaxChunks = 500
|
||||
|
||||
// previewTimeout caps how long the splitter is allowed to run for a single
|
||||
// preview call. CJK input at the 256k-rune ceiling can otherwise take
|
||||
// several seconds across all four tier attempts.
|
||||
// previewTimeout caps how long the handler waits for the splitter
|
||||
// goroutine before returning a 504. See note above on previewMaxChars.
|
||||
const previewTimeout = 5 * time.Second
|
||||
|
||||
// PreviewChunkingRequest is the body shape accepted by /chunker/preview.
|
||||
// Text is checked manually below so we can return a friendlier error than
|
||||
// gin's default "Field validation for 'Text' failed on the 'required' tag".
|
||||
type PreviewChunkingRequest struct {
|
||||
Text string `json:"text" binding:"required"`
|
||||
Text string `json:"text"`
|
||||
ChunkingConfig PreviewChunkingPayload `json:"chunking_config"`
|
||||
}
|
||||
|
||||
@@ -100,7 +112,15 @@ func PreviewChunking(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if len([]rune(req.Text)) > previewMaxChars {
|
||||
if strings.TrimSpace(req.Text) == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"success": false,
|
||||
"error": "text is empty — paste a sample to preview chunking",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if utf8.RuneCountInString(req.Text) > previewMaxChars {
|
||||
c.JSON(http.StatusRequestEntityTooLarge, gin.H{
|
||||
"success": false,
|
||||
"error": "text exceeds preview limit",
|
||||
@@ -159,24 +179,32 @@ func PreviewChunking(c *gin.Context) {
|
||||
lang = profile.DetectedLangs[0]
|
||||
}
|
||||
|
||||
// Compute rune lengths once per chunk; reused for stats and result
|
||||
// payload below. Avoids the previous triple-pass over each chunk's
|
||||
// content (stats + result + ApproxTokenCount each rune-counted).
|
||||
runeLens := make([]int, len(chunks))
|
||||
for i, ch := range chunks {
|
||||
runeLens[i] = utf8.RuneCountInString(ch.Content)
|
||||
}
|
||||
|
||||
// Compute stats over the FULL chunk set first so the metrics stay
|
||||
// representative even when we trim the response to previewMaxChunks.
|
||||
totalCount := len(chunks)
|
||||
stats := computeChunkSizeStats(chunks, lang)
|
||||
stats := computeChunkSizeStats(runeLens)
|
||||
if totalCount > previewMaxChunks {
|
||||
stats.TruncatedTo = totalCount
|
||||
chunks = chunks[:previewMaxChunks]
|
||||
runeLens = runeLens[:previewMaxChunks]
|
||||
}
|
||||
|
||||
results := make([]PreviewChunkResult, 0, len(chunks))
|
||||
for _, ch := range chunks {
|
||||
runeLen := len([]rune(ch.Content))
|
||||
for i, ch := range chunks {
|
||||
results = append(results, PreviewChunkResult{
|
||||
Seq: ch.Seq,
|
||||
Start: ch.Start,
|
||||
End: ch.End,
|
||||
SizeChars: runeLen,
|
||||
SizeTokensApprox: chunker.ApproxTokenCount(ch.Content, lang),
|
||||
SizeChars: runeLens[i],
|
||||
SizeTokensApprox: chunker.ApproxTokenCountFromRuneLen(runeLens[i], lang),
|
||||
ContextHeader: ch.ContextHeader,
|
||||
Content: ch.Content,
|
||||
})
|
||||
@@ -193,23 +221,20 @@ func PreviewChunking(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": resp})
|
||||
}
|
||||
|
||||
// computeChunkSizeStats walks the full chunk slice once and returns the
|
||||
// size distribution stats. Operates directly on chunker.Chunk so we don't
|
||||
// need to materialize PreviewChunkResult before truncation.
|
||||
//
|
||||
// lang is forwarded to ApproxTokenCount only if callers extend the stats
|
||||
// later — currently the result struct only tracks chars.
|
||||
func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStats {
|
||||
stats := PreviewChunkingStats{Count: len(chunks)}
|
||||
if len(chunks) == 0 {
|
||||
// computeChunkSizeStats summarizes count / avg / min / max / stddev from
|
||||
// a pre-computed rune-length slice. Decoupling from chunker.Chunk lets
|
||||
// the caller compute rune lengths once and reuse them for the response
|
||||
// payload (avoids a second []rune allocation per chunk).
|
||||
func computeChunkSizeStats(runeLens []int) PreviewChunkingStats {
|
||||
stats := PreviewChunkingStats{Count: len(runeLens)}
|
||||
if len(runeLens) == 0 {
|
||||
return stats
|
||||
}
|
||||
var sum, sumSq float64
|
||||
minLen, maxLen := math.MaxInt32, 0
|
||||
for _, ch := range chunks {
|
||||
l := len([]rune(ch.Content))
|
||||
for _, l := range runeLens {
|
||||
sum += float64(l)
|
||||
sumSq += float64(l * l)
|
||||
sumSq += float64(l) * float64(l)
|
||||
if l < minLen {
|
||||
minLen = l
|
||||
}
|
||||
@@ -217,8 +242,8 @@ func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStat
|
||||
maxLen = l
|
||||
}
|
||||
}
|
||||
avg := sum / float64(len(chunks))
|
||||
variance := sumSq/float64(len(chunks)) - avg*avg
|
||||
avg := sum / float64(len(runeLens))
|
||||
variance := sumSq/float64(len(runeLens)) - avg*avg
|
||||
if variance < 0 {
|
||||
// Float precision can push the variance slightly below zero on
|
||||
// near-uniform inputs; clamp so sqrt doesn't return NaN.
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func init() {
|
||||
gin.SetMode(gin.TestMode)
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_Empty(t *testing.T) {
|
||||
stats := computeChunkSizeStats(nil)
|
||||
if stats.Count != 0 || stats.AvgChars != 0 || stats.MaxChars != 0 {
|
||||
t.Errorf("empty input should yield zero stats, got %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_SingleChunk(t *testing.T) {
|
||||
stats := computeChunkSizeStats([]int{500})
|
||||
if stats.Count != 1 {
|
||||
t.Errorf("count: got %d want 1", stats.Count)
|
||||
}
|
||||
if stats.AvgChars != 500 || stats.MinChars != 500 || stats.MaxChars != 500 {
|
||||
t.Errorf("single-chunk stats should all equal 500, got %+v", stats)
|
||||
}
|
||||
if stats.StddevChars != 0 {
|
||||
t.Errorf("stddev for one element should be 0, got %d", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_VaryingSizes(t *testing.T) {
|
||||
// 100, 200, 300, 400, 500 → avg 300, stddev ≈ 141
|
||||
stats := computeChunkSizeStats([]int{100, 200, 300, 400, 500})
|
||||
if stats.Count != 5 {
|
||||
t.Errorf("count: got %d want 5", stats.Count)
|
||||
}
|
||||
if stats.AvgChars != 300 {
|
||||
t.Errorf("avg: got %d want 300", stats.AvgChars)
|
||||
}
|
||||
if stats.MinChars != 100 || stats.MaxChars != 500 {
|
||||
t.Errorf("min/max: got %d/%d want 100/500", stats.MinChars, stats.MaxChars)
|
||||
}
|
||||
if stats.StddevChars < 130 || stats.StddevChars > 150 {
|
||||
t.Errorf("stddev: got %d, want ~141", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_NoVarianceUnderflow(t *testing.T) {
|
||||
// All identical — variance must clamp to 0 not flip negative on
|
||||
// float-precision rounding.
|
||||
stats := computeChunkSizeStats([]int{1234, 1234, 1234, 1234})
|
||||
if stats.StddevChars != 0 {
|
||||
t.Errorf("identical values must yield stddev=0, got %d", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
// --- PreviewChunking httptest -------------------------------------------------
|
||||
|
||||
func newPreviewRouter() *gin.Engine {
|
||||
r := gin.New()
|
||||
r.POST("/chunker/preview", PreviewChunking)
|
||||
return r
|
||||
}
|
||||
|
||||
func postPreview(t *testing.T, body any) (*httptest.ResponseRecorder, map[string]any) {
|
||||
t.Helper()
|
||||
r := newPreviewRouter()
|
||||
buf := &bytes.Buffer{}
|
||||
if err := json.NewEncoder(buf).Encode(body); err != nil {
|
||||
t.Fatalf("encode body: %v", err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/chunker/preview", buf)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
var parsed map[string]any
|
||||
if w.Body.Len() > 0 {
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &parsed)
|
||||
}
|
||||
return w, parsed
|
||||
}
|
||||
|
||||
func TestPreviewChunking_HappyPath_AutoStrategy(t *testing.T) {
|
||||
body := PreviewChunkingRequest{
|
||||
Text: "# Top\nintro paragraph here.\n\n## Section A\nbody A.\n\n## Section B\nbody B.",
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 20,
|
||||
Separators: []string{"\n\n", "\n"},
|
||||
Strategy: "auto",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d want 200; body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if parsed["success"] != true {
|
||||
t.Fatalf("success flag missing or false: %v", parsed)
|
||||
}
|
||||
data, ok := parsed["data"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data missing: %v", parsed)
|
||||
}
|
||||
if data["selected_tier"] == "" {
|
||||
t.Errorf("selected_tier must be set, got %v", data["selected_tier"])
|
||||
}
|
||||
if _, ok := data["chunks"].([]any); !ok {
|
||||
t.Errorf("chunks must be an array, got %T", data["chunks"])
|
||||
}
|
||||
stats, ok := data["stats"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("stats must be an object, got %T", data["stats"])
|
||||
}
|
||||
if c, _ := stats["count"].(float64); c <= 0 {
|
||||
t.Errorf("stats.count should be > 0, got %v", stats["count"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_RejectsEmptyText(t *testing.T) {
|
||||
w, parsed := postPreview(t, PreviewChunkingRequest{Text: " \n\t "})
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("status: got %d want 400", w.Code)
|
||||
}
|
||||
if errStr, _ := parsed["error"].(string); !strings.Contains(errStr, "empty") {
|
||||
t.Errorf("error should mention 'empty', got %q", errStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_RejectsOversizedText(t *testing.T) {
|
||||
body := PreviewChunkingRequest{Text: strings.Repeat("a", previewMaxChars+1)}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("status: got %d want 413", w.Code)
|
||||
}
|
||||
if parsed["limit"] == nil {
|
||||
t.Errorf("response should include limit hint, got %v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_LegacyStrategy_NoProfile(t *testing.T) {
|
||||
// Auto-strategy is the only path that produces a profile inside
|
||||
// SplitWithDiagnostics. For explicit strategies the handler
|
||||
// materializes one itself so the UI always sees stats.
|
||||
body := PreviewChunkingRequest{
|
||||
Text: "para one.\n\npara two.\n\npara three.\n\npara four.",
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 100,
|
||||
ChunkOverlap: 10,
|
||||
Separators: []string{"\n\n"},
|
||||
Strategy: "legacy",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
data := parsed["data"].(map[string]any)
|
||||
if data["profile"] == nil {
|
||||
t.Error("profile should be materialized for explicit strategy too")
|
||||
}
|
||||
if string(chunker.StrategyTier(data["selected_tier"].(string))) != string(chunker.TierLegacy) {
|
||||
t.Errorf("selected_tier: got %v want %s", data["selected_tier"], chunker.TierLegacy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_ChunkTruncation(t *testing.T) {
|
||||
// Build text that produces > previewMaxChunks chunks.
|
||||
body := PreviewChunkingRequest{
|
||||
Text: strings.Repeat("x.\n\n", previewMaxChunks+50),
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 3,
|
||||
ChunkOverlap: 0,
|
||||
Separators: []string{"\n\n"},
|
||||
Strategy: "legacy",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d", w.Code)
|
||||
}
|
||||
data := parsed["data"].(map[string]any)
|
||||
chunks := data["chunks"].([]any)
|
||||
if len(chunks) > previewMaxChunks {
|
||||
t.Errorf("chunks should be truncated to ≤%d, got %d", previewMaxChunks, len(chunks))
|
||||
}
|
||||
stats := data["stats"].(map[string]any)
|
||||
if truncated, _ := stats["truncated_to"].(float64); int(truncated) <= previewMaxChunks {
|
||||
t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user