fix(chunker): post-review fixes for preview endpoint robustness
Resolves issues from the review of be326aa..119f5e4. Each fix has a
regression test attached and overclaimed findings (parserEngineRules
defensive copy, runTier dead-code branch, init-vars architecture)
were intentionally not touched after re-evaluation.
Goroutine leak mitigation
- previewMaxChars dropped from 256k to 64k runes. The splitter does
not accept a context.Context, so when previewTimeout fires the
worker keeps running. Bounding input size keeps worst-case CPU
per request well under a second on commodity hardware. Sized so
10 concurrent timeouts don't pile up faster than they finish.
- Frontend MAX_CHARS lowered to match.
- Comment in handler explains the trade-off and points at the
follow-up: real cancellation needs the splitter to take a ctx.
Performance
- ApproxTokenCountFromRuneLen variant lets the preview handler
reuse a single rune-count per chunk for stats + size + token
estimation. Eliminates the previous triple []rune allocation per
chunk in the response loop.
- computeChunkSizeStats now takes []int (pre-computed rune lens)
instead of []chunker.Chunk; sumSq computed in float64 to avoid
the int*int overflow at l > ~46k.
Correctness / UX
- Preview panel sends strategy / token_limit / languages
unconditionally, mirroring the buildSubmitData convention so the
preview faithfully reflects what would happen on save.
- Empty-text returns a friendly 400 ("paste a sample…") instead of
gin's cryptic 'Field validation failed on the required tag'.
Tests
- TestSplit_DelegatesToSplitWithDiagnostics renamed to
TestSplit_AndDiagnostics_AgreeOnChunks (the post-audit refactor
made the original name a misnomer; the test still asserts the
right invariant under the new name).
- New TestSplitWithDiagnostics_ProfileSetForAuto and
TestSplitWithDiagnostics_ProfileNilForExplicit lock in the
profile-reuse contract that the preview endpoint depends on.
- New chunker_debug_test.go covers computeChunkSizeStats edge
cases (empty / single / varying / no-variance underflow) plus
PreviewChunking httptest scenarios (auto path, legacy strategy,
empty-text rejection, oversize rejection, chunk truncation with
full-set stats).
Doc cleanup
- runTier comment updated; the "stubbed in this scaffold" line was
obsolete since the heading and heuristic splitters shipped.
https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func init() {
|
||||
gin.SetMode(gin.TestMode)
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_Empty(t *testing.T) {
|
||||
stats := computeChunkSizeStats(nil)
|
||||
if stats.Count != 0 || stats.AvgChars != 0 || stats.MaxChars != 0 {
|
||||
t.Errorf("empty input should yield zero stats, got %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_SingleChunk(t *testing.T) {
|
||||
stats := computeChunkSizeStats([]int{500})
|
||||
if stats.Count != 1 {
|
||||
t.Errorf("count: got %d want 1", stats.Count)
|
||||
}
|
||||
if stats.AvgChars != 500 || stats.MinChars != 500 || stats.MaxChars != 500 {
|
||||
t.Errorf("single-chunk stats should all equal 500, got %+v", stats)
|
||||
}
|
||||
if stats.StddevChars != 0 {
|
||||
t.Errorf("stddev for one element should be 0, got %d", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_VaryingSizes(t *testing.T) {
|
||||
// 100, 200, 300, 400, 500 → avg 300, stddev ≈ 141
|
||||
stats := computeChunkSizeStats([]int{100, 200, 300, 400, 500})
|
||||
if stats.Count != 5 {
|
||||
t.Errorf("count: got %d want 5", stats.Count)
|
||||
}
|
||||
if stats.AvgChars != 300 {
|
||||
t.Errorf("avg: got %d want 300", stats.AvgChars)
|
||||
}
|
||||
if stats.MinChars != 100 || stats.MaxChars != 500 {
|
||||
t.Errorf("min/max: got %d/%d want 100/500", stats.MinChars, stats.MaxChars)
|
||||
}
|
||||
if stats.StddevChars < 130 || stats.StddevChars > 150 {
|
||||
t.Errorf("stddev: got %d, want ~141", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeChunkSizeStats_NoVarianceUnderflow(t *testing.T) {
|
||||
// All identical — variance must clamp to 0 not flip negative on
|
||||
// float-precision rounding.
|
||||
stats := computeChunkSizeStats([]int{1234, 1234, 1234, 1234})
|
||||
if stats.StddevChars != 0 {
|
||||
t.Errorf("identical values must yield stddev=0, got %d", stats.StddevChars)
|
||||
}
|
||||
}
|
||||
|
||||
// --- PreviewChunking httptest -------------------------------------------------
|
||||
|
||||
func newPreviewRouter() *gin.Engine {
|
||||
r := gin.New()
|
||||
r.POST("/chunker/preview", PreviewChunking)
|
||||
return r
|
||||
}
|
||||
|
||||
func postPreview(t *testing.T, body any) (*httptest.ResponseRecorder, map[string]any) {
|
||||
t.Helper()
|
||||
r := newPreviewRouter()
|
||||
buf := &bytes.Buffer{}
|
||||
if err := json.NewEncoder(buf).Encode(body); err != nil {
|
||||
t.Fatalf("encode body: %v", err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/chunker/preview", buf)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
var parsed map[string]any
|
||||
if w.Body.Len() > 0 {
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &parsed)
|
||||
}
|
||||
return w, parsed
|
||||
}
|
||||
|
||||
func TestPreviewChunking_HappyPath_AutoStrategy(t *testing.T) {
|
||||
body := PreviewChunkingRequest{
|
||||
Text: "# Top\nintro paragraph here.\n\n## Section A\nbody A.\n\n## Section B\nbody B.",
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 20,
|
||||
Separators: []string{"\n\n", "\n"},
|
||||
Strategy: "auto",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d want 200; body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if parsed["success"] != true {
|
||||
t.Fatalf("success flag missing or false: %v", parsed)
|
||||
}
|
||||
data, ok := parsed["data"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data missing: %v", parsed)
|
||||
}
|
||||
if data["selected_tier"] == "" {
|
||||
t.Errorf("selected_tier must be set, got %v", data["selected_tier"])
|
||||
}
|
||||
if _, ok := data["chunks"].([]any); !ok {
|
||||
t.Errorf("chunks must be an array, got %T", data["chunks"])
|
||||
}
|
||||
stats, ok := data["stats"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("stats must be an object, got %T", data["stats"])
|
||||
}
|
||||
if c, _ := stats["count"].(float64); c <= 0 {
|
||||
t.Errorf("stats.count should be > 0, got %v", stats["count"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_RejectsEmptyText(t *testing.T) {
|
||||
w, parsed := postPreview(t, PreviewChunkingRequest{Text: " \n\t "})
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("status: got %d want 400", w.Code)
|
||||
}
|
||||
if errStr, _ := parsed["error"].(string); !strings.Contains(errStr, "empty") {
|
||||
t.Errorf("error should mention 'empty', got %q", errStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_RejectsOversizedText(t *testing.T) {
|
||||
body := PreviewChunkingRequest{Text: strings.Repeat("a", previewMaxChars+1)}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("status: got %d want 413", w.Code)
|
||||
}
|
||||
if parsed["limit"] == nil {
|
||||
t.Errorf("response should include limit hint, got %v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_LegacyStrategy_NoProfile(t *testing.T) {
|
||||
// Auto-strategy is the only path that produces a profile inside
|
||||
// SplitWithDiagnostics. For explicit strategies the handler
|
||||
// materializes one itself so the UI always sees stats.
|
||||
body := PreviewChunkingRequest{
|
||||
Text: "para one.\n\npara two.\n\npara three.\n\npara four.",
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 100,
|
||||
ChunkOverlap: 10,
|
||||
Separators: []string{"\n\n"},
|
||||
Strategy: "legacy",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
data := parsed["data"].(map[string]any)
|
||||
if data["profile"] == nil {
|
||||
t.Error("profile should be materialized for explicit strategy too")
|
||||
}
|
||||
if string(chunker.StrategyTier(data["selected_tier"].(string))) != string(chunker.TierLegacy) {
|
||||
t.Errorf("selected_tier: got %v want %s", data["selected_tier"], chunker.TierLegacy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_ChunkTruncation(t *testing.T) {
|
||||
// Build text that produces > previewMaxChunks chunks.
|
||||
body := PreviewChunkingRequest{
|
||||
Text: strings.Repeat("x.\n\n", previewMaxChunks+50),
|
||||
ChunkingConfig: PreviewChunkingPayload{
|
||||
ChunkSize: 3,
|
||||
ChunkOverlap: 0,
|
||||
Separators: []string{"\n\n"},
|
||||
Strategy: "legacy",
|
||||
},
|
||||
}
|
||||
w, parsed := postPreview(t, body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d", w.Code)
|
||||
}
|
||||
data := parsed["data"].(map[string]any)
|
||||
chunks := data["chunks"].([]any)
|
||||
if len(chunks) > previewMaxChunks {
|
||||
t.Errorf("chunks should be truncated to ≤%d, got %d", previewMaxChunks, len(chunks))
|
||||
}
|
||||
stats := data["stats"].(map[string]any)
|
||||
if truncated, _ := stats["truncated_to"].(float64); int(truncated) <= previewMaxChunks {
|
||||
t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user