fix(chunker): QA audit — pointer DTOs, profile reuse, stats correctness

Resolves the issues surfaced by the post-Phase-3 self-review.

High severity
- Strategy / TokenLimit / Languages can now be reset. The KB-config
  handler uses pointer DTOs so the absence of a field in the payload
  means "no change" while an explicit empty string / 0 / [] clears
  the value. The frontend now sends the fields unconditionally so
  the user can revert to defaults via the clear icon — previously
  these fields were write-once.
- DocProfile is no longer computed twice in the preview endpoint.
  Diagnostics carries the profile that drove tier selection (when
  auto-strategy ran); the handler only re-profiles for explicit
  strategies that bypass the profiler.
- Chunk-size stats (avg / min / max / stddev) are now computed over
  the FULL chunk set before the response is truncated to
  previewMaxChunks. Avoids systematically misleading metrics for
  large documents.
- SplitWithDiagnostics defaults SelectedTier to TierLegacy so an
  empty-text request never leaves the diag with a blank tier string.

Medium severity
- Split() reverts to its own loop instead of delegating to
  SplitWithDiagnostics. Diagnostics struct allocation is reserved
  for callers that actually want the trace (preview endpoint).
  Matters in SplitParentChild where Split runs once per parent.
- Preview endpoint runs the splitter on a goroutine wrapped in a
  5s context.WithTimeout. CJK input at the 256k-rune ceiling
  could otherwise tie up the worker for several seconds.

Low severity
- KBChunkingSettings emits arrays via spread copies so parent-side
  mutation cannot drift the form's reactive state.
- KBChunkingDebug font-family includes a monospace fallback so the
  chunk content stays monospaced even when the TDesign CSS variable
  is missing.

Documentation
- DocProfile JSON shape is now annotated as part of the public
  preview API; field renames would be breaking.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-03 07:01:20 +00:00
committed by lyingbug
parent 6864a9f9e2
commit beb7d005e9
8 changed files with 178 additions and 74 deletions
+6 -2
View File
@@ -103,11 +103,15 @@ export interface KBModelConfigRequest {
parentChunkSize?: number
childChunkSize?: number
// Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy").
// Empty string keeps the existing backend default (legacy / no change).
// The backend uses pointer-based DTOs for these three fields:
// - undefined / not set in payload → no change on server
// - "" / 0 / [] explicitly sent → clears the value
// Send the field whenever the user has opened the editor — even
// empty values — so the user can always reset back to defaults.
strategy?: string
// Approximate token budget per chunk; 0 = char-based.
tokenLimit?: number
// Language hints for heuristic patterns. Empty = auto-detect.
// Language hints for heuristic patterns. Empty array = auto-detect.
languages?: string[]
}
multimodal: {
@@ -849,17 +849,12 @@ const buildSubmitData = () => {
enable_parent_child: formData.value.chunkingConfig.enableParentChild,
parent_chunk_size: formData.value.chunkingConfig.parentChunkSize,
child_chunk_size: formData.value.chunkingConfig.childChunkSize,
// Adaptive chunking fields. Empty/zero values are omitted on the
// backend side via omitempty; we send them through unconditionally.
...(formData.value.chunkingConfig.strategy
? { strategy: formData.value.chunkingConfig.strategy }
: {}),
...(formData.value.chunkingConfig.tokenLimit
? { token_limit: formData.value.chunkingConfig.tokenLimit }
: {}),
...(formData.value.chunkingConfig.languages?.length
? { languages: formData.value.chunkingConfig.languages }
: {}),
// Adaptive chunking fields are always sent (empty/zero values
// included) so the user can clear them — backend uses pointer DTOs
// to distinguish "not in payload" from "explicitly empty".
strategy: formData.value.chunkingConfig.strategy ?? '',
token_limit: formData.value.chunkingConfig.tokenLimit ?? 0,
languages: formData.value.chunkingConfig.languages ?? [],
...(formData.value.chunkingConfig.parserEngineRules?.length
? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules }
: {})
@@ -1044,10 +1039,12 @@ const doSubmit = async () => {
enableParentChild: data.chunking_config.enable_parent_child || false,
parentChunkSize: data.chunking_config.parent_chunk_size || 4096,
childChunkSize: data.chunking_config.child_chunk_size || 384,
// Adaptive chunking — empty / 0 / [] means "leave server default".
strategy: data.chunking_config.strategy || undefined,
tokenLimit: data.chunking_config.token_limit || undefined,
languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined
// Always send strategy / tokenLimit / languages — backend treats
// empty/0/[] as a valid clear, so we must include them in the
// payload to let users reset back to defaults.
strategy: formData.value?.chunkingConfig.strategy ?? '',
tokenLimit: formData.value?.chunkingConfig.tokenLimit ?? 0,
languages: formData.value?.chunkingConfig.languages ?? []
},
multimodal: {
enabled: !!data.vlm_config?.enabled
@@ -468,7 +468,7 @@ const tierTheme = (tier: StrategyTier) => {
color: var(--td-brand-color);
white-space: pre-wrap;
word-break: break-word;
font-family: var(--td-font-family-mono);
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
}
}
@@ -482,7 +482,7 @@ const tierTheme = (tier: StrategyTier) => {
white-space: pre-wrap;
word-break: break-word;
color: var(--td-text-color-primary);
font-family: var(--td-font-family-mono);
font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace);
}
&.truncated pre::after {
@@ -343,17 +343,20 @@ const handleTokenLimitChange = () => { emitUpdate() }
const handleLanguagesChange = () => { emitUpdate() }
const emitUpdate = () => {
// Spread arrays so the parent gets its own copy. Mutating the emitted
// arrays from outside must not leak back into our reactive state and
// cause two-way ref drift between the form and the editor model.
emit('update:config', {
chunkSize: localChunkSize.value,
chunkOverlap: localChunkOverlap.value,
separators: localSeparators.value,
separators: [...localSeparators.value],
parserEngineRules: props.config.parserEngineRules,
enableParentChild: localEnableParentChild.value,
parentChunkSize: localParentChunkSize.value,
childChunkSize: localChildChunkSize.value,
strategy: localStrategy.value,
tokenLimit: localTokenLimit.value,
languages: localLanguages.value
languages: [...localLanguages.value]
})
}
</script>
+78 -24
View File
@@ -5,8 +5,10 @@
package handler
import (
"context"
"math"
"net/http"
"time"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/logger"
@@ -19,9 +21,16 @@ import (
const previewMaxChars = 256 * 1024
// previewMaxChunks caps the number of chunks returned in a single preview
// response so the UI doesn't choke on pathological splits.
// response so the UI doesn't choke on pathological splits. Stats are
// computed over the full chunk set before truncation so the displayed
// avg/min/max/stddev stay representative.
const previewMaxChunks = 500
// previewTimeout caps how long the splitter is allowed to run for a single
// preview call. CJK input at the 256k-rune ceiling can otherwise take
// several seconds across all four tier attempts.
const previewTimeout = 5 * time.Second
// PreviewChunkingRequest is the body shape accepted by /chunker/preview.
type PreviewChunkingRequest struct {
Text string `json:"text" binding:"required"`
@@ -52,14 +61,19 @@ type PreviewChunkResult struct {
Content string `json:"content"`
}
// PreviewChunkingStats summarizes chunk-size distribution.
// PreviewChunkingStats summarizes chunk-size distribution. Computed over
// the FULL chunk set, even when the response truncates to previewMaxChunks
// items, so avg/min/max/stddev reflect the real distribution, not just
// the first N.
type PreviewChunkingStats struct {
Count int `json:"count"`
AvgChars int `json:"avg_chars"`
MinChars int `json:"min_chars"`
MaxChars int `json:"max_chars"`
StddevChars int `json:"stddev_chars"`
TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response
Count int `json:"count"`
AvgChars int `json:"avg_chars"`
MinChars int `json:"min_chars"`
MaxChars int `json:"max_chars"`
StddevChars int `json:"stddev_chars"`
// TruncatedTo, when set, is the original chunk count before the
// response was truncated to previewMaxChunks for transport.
TruncatedTo int `json:"truncated_to,omitempty"`
}
// PreviewChunkingResponse is the body returned by /chunker/preview.
@@ -77,7 +91,8 @@ type PreviewChunkingResponse struct {
// information about which tier won. Read-only: no DB writes, no embedding
// calls, no logging of the supplied text.
func PreviewChunking(c *gin.Context) {
ctx := c.Request.Context()
ctx, cancel := context.WithTimeout(c.Request.Context(), previewTimeout)
defer cancel()
var req PreviewChunkingRequest
if err := c.ShouldBindJSON(&req); err != nil {
@@ -103,23 +118,56 @@ func PreviewChunking(c *gin.Context) {
Languages: req.ChunkingConfig.Languages,
}
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
profile := chunker.ProfileDocument(req.Text)
// Run the splitter on a goroutine so we can honor the request timeout.
// The splitter is CPU-bound and doesn't accept a context — wrapping
// here is the cheapest cancellation we can offer.
type splitResult struct {
chunks []chunker.Chunk
diag *chunker.Diagnostics
}
resCh := make(chan splitResult, 1)
go func() {
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
resCh <- splitResult{chunks: chunks, diag: diag}
}()
var sr splitResult
select {
case sr = <-resCh:
case <-ctx.Done():
c.JSON(http.StatusGatewayTimeout, gin.H{
"success": false,
"error": "chunker preview timed out",
})
return
}
chunks, diag := sr.chunks, sr.diag
// Diagnostics carries the profile when auto-strategy ran; for explicit
// strategies the profile is nil and we materialize it here so the UI
// can still show document stats. Avoids the previous double-pass.
profile := diag.Profile
if profile == nil {
profile = chunker.ProfileDocument(req.Text)
}
logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks))
totalCount := len(chunks)
truncatedTo := 0
if totalCount > previewMaxChunks {
truncatedTo = totalCount
chunks = chunks[:previewMaxChunks]
}
lang := chunker.LangMixed
if len(profile.DetectedLangs) > 0 {
lang = profile.DetectedLangs[0]
}
// Compute stats over the FULL chunk set first so the metrics stay
// representative even when we trim the response to previewMaxChunks.
totalCount := len(chunks)
stats := computeChunkSizeStats(chunks, lang)
if totalCount > previewMaxChunks {
stats.TruncatedTo = totalCount
chunks = chunks[:previewMaxChunks]
}
results := make([]PreviewChunkResult, 0, len(chunks))
for _, ch := range chunks {
runeLen := len([]rune(ch.Content))
@@ -140,22 +188,26 @@ func PreviewChunking(c *gin.Context) {
Rejected: diag.Rejected,
Profile: profile,
Chunks: results,
Stats: chunkStats(results, truncatedTo),
Stats: stats,
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": resp})
}
// chunkStats computes count / avg / min / max / stddev over the chunks'
// SizeChars values. Returns zeroes for an empty slice.
func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats {
stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo}
// computeChunkSizeStats walks the full chunk slice once and returns the
// size distribution stats. Operates directly on chunker.Chunk so we don't
// need to materialize PreviewChunkResult before truncation.
//
// lang is forwarded to ApproxTokenCount only if callers extend the stats
// later — currently the result struct only tracks chars.
func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStats {
stats := PreviewChunkingStats{Count: len(chunks)}
if len(chunks) == 0 {
return stats
}
var sum, sumSq float64
minLen, maxLen := math.MaxInt32, 0
for _, ch := range chunks {
l := ch.SizeChars
l := len([]rune(ch.Content))
sum += float64(l)
sumSq += float64(l * l)
if l < minLen {
@@ -168,6 +220,8 @@ func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingSta
avg := sum / float64(len(chunks))
variance := sumSq/float64(len(chunks)) - avg*avg
if variance < 0 {
// Float precision can push the variance slightly below zero on
// near-uniform inputs; clamp so sqrt doesn't return NaN.
variance = 0
}
stats.AvgChars = int(avg + 0.5)
+17 -9
View File
@@ -102,9 +102,14 @@ type KBModelConfigRequest struct {
EnableParentChild bool `json:"enableParentChild"`
ParentChunkSize int `json:"parentChunkSize,omitempty"`
ChildChunkSize int `json:"childChunkSize,omitempty"`
Strategy string `json:"strategy,omitempty"`
TokenLimit int `json:"tokenLimit,omitempty"`
Languages []string `json:"languages,omitempty"`
// Strategy / TokenLimit / Languages use pointer types so the
// handler can distinguish "field absent in payload" (no change)
// from "field present with empty/zero value" (clear / disable).
// Without that distinction, users could set strategy="auto" once
// but never reset it back to legacy / unset.
Strategy *string `json:"strategy,omitempty"`
TokenLimit *int `json:"tokenLimit,omitempty"`
Languages *[]string `json:"languages,omitempty"`
} `json:"documentSplitting"`
// 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置)
@@ -323,14 +328,17 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
if req.DocumentSplitting.ChildChunkSize > 0 {
kb.ChunkingConfig.ChildChunkSize = req.DocumentSplitting.ChildChunkSize
}
if req.DocumentSplitting.Strategy != "" {
kb.ChunkingConfig.Strategy = req.DocumentSplitting.Strategy
// Pointer-based fields support clearing (empty string / 0 / empty slice
// is a valid "user picked default again" signal; absent in payload means
// "no change").
if req.DocumentSplitting.Strategy != nil {
kb.ChunkingConfig.Strategy = *req.DocumentSplitting.Strategy
}
if req.DocumentSplitting.TokenLimit > 0 {
kb.ChunkingConfig.TokenLimit = req.DocumentSplitting.TokenLimit
if req.DocumentSplitting.TokenLimit != nil {
kb.ChunkingConfig.TokenLimit = *req.DocumentSplitting.TokenLimit
}
if len(req.DocumentSplitting.Languages) > 0 {
kb.ChunkingConfig.Languages = req.DocumentSplitting.Languages
if req.DocumentSplitting.Languages != nil {
kb.ChunkingConfig.Languages = *req.DocumentSplitting.Languages
}
// 更新多模态配置
+4 -2
View File
@@ -10,8 +10,10 @@ import (
)
// DocProfile holds the document-level signals used to choose a chunking tier.
// JSON tags allow the profile to be surfaced via the chunker preview endpoint
// for debug UIs.
//
// The JSON shape (snake_case via struct tags) is part of the public preview
// endpoint API — keep field names stable. Internal callers should use the Go
// field names; only the preview handler relies on the wire format.
type DocProfile struct {
TotalChars int `json:"total_chars"`
TotalLines int `json:"total_lines"`
+54 -18
View File
@@ -26,9 +26,34 @@ const (
// is empty or "auto" the document profiler picks the tier. The function
// always returns a non-nil result: on tier failure the chain falls through
// to the legacy splitter, which is the original Tier 3 implementation.
//
// Hot path: avoids the Diagnostics struct allocation that
// SplitWithDiagnostics performs (matters in SplitParentChild where
// Split is called once per parent).
func Split(text string, cfg SplitterConfig) []Chunk {
chunks, _ := SplitWithDiagnostics(text, cfg)
return chunks
if text == "" {
return nil
}
cfg = ensureDefaults(cfg)
chain, _ := resolveChainWithProfile(text, cfg)
totalChars := len([]rune(text))
var lastOut []Chunk
for i, tier := range chain {
out := runTier(tier, text, cfg)
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
}
if tier == TierLegacy && i == len(chain)-1 {
lastOut = out
}
}
if lastOut != nil {
return lastOut
}
return SplitText(text, cfg)
}
// TierRejection records why a tier was rejected by the validator and the
@@ -40,26 +65,36 @@ type TierRejection struct {
}
// Diagnostics captures which tier produced the returned chunks plus the
// chain that was attempted and any rejected tiers along the way. Useful
// for surfacing in a debug UI; not produced by the normal Split path.
// chain that was attempted, any rejected tiers along the way, and the
// document profile that drove tier selection.
//
// Useful for surfacing in a debug UI; not produced by the normal Split
// path. The JSON shape is part of the public preview-endpoint API — keep
// field names stable.
type Diagnostics struct {
SelectedTier StrategyTier `json:"selected_tier"`
TierChain []StrategyTier `json:"tier_chain"`
Rejected []TierRejection `json:"rejected"`
// Profile is set when the auto strategy resolved the chain via the
// document profiler. nil when an explicit Strategy bypassed profiling.
Profile *DocProfile `json:"profile,omitempty"`
}
// SplitWithDiagnostics is the same as Split but also returns the
// diagnostic trace (selected tier, full chain, rejection reasons). Use
// this for the chunker preview endpoint where the caller wants to know
// which tier won and why others lost.
// diagnostic trace (selected tier, full chain, rejection reasons,
// profile when available). Use this for the chunker preview endpoint
// where the caller wants to know which tier won and why others lost.
func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) {
diag := &Diagnostics{}
// Default selected tier to legacy so an empty diag never carries the
// zero string — that would render as a blank tag in the debug UI.
diag := &Diagnostics{SelectedTier: TierLegacy}
if text == "" {
return nil, diag
}
cfg = ensureDefaults(cfg)
chain := resolveChain(text, cfg)
chain, profile := resolveChainWithProfile(text, cfg)
diag.TierChain = chain
diag.Profile = profile
totalChars := len([]rune(text))
var lastOut []Chunk
@@ -83,7 +118,6 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
return lastOut, diag
}
// Defensive last-ditch fallback.
diag.SelectedTier = TierLegacy
return SplitText(text, cfg), diag
}
@@ -140,25 +174,27 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
return ParentChildResult{Parents: newParents, Children: children}
}
// resolveChain returns the strategy chain to attempt. An explicit non-auto
// strategy bypasses the profiler entirely and pins to the requested tier.
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
// resolveChainWithProfile returns the strategy chain to attempt and, when
// the chain was selected by the profiler (auto strategy), the DocProfile
// that drove the selection. Profile is nil for explicit non-auto strategies
// so callers don't pay for an unused profiling pass.
func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *DocProfile) {
switch cfg.Strategy {
case StrategyHeading:
return []StrategyTier{TierHeading, TierLegacy}
return []StrategyTier{TierHeading, TierLegacy}, nil
case StrategyHeuristic:
return []StrategyTier{TierHeuristic, TierLegacy}
return []StrategyTier{TierHeuristic, TierLegacy}, nil
case StrategyRecursive:
return []StrategyTier{TierRecursive, TierLegacy}
return []StrategyTier{TierRecursive, TierLegacy}, nil
case StrategyLegacy, "":
// Empty == legacy preserves backwards compatibility with stored
// ChunkingConfig rows that pre-date the Strategy field.
return []StrategyTier{TierLegacy}
return []StrategyTier{TierLegacy}, nil
case StrategyAuto:
fallthrough
default:
profile := ProfileDocument(text)
return SelectStrategy(profile)
return SelectStrategy(profile), profile
}
}