From beb7d005e9129569261d37222d89ec49cd54d88a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 3 May 2026 07:01:20 +0000 Subject: [PATCH] =?UTF-8?q?fix(chunker):=20QA=20audit=20=E2=80=94=20pointe?= =?UTF-8?q?r=20DTOs,=20profile=20reuse,=20stats=20correctness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the issues surfaced by the post-Phase-3 self-review. High severity - Strategy / TokenLimit / Languages can now be reset. The KB-config handler uses pointer DTOs so the absence of a field in the payload means "no change" while an explicit empty string / 0 / [] clears the value. The frontend now sends the fields unconditionally so the user can revert to defaults via the clear icon — previously these fields were write-once. - DocProfile is no longer computed twice in the preview endpoint. Diagnostics carries the profile that drove tier selection (when auto-strategy ran); the handler only re-profiles for explicit strategies that bypass the profiler. - Chunk-size stats (avg / min / max / stddev) are now computed over the FULL chunk set before the response is truncated to previewMaxChunks. Avoids systematically misleading metrics for large documents. - SplitWithDiagnostics defaults SelectedTier to TierLegacy so an empty-text request never leaves the diag with a blank tier string. Medium severity - Split() reverts to its own loop instead of delegating to SplitWithDiagnostics. Diagnostics struct allocation is reserved for callers that actually want the trace (preview endpoint). Matters in SplitParentChild where Split runs once per parent. - Preview endpoint runs the splitter on a goroutine wrapped in a 5s context.WithTimeout. CJK input at the 256k-rune ceiling could otherwise tie up the worker for several seconds. Low severity - KBChunkingSettings emits arrays via spread copies so parent-side mutation cannot drift the form's reactive state. - KBChunkingDebug font-family includes a monospace fallback so the chunk content stays monospaced even when the TDesign CSS variable is missing. Documentation - DocProfile JSON shape is now annotated as part of the public preview API; field renames would be breaking. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k --- frontend/src/api/initialization/index.ts | 8 +- .../knowledge/KnowledgeBaseEditorModal.vue | 27 +++-- .../knowledge/settings/KBChunkingDebug.vue | 4 +- .../knowledge/settings/KBChunkingSettings.vue | 7 +- internal/handler/chunker_debug.go | 102 +++++++++++++----- internal/handler/initialization.go | 26 +++-- internal/infrastructure/chunker/profiler.go | 6 +- internal/infrastructure/chunker/strategy.go | 72 +++++++++---- 8 files changed, 178 insertions(+), 74 deletions(-) diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index 5c5fec4c..db7a9376 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -103,11 +103,15 @@ export interface KBModelConfigRequest { parentChunkSize?: number childChunkSize?: number // Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy"). - // Empty string keeps the existing backend default (legacy / no change). + // The backend uses pointer-based DTOs for these three fields: + // - undefined / not set in payload → no change on server + // - "" / 0 / [] explicitly sent → clears the value + // Send the field whenever the user has opened the editor — even + // empty values — so the user can always reset back to defaults. strategy?: string // Approximate token budget per chunk; 0 = char-based. tokenLimit?: number - // Language hints for heuristic patterns. Empty = auto-detect. + // Language hints for heuristic patterns. Empty array = auto-detect. languages?: string[] } multimodal: { diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 2f868190..81578465 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -849,17 +849,12 @@ const buildSubmitData = () => { enable_parent_child: formData.value.chunkingConfig.enableParentChild, parent_chunk_size: formData.value.chunkingConfig.parentChunkSize, child_chunk_size: formData.value.chunkingConfig.childChunkSize, - // Adaptive chunking fields. Empty/zero values are omitted on the - // backend side via omitempty; we send them through unconditionally. - ...(formData.value.chunkingConfig.strategy - ? { strategy: formData.value.chunkingConfig.strategy } - : {}), - ...(formData.value.chunkingConfig.tokenLimit - ? { token_limit: formData.value.chunkingConfig.tokenLimit } - : {}), - ...(formData.value.chunkingConfig.languages?.length - ? { languages: formData.value.chunkingConfig.languages } - : {}), + // Adaptive chunking fields are always sent (empty/zero values + // included) so the user can clear them — backend uses pointer DTOs + // to distinguish "not in payload" from "explicitly empty". + strategy: formData.value.chunkingConfig.strategy ?? '', + token_limit: formData.value.chunkingConfig.tokenLimit ?? 0, + languages: formData.value.chunkingConfig.languages ?? [], ...(formData.value.chunkingConfig.parserEngineRules?.length ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } : {}) @@ -1044,10 +1039,12 @@ const doSubmit = async () => { enableParentChild: data.chunking_config.enable_parent_child || false, parentChunkSize: data.chunking_config.parent_chunk_size || 4096, childChunkSize: data.chunking_config.child_chunk_size || 384, - // Adaptive chunking — empty / 0 / [] means "leave server default". - strategy: data.chunking_config.strategy || undefined, - tokenLimit: data.chunking_config.token_limit || undefined, - languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined + // Always send strategy / tokenLimit / languages — backend treats + // empty/0/[] as a valid clear, so we must include them in the + // payload to let users reset back to defaults. + strategy: formData.value?.chunkingConfig.strategy ?? '', + tokenLimit: formData.value?.chunkingConfig.tokenLimit ?? 0, + languages: formData.value?.chunkingConfig.languages ?? [] }, multimodal: { enabled: !!data.vlm_config?.enabled diff --git a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue index e563f33b..1a86c15e 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue @@ -468,7 +468,7 @@ const tierTheme = (tier: StrategyTier) => { color: var(--td-brand-color); white-space: pre-wrap; word-break: break-word; - font-family: var(--td-font-family-mono); + font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace); } } @@ -482,7 +482,7 @@ const tierTheme = (tier: StrategyTier) => { white-space: pre-wrap; word-break: break-word; color: var(--td-text-color-primary); - font-family: var(--td-font-family-mono); + font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace); } &.truncated pre::after { diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index c5492c8d..4f25307f 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -343,17 +343,20 @@ const handleTokenLimitChange = () => { emitUpdate() } const handleLanguagesChange = () => { emitUpdate() } const emitUpdate = () => { + // Spread arrays so the parent gets its own copy. Mutating the emitted + // arrays from outside must not leak back into our reactive state and + // cause two-way ref drift between the form and the editor model. emit('update:config', { chunkSize: localChunkSize.value, chunkOverlap: localChunkOverlap.value, - separators: localSeparators.value, + separators: [...localSeparators.value], parserEngineRules: props.config.parserEngineRules, enableParentChild: localEnableParentChild.value, parentChunkSize: localParentChunkSize.value, childChunkSize: localChildChunkSize.value, strategy: localStrategy.value, tokenLimit: localTokenLimit.value, - languages: localLanguages.value + languages: [...localLanguages.value] }) } diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go index e2ceaf47..213a0a70 100644 --- a/internal/handler/chunker_debug.go +++ b/internal/handler/chunker_debug.go @@ -5,8 +5,10 @@ package handler import ( + "context" "math" "net/http" + "time" "github.com/Tencent/WeKnora/internal/infrastructure/chunker" "github.com/Tencent/WeKnora/internal/logger" @@ -19,9 +21,16 @@ import ( const previewMaxChars = 256 * 1024 // previewMaxChunks caps the number of chunks returned in a single preview -// response so the UI doesn't choke on pathological splits. +// response so the UI doesn't choke on pathological splits. Stats are +// computed over the full chunk set before truncation so the displayed +// avg/min/max/stddev stay representative. const previewMaxChunks = 500 +// previewTimeout caps how long the splitter is allowed to run for a single +// preview call. CJK input at the 256k-rune ceiling can otherwise take +// several seconds across all four tier attempts. +const previewTimeout = 5 * time.Second + // PreviewChunkingRequest is the body shape accepted by /chunker/preview. type PreviewChunkingRequest struct { Text string `json:"text" binding:"required"` @@ -52,14 +61,19 @@ type PreviewChunkResult struct { Content string `json:"content"` } -// PreviewChunkingStats summarizes chunk-size distribution. +// PreviewChunkingStats summarizes chunk-size distribution. Computed over +// the FULL chunk set, even when the response truncates to previewMaxChunks +// items, so avg/min/max/stddev reflect the real distribution, not just +// the first N. type PreviewChunkingStats struct { - Count int `json:"count"` - AvgChars int `json:"avg_chars"` - MinChars int `json:"min_chars"` - MaxChars int `json:"max_chars"` - StddevChars int `json:"stddev_chars"` - TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response + Count int `json:"count"` + AvgChars int `json:"avg_chars"` + MinChars int `json:"min_chars"` + MaxChars int `json:"max_chars"` + StddevChars int `json:"stddev_chars"` + // TruncatedTo, when set, is the original chunk count before the + // response was truncated to previewMaxChunks for transport. + TruncatedTo int `json:"truncated_to,omitempty"` } // PreviewChunkingResponse is the body returned by /chunker/preview. @@ -77,7 +91,8 @@ type PreviewChunkingResponse struct { // information about which tier won. Read-only: no DB writes, no embedding // calls, no logging of the supplied text. func PreviewChunking(c *gin.Context) { - ctx := c.Request.Context() + ctx, cancel := context.WithTimeout(c.Request.Context(), previewTimeout) + defer cancel() var req PreviewChunkingRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -103,23 +118,56 @@ func PreviewChunking(c *gin.Context) { Languages: req.ChunkingConfig.Languages, } - chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) - profile := chunker.ProfileDocument(req.Text) + // Run the splitter on a goroutine so we can honor the request timeout. + // The splitter is CPU-bound and doesn't accept a context — wrapping + // here is the cheapest cancellation we can offer. + type splitResult struct { + chunks []chunker.Chunk + diag *chunker.Diagnostics + } + resCh := make(chan splitResult, 1) + go func() { + chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) + resCh <- splitResult{chunks: chunks, diag: diag} + }() + + var sr splitResult + select { + case sr = <-resCh: + case <-ctx.Done(): + c.JSON(http.StatusGatewayTimeout, gin.H{ + "success": false, + "error": "chunker preview timed out", + }) + return + } + + chunks, diag := sr.chunks, sr.diag + + // Diagnostics carries the profile when auto-strategy ran; for explicit + // strategies the profile is nil and we materialize it here so the UI + // can still show document stats. Avoids the previous double-pass. + profile := diag.Profile + if profile == nil { + profile = chunker.ProfileDocument(req.Text) + } logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks)) - totalCount := len(chunks) - truncatedTo := 0 - if totalCount > previewMaxChunks { - truncatedTo = totalCount - chunks = chunks[:previewMaxChunks] - } - lang := chunker.LangMixed if len(profile.DetectedLangs) > 0 { lang = profile.DetectedLangs[0] } + // Compute stats over the FULL chunk set first so the metrics stay + // representative even when we trim the response to previewMaxChunks. + totalCount := len(chunks) + stats := computeChunkSizeStats(chunks, lang) + if totalCount > previewMaxChunks { + stats.TruncatedTo = totalCount + chunks = chunks[:previewMaxChunks] + } + results := make([]PreviewChunkResult, 0, len(chunks)) for _, ch := range chunks { runeLen := len([]rune(ch.Content)) @@ -140,22 +188,26 @@ func PreviewChunking(c *gin.Context) { Rejected: diag.Rejected, Profile: profile, Chunks: results, - Stats: chunkStats(results, truncatedTo), + Stats: stats, } c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } -// chunkStats computes count / avg / min / max / stddev over the chunks' -// SizeChars values. Returns zeroes for an empty slice. -func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats { - stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo} +// computeChunkSizeStats walks the full chunk slice once and returns the +// size distribution stats. Operates directly on chunker.Chunk so we don't +// need to materialize PreviewChunkResult before truncation. +// +// lang is forwarded to ApproxTokenCount only if callers extend the stats +// later — currently the result struct only tracks chars. +func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStats { + stats := PreviewChunkingStats{Count: len(chunks)} if len(chunks) == 0 { return stats } var sum, sumSq float64 minLen, maxLen := math.MaxInt32, 0 for _, ch := range chunks { - l := ch.SizeChars + l := len([]rune(ch.Content)) sum += float64(l) sumSq += float64(l * l) if l < minLen { @@ -168,6 +220,8 @@ func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingSta avg := sum / float64(len(chunks)) variance := sumSq/float64(len(chunks)) - avg*avg if variance < 0 { + // Float precision can push the variance slightly below zero on + // near-uniform inputs; clamp so sqrt doesn't return NaN. variance = 0 } stats.AvgChars = int(avg + 0.5) diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 681a47c0..3e88c533 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -102,9 +102,14 @@ type KBModelConfigRequest struct { EnableParentChild bool `json:"enableParentChild"` ParentChunkSize int `json:"parentChunkSize,omitempty"` ChildChunkSize int `json:"childChunkSize,omitempty"` - Strategy string `json:"strategy,omitempty"` - TokenLimit int `json:"tokenLimit,omitempty"` - Languages []string `json:"languages,omitempty"` + // Strategy / TokenLimit / Languages use pointer types so the + // handler can distinguish "field absent in payload" (no change) + // from "field present with empty/zero value" (clear / disable). + // Without that distinction, users could set strategy="auto" once + // but never reset it back to legacy / unset. + Strategy *string `json:"strategy,omitempty"` + TokenLimit *int `json:"tokenLimit,omitempty"` + Languages *[]string `json:"languages,omitempty"` } `json:"documentSplitting"` // 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置) @@ -323,14 +328,17 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { if req.DocumentSplitting.ChildChunkSize > 0 { kb.ChunkingConfig.ChildChunkSize = req.DocumentSplitting.ChildChunkSize } - if req.DocumentSplitting.Strategy != "" { - kb.ChunkingConfig.Strategy = req.DocumentSplitting.Strategy + // Pointer-based fields support clearing (empty string / 0 / empty slice + // is a valid "user picked default again" signal; absent in payload means + // "no change"). + if req.DocumentSplitting.Strategy != nil { + kb.ChunkingConfig.Strategy = *req.DocumentSplitting.Strategy } - if req.DocumentSplitting.TokenLimit > 0 { - kb.ChunkingConfig.TokenLimit = req.DocumentSplitting.TokenLimit + if req.DocumentSplitting.TokenLimit != nil { + kb.ChunkingConfig.TokenLimit = *req.DocumentSplitting.TokenLimit } - if len(req.DocumentSplitting.Languages) > 0 { - kb.ChunkingConfig.Languages = req.DocumentSplitting.Languages + if req.DocumentSplitting.Languages != nil { + kb.ChunkingConfig.Languages = *req.DocumentSplitting.Languages } // 更新多模态配置 diff --git a/internal/infrastructure/chunker/profiler.go b/internal/infrastructure/chunker/profiler.go index 4f9e0294..6a89e336 100644 --- a/internal/infrastructure/chunker/profiler.go +++ b/internal/infrastructure/chunker/profiler.go @@ -10,8 +10,10 @@ import ( ) // DocProfile holds the document-level signals used to choose a chunking tier. -// JSON tags allow the profile to be surfaced via the chunker preview endpoint -// for debug UIs. +// +// The JSON shape (snake_case via struct tags) is part of the public preview +// endpoint API — keep field names stable. Internal callers should use the Go +// field names; only the preview handler relies on the wire format. type DocProfile struct { TotalChars int `json:"total_chars"` TotalLines int `json:"total_lines"` diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 6e5bc5cf..aaef43bb 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -26,9 +26,34 @@ const ( // is empty or "auto" the document profiler picks the tier. The function // always returns a non-nil result: on tier failure the chain falls through // to the legacy splitter, which is the original Tier 3 implementation. +// +// Hot path: avoids the Diagnostics struct allocation that +// SplitWithDiagnostics performs (matters in SplitParentChild where +// Split is called once per parent). func Split(text string, cfg SplitterConfig) []Chunk { - chunks, _ := SplitWithDiagnostics(text, cfg) - return chunks + if text == "" { + return nil + } + cfg = ensureDefaults(cfg) + chain, _ := resolveChainWithProfile(text, cfg) + totalChars := len([]rune(text)) + + var lastOut []Chunk + for i, tier := range chain { + out := runTier(tier, text, cfg) + if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { + return out + } else { + logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason) + } + if tier == TierLegacy && i == len(chain)-1 { + lastOut = out + } + } + if lastOut != nil { + return lastOut + } + return SplitText(text, cfg) } // TierRejection records why a tier was rejected by the validator and the @@ -40,26 +65,36 @@ type TierRejection struct { } // Diagnostics captures which tier produced the returned chunks plus the -// chain that was attempted and any rejected tiers along the way. Useful -// for surfacing in a debug UI; not produced by the normal Split path. +// chain that was attempted, any rejected tiers along the way, and the +// document profile that drove tier selection. +// +// Useful for surfacing in a debug UI; not produced by the normal Split +// path. The JSON shape is part of the public preview-endpoint API — keep +// field names stable. type Diagnostics struct { SelectedTier StrategyTier `json:"selected_tier"` TierChain []StrategyTier `json:"tier_chain"` Rejected []TierRejection `json:"rejected"` + // Profile is set when the auto strategy resolved the chain via the + // document profiler. nil when an explicit Strategy bypassed profiling. + Profile *DocProfile `json:"profile,omitempty"` } // SplitWithDiagnostics is the same as Split but also returns the -// diagnostic trace (selected tier, full chain, rejection reasons). Use -// this for the chunker preview endpoint where the caller wants to know -// which tier won and why others lost. +// diagnostic trace (selected tier, full chain, rejection reasons, +// profile when available). Use this for the chunker preview endpoint +// where the caller wants to know which tier won and why others lost. func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) { - diag := &Diagnostics{} + // Default selected tier to legacy so an empty diag never carries the + // zero string — that would render as a blank tag in the debug UI. + diag := &Diagnostics{SelectedTier: TierLegacy} if text == "" { return nil, diag } cfg = ensureDefaults(cfg) - chain := resolveChain(text, cfg) + chain, profile := resolveChainWithProfile(text, cfg) diag.TierChain = chain + diag.Profile = profile totalChars := len([]rune(text)) var lastOut []Chunk @@ -83,7 +118,6 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic return lastOut, diag } // Defensive last-ditch fallback. - diag.SelectedTier = TierLegacy return SplitText(text, cfg), diag } @@ -140,25 +174,27 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi return ParentChildResult{Parents: newParents, Children: children} } -// resolveChain returns the strategy chain to attempt. An explicit non-auto -// strategy bypasses the profiler entirely and pins to the requested tier. -func resolveChain(text string, cfg SplitterConfig) []StrategyTier { +// resolveChainWithProfile returns the strategy chain to attempt and, when +// the chain was selected by the profiler (auto strategy), the DocProfile +// that drove the selection. Profile is nil for explicit non-auto strategies +// so callers don't pay for an unused profiling pass. +func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *DocProfile) { switch cfg.Strategy { case StrategyHeading: - return []StrategyTier{TierHeading, TierLegacy} + return []StrategyTier{TierHeading, TierLegacy}, nil case StrategyHeuristic: - return []StrategyTier{TierHeuristic, TierLegacy} + return []StrategyTier{TierHeuristic, TierLegacy}, nil case StrategyRecursive: - return []StrategyTier{TierRecursive, TierLegacy} + return []StrategyTier{TierRecursive, TierLegacy}, nil case StrategyLegacy, "": // Empty == legacy preserves backwards compatibility with stored // ChunkingConfig rows that pre-date the Strategy field. - return []StrategyTier{TierLegacy} + return []StrategyTier{TierLegacy}, nil case StrategyAuto: fallthrough default: profile := ProfileDocument(text) - return SelectStrategy(profile) + return SelectStrategy(profile), profile } }