diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index 5c5fec4c..db7a9376 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -103,11 +103,15 @@ export interface KBModelConfigRequest { parentChunkSize?: number childChunkSize?: number // Adaptive chunking strategy ("auto" | "heading" | "heuristic" | "legacy"). - // Empty string keeps the existing backend default (legacy / no change). + // The backend uses pointer-based DTOs for these three fields: + // - undefined / not set in payload → no change on server + // - "" / 0 / [] explicitly sent → clears the value + // Send the field whenever the user has opened the editor — even + // empty values — so the user can always reset back to defaults. strategy?: string // Approximate token budget per chunk; 0 = char-based. tokenLimit?: number - // Language hints for heuristic patterns. Empty = auto-detect. + // Language hints for heuristic patterns. Empty array = auto-detect. languages?: string[] } multimodal: { diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 2f868190..81578465 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -849,17 +849,12 @@ const buildSubmitData = () => { enable_parent_child: formData.value.chunkingConfig.enableParentChild, parent_chunk_size: formData.value.chunkingConfig.parentChunkSize, child_chunk_size: formData.value.chunkingConfig.childChunkSize, - // Adaptive chunking fields. Empty/zero values are omitted on the - // backend side via omitempty; we send them through unconditionally. - ...(formData.value.chunkingConfig.strategy - ? { strategy: formData.value.chunkingConfig.strategy } - : {}), - ...(formData.value.chunkingConfig.tokenLimit - ? { token_limit: formData.value.chunkingConfig.tokenLimit } - : {}), - ...(formData.value.chunkingConfig.languages?.length - ? { languages: formData.value.chunkingConfig.languages } - : {}), + // Adaptive chunking fields are always sent (empty/zero values + // included) so the user can clear them — backend uses pointer DTOs + // to distinguish "not in payload" from "explicitly empty". + strategy: formData.value.chunkingConfig.strategy ?? '', + token_limit: formData.value.chunkingConfig.tokenLimit ?? 0, + languages: formData.value.chunkingConfig.languages ?? [], ...(formData.value.chunkingConfig.parserEngineRules?.length ? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules } : {}) @@ -1044,10 +1039,12 @@ const doSubmit = async () => { enableParentChild: data.chunking_config.enable_parent_child || false, parentChunkSize: data.chunking_config.parent_chunk_size || 4096, childChunkSize: data.chunking_config.child_chunk_size || 384, - // Adaptive chunking — empty / 0 / [] means "leave server default". - strategy: data.chunking_config.strategy || undefined, - tokenLimit: data.chunking_config.token_limit || undefined, - languages: data.chunking_config.languages?.length ? data.chunking_config.languages : undefined + // Always send strategy / tokenLimit / languages — backend treats + // empty/0/[] as a valid clear, so we must include them in the + // payload to let users reset back to defaults. + strategy: formData.value?.chunkingConfig.strategy ?? '', + tokenLimit: formData.value?.chunkingConfig.tokenLimit ?? 0, + languages: formData.value?.chunkingConfig.languages ?? [] }, multimodal: { enabled: !!data.vlm_config?.enabled diff --git a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue index e563f33b..1a86c15e 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue @@ -468,7 +468,7 @@ const tierTheme = (tier: StrategyTier) => { color: var(--td-brand-color); white-space: pre-wrap; word-break: break-word; - font-family: var(--td-font-family-mono); + font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace); } } @@ -482,7 +482,7 @@ const tierTheme = (tier: StrategyTier) => { white-space: pre-wrap; word-break: break-word; color: var(--td-text-color-primary); - font-family: var(--td-font-family-mono); + font-family: var(--td-font-family-mono, ui-monospace, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace); } &.truncated pre::after { diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index c5492c8d..4f25307f 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -343,17 +343,20 @@ const handleTokenLimitChange = () => { emitUpdate() } const handleLanguagesChange = () => { emitUpdate() } const emitUpdate = () => { + // Spread arrays so the parent gets its own copy. Mutating the emitted + // arrays from outside must not leak back into our reactive state and + // cause two-way ref drift between the form and the editor model. emit('update:config', { chunkSize: localChunkSize.value, chunkOverlap: localChunkOverlap.value, - separators: localSeparators.value, + separators: [...localSeparators.value], parserEngineRules: props.config.parserEngineRules, enableParentChild: localEnableParentChild.value, parentChunkSize: localParentChunkSize.value, childChunkSize: localChildChunkSize.value, strategy: localStrategy.value, tokenLimit: localTokenLimit.value, - languages: localLanguages.value + languages: [...localLanguages.value] }) } diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go index e2ceaf47..213a0a70 100644 --- a/internal/handler/chunker_debug.go +++ b/internal/handler/chunker_debug.go @@ -5,8 +5,10 @@ package handler import ( + "context" "math" "net/http" + "time" "github.com/Tencent/WeKnora/internal/infrastructure/chunker" "github.com/Tencent/WeKnora/internal/logger" @@ -19,9 +21,16 @@ import ( const previewMaxChars = 256 * 1024 // previewMaxChunks caps the number of chunks returned in a single preview -// response so the UI doesn't choke on pathological splits. +// response so the UI doesn't choke on pathological splits. Stats are +// computed over the full chunk set before truncation so the displayed +// avg/min/max/stddev stay representative. const previewMaxChunks = 500 +// previewTimeout caps how long the splitter is allowed to run for a single +// preview call. CJK input at the 256k-rune ceiling can otherwise take +// several seconds across all four tier attempts. +const previewTimeout = 5 * time.Second + // PreviewChunkingRequest is the body shape accepted by /chunker/preview. type PreviewChunkingRequest struct { Text string `json:"text" binding:"required"` @@ -52,14 +61,19 @@ type PreviewChunkResult struct { Content string `json:"content"` } -// PreviewChunkingStats summarizes chunk-size distribution. +// PreviewChunkingStats summarizes chunk-size distribution. Computed over +// the FULL chunk set, even when the response truncates to previewMaxChunks +// items, so avg/min/max/stddev reflect the real distribution, not just +// the first N. type PreviewChunkingStats struct { - Count int `json:"count"` - AvgChars int `json:"avg_chars"` - MinChars int `json:"min_chars"` - MaxChars int `json:"max_chars"` - StddevChars int `json:"stddev_chars"` - TruncatedTo int `json:"truncated_to,omitempty"` // set when chunks were truncated for the response + Count int `json:"count"` + AvgChars int `json:"avg_chars"` + MinChars int `json:"min_chars"` + MaxChars int `json:"max_chars"` + StddevChars int `json:"stddev_chars"` + // TruncatedTo, when set, is the original chunk count before the + // response was truncated to previewMaxChunks for transport. + TruncatedTo int `json:"truncated_to,omitempty"` } // PreviewChunkingResponse is the body returned by /chunker/preview. @@ -77,7 +91,8 @@ type PreviewChunkingResponse struct { // information about which tier won. Read-only: no DB writes, no embedding // calls, no logging of the supplied text. func PreviewChunking(c *gin.Context) { - ctx := c.Request.Context() + ctx, cancel := context.WithTimeout(c.Request.Context(), previewTimeout) + defer cancel() var req PreviewChunkingRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -103,23 +118,56 @@ func PreviewChunking(c *gin.Context) { Languages: req.ChunkingConfig.Languages, } - chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) - profile := chunker.ProfileDocument(req.Text) + // Run the splitter on a goroutine so we can honor the request timeout. + // The splitter is CPU-bound and doesn't accept a context — wrapping + // here is the cheapest cancellation we can offer. + type splitResult struct { + chunks []chunker.Chunk + diag *chunker.Diagnostics + } + resCh := make(chan splitResult, 1) + go func() { + chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) + resCh <- splitResult{chunks: chunks, diag: diag} + }() + + var sr splitResult + select { + case sr = <-resCh: + case <-ctx.Done(): + c.JSON(http.StatusGatewayTimeout, gin.H{ + "success": false, + "error": "chunker preview timed out", + }) + return + } + + chunks, diag := sr.chunks, sr.diag + + // Diagnostics carries the profile when auto-strategy ran; for explicit + // strategies the profile is nil and we materialize it here so the UI + // can still show document stats. Avoids the previous double-pass. + profile := diag.Profile + if profile == nil { + profile = chunker.ProfileDocument(req.Text) + } logger.Debugf(ctx, "chunker preview: tier=%s chunks=%d", diag.SelectedTier, len(chunks)) - totalCount := len(chunks) - truncatedTo := 0 - if totalCount > previewMaxChunks { - truncatedTo = totalCount - chunks = chunks[:previewMaxChunks] - } - lang := chunker.LangMixed if len(profile.DetectedLangs) > 0 { lang = profile.DetectedLangs[0] } + // Compute stats over the FULL chunk set first so the metrics stay + // representative even when we trim the response to previewMaxChunks. + totalCount := len(chunks) + stats := computeChunkSizeStats(chunks, lang) + if totalCount > previewMaxChunks { + stats.TruncatedTo = totalCount + chunks = chunks[:previewMaxChunks] + } + results := make([]PreviewChunkResult, 0, len(chunks)) for _, ch := range chunks { runeLen := len([]rune(ch.Content)) @@ -140,22 +188,26 @@ func PreviewChunking(c *gin.Context) { Rejected: diag.Rejected, Profile: profile, Chunks: results, - Stats: chunkStats(results, truncatedTo), + Stats: stats, } c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } -// chunkStats computes count / avg / min / max / stddev over the chunks' -// SizeChars values. Returns zeroes for an empty slice. -func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingStats { - stats := PreviewChunkingStats{Count: len(chunks), TruncatedTo: truncatedTo} +// computeChunkSizeStats walks the full chunk slice once and returns the +// size distribution stats. Operates directly on chunker.Chunk so we don't +// need to materialize PreviewChunkResult before truncation. +// +// lang is forwarded to ApproxTokenCount only if callers extend the stats +// later — currently the result struct only tracks chars. +func computeChunkSizeStats(chunks []chunker.Chunk, _ string) PreviewChunkingStats { + stats := PreviewChunkingStats{Count: len(chunks)} if len(chunks) == 0 { return stats } var sum, sumSq float64 minLen, maxLen := math.MaxInt32, 0 for _, ch := range chunks { - l := ch.SizeChars + l := len([]rune(ch.Content)) sum += float64(l) sumSq += float64(l * l) if l < minLen { @@ -168,6 +220,8 @@ func chunkStats(chunks []PreviewChunkResult, truncatedTo int) PreviewChunkingSta avg := sum / float64(len(chunks)) variance := sumSq/float64(len(chunks)) - avg*avg if variance < 0 { + // Float precision can push the variance slightly below zero on + // near-uniform inputs; clamp so sqrt doesn't return NaN. variance = 0 } stats.AvgChars = int(avg + 0.5) diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 681a47c0..3e88c533 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -102,9 +102,14 @@ type KBModelConfigRequest struct { EnableParentChild bool `json:"enableParentChild"` ParentChunkSize int `json:"parentChunkSize,omitempty"` ChildChunkSize int `json:"childChunkSize,omitempty"` - Strategy string `json:"strategy,omitempty"` - TokenLimit int `json:"tokenLimit,omitempty"` - Languages []string `json:"languages,omitempty"` + // Strategy / TokenLimit / Languages use pointer types so the + // handler can distinguish "field absent in payload" (no change) + // from "field present with empty/zero value" (clear / disable). + // Without that distinction, users could set strategy="auto" once + // but never reset it back to legacy / unset. + Strategy *string `json:"strategy,omitempty"` + TokenLimit *int `json:"tokenLimit,omitempty"` + Languages *[]string `json:"languages,omitempty"` } `json:"documentSplitting"` // 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置) @@ -323,14 +328,17 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { if req.DocumentSplitting.ChildChunkSize > 0 { kb.ChunkingConfig.ChildChunkSize = req.DocumentSplitting.ChildChunkSize } - if req.DocumentSplitting.Strategy != "" { - kb.ChunkingConfig.Strategy = req.DocumentSplitting.Strategy + // Pointer-based fields support clearing (empty string / 0 / empty slice + // is a valid "user picked default again" signal; absent in payload means + // "no change"). + if req.DocumentSplitting.Strategy != nil { + kb.ChunkingConfig.Strategy = *req.DocumentSplitting.Strategy } - if req.DocumentSplitting.TokenLimit > 0 { - kb.ChunkingConfig.TokenLimit = req.DocumentSplitting.TokenLimit + if req.DocumentSplitting.TokenLimit != nil { + kb.ChunkingConfig.TokenLimit = *req.DocumentSplitting.TokenLimit } - if len(req.DocumentSplitting.Languages) > 0 { - kb.ChunkingConfig.Languages = req.DocumentSplitting.Languages + if req.DocumentSplitting.Languages != nil { + kb.ChunkingConfig.Languages = *req.DocumentSplitting.Languages } // 更新多模态配置 diff --git a/internal/infrastructure/chunker/profiler.go b/internal/infrastructure/chunker/profiler.go index 4f9e0294..6a89e336 100644 --- a/internal/infrastructure/chunker/profiler.go +++ b/internal/infrastructure/chunker/profiler.go @@ -10,8 +10,10 @@ import ( ) // DocProfile holds the document-level signals used to choose a chunking tier. -// JSON tags allow the profile to be surfaced via the chunker preview endpoint -// for debug UIs. +// +// The JSON shape (snake_case via struct tags) is part of the public preview +// endpoint API — keep field names stable. Internal callers should use the Go +// field names; only the preview handler relies on the wire format. type DocProfile struct { TotalChars int `json:"total_chars"` TotalLines int `json:"total_lines"` diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 6e5bc5cf..aaef43bb 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -26,9 +26,34 @@ const ( // is empty or "auto" the document profiler picks the tier. The function // always returns a non-nil result: on tier failure the chain falls through // to the legacy splitter, which is the original Tier 3 implementation. +// +// Hot path: avoids the Diagnostics struct allocation that +// SplitWithDiagnostics performs (matters in SplitParentChild where +// Split is called once per parent). func Split(text string, cfg SplitterConfig) []Chunk { - chunks, _ := SplitWithDiagnostics(text, cfg) - return chunks + if text == "" { + return nil + } + cfg = ensureDefaults(cfg) + chain, _ := resolveChainWithProfile(text, cfg) + totalChars := len([]rune(text)) + + var lastOut []Chunk + for i, tier := range chain { + out := runTier(tier, text, cfg) + if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { + return out + } else { + logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason) + } + if tier == TierLegacy && i == len(chain)-1 { + lastOut = out + } + } + if lastOut != nil { + return lastOut + } + return SplitText(text, cfg) } // TierRejection records why a tier was rejected by the validator and the @@ -40,26 +65,36 @@ type TierRejection struct { } // Diagnostics captures which tier produced the returned chunks plus the -// chain that was attempted and any rejected tiers along the way. Useful -// for surfacing in a debug UI; not produced by the normal Split path. +// chain that was attempted, any rejected tiers along the way, and the +// document profile that drove tier selection. +// +// Useful for surfacing in a debug UI; not produced by the normal Split +// path. The JSON shape is part of the public preview-endpoint API — keep +// field names stable. type Diagnostics struct { SelectedTier StrategyTier `json:"selected_tier"` TierChain []StrategyTier `json:"tier_chain"` Rejected []TierRejection `json:"rejected"` + // Profile is set when the auto strategy resolved the chain via the + // document profiler. nil when an explicit Strategy bypassed profiling. + Profile *DocProfile `json:"profile,omitempty"` } // SplitWithDiagnostics is the same as Split but also returns the -// diagnostic trace (selected tier, full chain, rejection reasons). Use -// this for the chunker preview endpoint where the caller wants to know -// which tier won and why others lost. +// diagnostic trace (selected tier, full chain, rejection reasons, +// profile when available). Use this for the chunker preview endpoint +// where the caller wants to know which tier won and why others lost. func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostics) { - diag := &Diagnostics{} + // Default selected tier to legacy so an empty diag never carries the + // zero string — that would render as a blank tag in the debug UI. + diag := &Diagnostics{SelectedTier: TierLegacy} if text == "" { return nil, diag } cfg = ensureDefaults(cfg) - chain := resolveChain(text, cfg) + chain, profile := resolveChainWithProfile(text, cfg) diag.TierChain = chain + diag.Profile = profile totalChars := len([]rune(text)) var lastOut []Chunk @@ -83,7 +118,6 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic return lastOut, diag } // Defensive last-ditch fallback. - diag.SelectedTier = TierLegacy return SplitText(text, cfg), diag } @@ -140,25 +174,27 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi return ParentChildResult{Parents: newParents, Children: children} } -// resolveChain returns the strategy chain to attempt. An explicit non-auto -// strategy bypasses the profiler entirely and pins to the requested tier. -func resolveChain(text string, cfg SplitterConfig) []StrategyTier { +// resolveChainWithProfile returns the strategy chain to attempt and, when +// the chain was selected by the profiler (auto strategy), the DocProfile +// that drove the selection. Profile is nil for explicit non-auto strategies +// so callers don't pay for an unused profiling pass. +func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *DocProfile) { switch cfg.Strategy { case StrategyHeading: - return []StrategyTier{TierHeading, TierLegacy} + return []StrategyTier{TierHeading, TierLegacy}, nil case StrategyHeuristic: - return []StrategyTier{TierHeuristic, TierLegacy} + return []StrategyTier{TierHeuristic, TierLegacy}, nil case StrategyRecursive: - return []StrategyTier{TierRecursive, TierLegacy} + return []StrategyTier{TierRecursive, TierLegacy}, nil case StrategyLegacy, "": // Empty == legacy preserves backwards compatibility with stored // ChunkingConfig rows that pre-date the Strategy field. - return []StrategyTier{TierLegacy} + return []StrategyTier{TierLegacy}, nil case StrategyAuto: fallthrough default: profile := ProfileDocument(text) - return SelectStrategy(profile) + return SelectStrategy(profile), profile } }