From 7b9a831db3fa8ce1e991d9b6b83c973b34f91c0c Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 6 May 2026 21:02:06 +0800 Subject: [PATCH] fix(chunker): trim surrounding whitespace before embedding Heuristic and recursive splitters preserve original byte positions in Chunk.Content (the End-Start == RuneCountInString invariant is required by document-reconstruction code paths), which means a chunk sliced at a boundary often carries leading/trailing newlines from the boundary itself. Feeding that whitespace into the embedding model dilutes the vector and wastes tokens for no benefit. TrimSpace the body inside EmbeddingContent for both chunker.Chunk and the two types.Chunk / types.ParsedChunk mirrors. Inner whitespace is preserved; positions/Content are unchanged. Also clean up appendChunk: the previous code computed a trimmed string for the empty-check then stored a separate untrimmed copy, which read as if it might be intentionally divergent. Replace with an explicit 'raw text, skip if pure whitespace' shape and document the invariant. --- internal/infrastructure/chunker/heuristic_splitter.go | 11 ++++++++--- internal/infrastructure/chunker/splitter.go | 11 +++++++++-- internal/types/chunk.go | 10 +++++++--- internal/types/docparser.go | 11 ++++++++--- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go index 3bac3098..dba86a5d 100644 --- a/internal/infrastructure/chunker/heuristic_splitter.go +++ b/internal/infrastructure/chunker/heuristic_splitter.go @@ -229,15 +229,20 @@ func allRuneIndices(text, needle string) []int { } // appendChunk slices runes[start:end] into a Chunk and appends it to out. +// Pure-whitespace slices are skipped (boundary clustering can occasionally +// produce them). The Content stored is the raw slice — Start/End rune +// offsets must match utf8.RuneCountInString(Content) for downstream +// reconstruction code; whitespace stripping for embedding happens in +// Chunk.EmbeddingContent. func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk { if end <= start { return out } - content := strings.TrimSpace(string(runes[start:end])) - if content == "" { + raw := string(runes[start:end]) + if strings.TrimSpace(raw) == "" { return out } - c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end} + c := Chunk{Content: raw, Seq: *seq, Start: start, End: end} *seq++ return append(out, c) } diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 514e2c56..d27a5a57 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -34,11 +34,18 @@ type Chunk struct { // EmbeddingContent returns the text that should be fed to the embedding // model — the ContextHeader prepended (when set) plus the chunk content. // Use this where Content alone would lose semantic context (Tier-1 chunks). +// +// Content is returned verbatim from the source document (the End-Start +// rune-count invariant requires that), but for embedding we trim the +// surrounding whitespace so leading/trailing newlines from boundary slices +// don't dilute the embedded vector or waste tokens. Inner whitespace is +// preserved. func (c Chunk) EmbeddingContent() string { + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // ImageRef is an image reference found within a chunk's content. diff --git a/internal/types/chunk.go b/internal/types/chunk.go index f171b067..e27c331d 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -3,6 +3,7 @@ package types import ( + "strings" "time" "gorm.io/gorm" @@ -172,15 +173,18 @@ type Chunk struct { // EmbeddingContent returns the chunk content with ContextHeader prepended // when set. Use this where the embedding model needs section context that -// isn't part of the literal Content. +// isn't part of the literal Content. Surrounding whitespace on Content is +// trimmed so leading/trailing newlines from boundary slicing don't dilute +// the embedded vector. func (c *Chunk) EmbeddingContent() string { if c == nil { return "" } + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0. diff --git a/internal/types/docparser.go b/internal/types/docparser.go index 44a6fa6e..7db9d554 100644 --- a/internal/types/docparser.go +++ b/internal/types/docparser.go @@ -1,5 +1,7 @@ package types +import "strings" + // ReadRequest is the unified transport-agnostic request for document reading. // Set FileContent for file mode, URL for URL mode. type ReadRequest struct { @@ -90,12 +92,15 @@ type ParsedChunk struct { // EmbeddingContent returns the text that should be sent to the embedding // model: ContextHeader (if any) prepended to Content. Mirrors // chunker.Chunk.EmbeddingContent so the choice is consistent across the -// chunker output and the indexing pipeline. +// chunker output and the indexing pipeline. Surrounding whitespace on +// Content is trimmed so leading/trailing newlines from boundary slicing +// don't dilute the embedded vector. func (c ParsedChunk) EmbeddingContent() string { + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // ParsedParentChunk represents a parent chunk in the parent-child strategy.