fix(chunker): trim surrounding whitespace before embedding

Heuristic and recursive splitters preserve original byte positions in
Chunk.Content (the End-Start == RuneCountInString invariant is required
by document-reconstruction code paths), which means a chunk sliced at a
boundary often carries leading/trailing newlines from the boundary
itself. Feeding that whitespace into the embedding model dilutes the
vector and wastes tokens for no benefit.

TrimSpace the body inside EmbeddingContent for both chunker.Chunk and
the two types.Chunk / types.ParsedChunk mirrors. Inner whitespace is
preserved; positions/Content are unchanged.

Also clean up appendChunk: the previous code computed a trimmed string
for the empty-check then stored a separate untrimmed copy, which read
as if it might be intentionally divergent. Replace with an explicit
'raw text, skip if pure whitespace' shape and document the invariant.
This commit is contained in:
wizardchen
2026-05-06 21:02:06 +08:00
committed by lyingbug
parent ad69240ac1
commit 7b9a831db3
4 changed files with 32 additions and 11 deletions
@@ -229,15 +229,20 @@ func allRuneIndices(text, needle string) []int {
}
// appendChunk slices runes[start:end] into a Chunk and appends it to out.
// Pure-whitespace slices are skipped (boundary clustering can occasionally
// produce them). The Content stored is the raw slice — Start/End rune
// offsets must match utf8.RuneCountInString(Content) for downstream
// reconstruction code; whitespace stripping for embedding happens in
// Chunk.EmbeddingContent.
func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk {
if end <= start {
return out
}
content := strings.TrimSpace(string(runes[start:end]))
if content == "" {
raw := string(runes[start:end])
if strings.TrimSpace(raw) == "" {
return out
}
c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end}
c := Chunk{Content: raw, Seq: *seq, Start: start, End: end}
*seq++
return append(out, c)
}
+9 -2
View File
@@ -34,11 +34,18 @@ type Chunk struct {
// EmbeddingContent returns the text that should be fed to the embedding
// model — the ContextHeader prepended (when set) plus the chunk content.
// Use this where Content alone would lose semantic context (Tier-1 chunks).
//
// Content is returned verbatim from the source document (the End-Start
// rune-count invariant requires that), but for embedding we trim the
// surrounding whitespace so leading/trailing newlines from boundary slices
// don't dilute the embedded vector or waste tokens. Inner whitespace is
// preserved.
func (c Chunk) EmbeddingContent() string {
body := strings.TrimSpace(c.Content)
if c.ContextHeader == "" {
return c.Content
return body
}
return c.ContextHeader + "\n\n" + c.Content
return c.ContextHeader + "\n\n" + body
}
// ImageRef is an image reference found within a chunk's content.
+7 -3
View File
@@ -3,6 +3,7 @@
package types
import (
"strings"
"time"
"gorm.io/gorm"
@@ -172,15 +173,18 @@ type Chunk struct {
// EmbeddingContent returns the chunk content with ContextHeader prepended
// when set. Use this where the embedding model needs section context that
// isn't part of the literal Content.
// isn't part of the literal Content. Surrounding whitespace on Content is
// trimmed so leading/trailing newlines from boundary slicing don't dilute
// the embedded vector.
func (c *Chunk) EmbeddingContent() string {
if c == nil {
return ""
}
body := strings.TrimSpace(c.Content)
if c.ContextHeader == "" {
return c.Content
return body
}
return c.ContextHeader + "\n\n" + c.Content
return c.ContextHeader + "\n\n" + body
}
// AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0.
+8 -3
View File
@@ -1,5 +1,7 @@
package types
import "strings"
// ReadRequest is the unified transport-agnostic request for document reading.
// Set FileContent for file mode, URL for URL mode.
type ReadRequest struct {
@@ -90,12 +92,15 @@ type ParsedChunk struct {
// EmbeddingContent returns the text that should be sent to the embedding
// model: ContextHeader (if any) prepended to Content. Mirrors
// chunker.Chunk.EmbeddingContent so the choice is consistent across the
// chunker output and the indexing pipeline.
// chunker output and the indexing pipeline. Surrounding whitespace on
// Content is trimmed so leading/trailing newlines from boundary slicing
// don't dilute the embedded vector.
func (c ParsedChunk) EmbeddingContent() string {
body := strings.TrimSpace(c.Content)
if c.ContextHeader == "" {
return c.Content
return body
}
return c.ContextHeader + "\n\n" + c.Content
return c.ContextHeader + "\n\n" + body
}
// ParsedParentChunk represents a parent chunk in the parent-child strategy.