diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go index 3bac3098..dba86a5d 100644 --- a/internal/infrastructure/chunker/heuristic_splitter.go +++ b/internal/infrastructure/chunker/heuristic_splitter.go @@ -229,15 +229,20 @@ func allRuneIndices(text, needle string) []int { } // appendChunk slices runes[start:end] into a Chunk and appends it to out. +// Pure-whitespace slices are skipped (boundary clustering can occasionally +// produce them). The Content stored is the raw slice — Start/End rune +// offsets must match utf8.RuneCountInString(Content) for downstream +// reconstruction code; whitespace stripping for embedding happens in +// Chunk.EmbeddingContent. func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk { if end <= start { return out } - content := strings.TrimSpace(string(runes[start:end])) - if content == "" { + raw := string(runes[start:end]) + if strings.TrimSpace(raw) == "" { return out } - c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end} + c := Chunk{Content: raw, Seq: *seq, Start: start, End: end} *seq++ return append(out, c) } diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 514e2c56..d27a5a57 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -34,11 +34,18 @@ type Chunk struct { // EmbeddingContent returns the text that should be fed to the embedding // model — the ContextHeader prepended (when set) plus the chunk content. // Use this where Content alone would lose semantic context (Tier-1 chunks). +// +// Content is returned verbatim from the source document (the End-Start +// rune-count invariant requires that), but for embedding we trim the +// surrounding whitespace so leading/trailing newlines from boundary slices +// don't dilute the embedded vector or waste tokens. Inner whitespace is +// preserved. func (c Chunk) EmbeddingContent() string { + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // ImageRef is an image reference found within a chunk's content. diff --git a/internal/types/chunk.go b/internal/types/chunk.go index f171b067..e27c331d 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -3,6 +3,7 @@ package types import ( + "strings" "time" "gorm.io/gorm" @@ -172,15 +173,18 @@ type Chunk struct { // EmbeddingContent returns the chunk content with ContextHeader prepended // when set. Use this where the embedding model needs section context that -// isn't part of the literal Content. +// isn't part of the literal Content. Surrounding whitespace on Content is +// trimmed so leading/trailing newlines from boundary slicing don't dilute +// the embedded vector. func (c *Chunk) EmbeddingContent() string { if c == nil { return "" } + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0. diff --git a/internal/types/docparser.go b/internal/types/docparser.go index 44a6fa6e..7db9d554 100644 --- a/internal/types/docparser.go +++ b/internal/types/docparser.go @@ -1,5 +1,7 @@ package types +import "strings" + // ReadRequest is the unified transport-agnostic request for document reading. // Set FileContent for file mode, URL for URL mode. type ReadRequest struct { @@ -90,12 +92,15 @@ type ParsedChunk struct { // EmbeddingContent returns the text that should be sent to the embedding // model: ContextHeader (if any) prepended to Content. Mirrors // chunker.Chunk.EmbeddingContent so the choice is consistent across the -// chunker output and the indexing pipeline. +// chunker output and the indexing pipeline. Surrounding whitespace on +// Content is trimmed so leading/trailing newlines from boundary slicing +// don't dilute the embedded vector. func (c ParsedChunk) EmbeddingContent() string { + body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { - return c.Content + return body } - return c.ContextHeader + "\n\n" + c.Content + return c.ContextHeader + "\n\n" + body } // ParsedParentChunk represents a parent chunk in the parent-child strategy.