// Package chunker implements text splitting for document chunking. // // Ported from the Python docreader/splitter/splitter.py recursive text splitter. package chunker import ( "regexp" "strings" "unicode/utf8" ) // Chunk represents a piece of split text with position tracking. type Chunk struct { Content string Seq int Start int End int } // ImageRef is an image reference found within a chunk's content. type ImageRef struct { OriginalRef string AltText string Start int // offset within the chunk content End int } // SplitterConfig configures the text splitter. type SplitterConfig struct { ChunkSize int ChunkOverlap int Separators []string } // DefaultConfig returns sensible defaults. func DefaultConfig() SplitterConfig { return SplitterConfig{ ChunkSize: 512, ChunkOverlap: 128, Separators: []string{"\n\n", "\n", "。"}, } } // protectedPatterns are regex patterns for content that must not be split. var protectedPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks } type span struct { start, end int } // protectedSpans finds all non-overlapping protected regions in text. func protectedSpans(text string) []span { type match struct { start, end int } var all []match for _, pat := range protectedPatterns { locs := pat.FindAllStringIndex(text, -1) for _, loc := range locs { if loc[1]-loc[0] > 0 { all = append(all, match{loc[0], loc[1]}) } } } if len(all) == 0 { return nil } // Sort by start, then by length descending for i := 1; i < len(all); i++ { for j := i; j > 0; j-- { if all[j].start < all[j-1].start || (all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) { all[j], all[j-1] = all[j-1], all[j] } else { break } } } // Remove overlaps var result []span lastEnd := 0 for _, m := range all { if m.start >= lastEnd { result = append(result, span{m.start, m.end}) lastEnd = m.end } } return result } // splitUnit is a piece of text with its original position. type splitUnit struct { text string start, end int } // splitBySeparators splits text by separators in priority order, keeping separators. func splitBySeparators(text string, separators []string) []string { if len(separators) == 0 || text == "" { return []string{text} } // Build regex that captures separators var parts []string for _, sep := range separators { parts = append(parts, regexp.QuoteMeta(sep)) } pattern := "(" + strings.Join(parts, "|") + ")" re := regexp.MustCompile(pattern) splits := re.Split(text, -1) matches := re.FindAllString(text, -1) var result []string for i, s := range splits { if s != "" { result = append(result, s) } if i < len(matches) && matches[i] != "" { result = append(result, matches[i]) } } return result } // runeLen returns the number of runes in s. func runeLen(s string) int { return utf8.RuneCountInString(s) } // SplitText splits text into chunks with overlap, respecting protected patterns. func SplitText(text string, cfg SplitterConfig) []Chunk { if text == "" { return nil } chunkSize := cfg.ChunkSize chunkOverlap := cfg.ChunkOverlap separators := cfg.Separators if chunkSize <= 0 { chunkSize = 512 } if chunkOverlap < 0 { chunkOverlap = 0 } // Step 1: Find protected spans protected := protectedSpans(text) // Step 2: Split non-protected regions by separators, keep protected as atomic units units := buildUnitsWithProtection(text, protected, separators) // Step 3: Merge units into chunks with overlap return mergeUnits(units, chunkSize, chunkOverlap) } // buildUnitsWithProtection splits text into units, preserving protected spans as atomic. // Start/End positions in the returned units are rune offsets (not byte offsets), // because downstream merge logic indexes content via []rune slicing. // If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent // creating chunks that are too large for downstream processing (e.g., embedding APIs). func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit { const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等) var units []splitUnit bytePos := 0 runePos := 0 for _, p := range protected { if p.start > bytePos { pre := text[bytePos:p.start] parts := splitBySeparators(pre, separators) runeOffset := runePos for _, part := range parts { partRuneLen := runeLen(part) units = append(units, splitUnit{ text: part, start: runeOffset, end: runeOffset + partRuneLen, }) runeOffset += partRuneLen } runePos += runeLen(pre) } protText := text[p.start:p.end] protRuneLen := runeLen(protText) // If protected content is too large, forcibly split it if protRuneLen > maxProtectedSize { // Split into smaller chunks at line breaks or spaces runes := []rune(protText) offset := 0 for offset < len(runes) { chunkEnd := offset + maxProtectedSize if chunkEnd > len(runes) { chunkEnd = len(runes) } else { // Try to break at a newline or space for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { if runes[i] == '\n' || runes[i] == ' ' { chunkEnd = i + 1 break } } } chunkText := string(runes[offset:chunkEnd]) chunkLen := chunkEnd - offset units = append(units, splitUnit{ text: chunkText, start: runePos + offset, end: runePos + offset + chunkLen, }) offset = chunkEnd } } else { // Normal case: keep protected content as a single unit units = append(units, splitUnit{ text: protText, start: runePos, end: runePos + protRuneLen, }) } runePos += protRuneLen bytePos = p.end } if bytePos < len(text) { remaining := text[bytePos:] parts := splitBySeparators(remaining, separators) runeOffset := runePos for _, part := range parts { partRuneLen := runeLen(part) units = append(units, splitUnit{ text: part, start: runeOffset, end: runeOffset + partRuneLen, }) runeOffset += partRuneLen } } return units } // mergeUnits combines split units into chunks with overlap tracking. // Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs). func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if len(units) == 0 { return nil } // Absolute maximum chunk size (留余量给标题等额外内容) const absoluteMaxSize = 7500 var chunks []Chunk var current []splitUnit curLen := 0 for _, u := range units { uLen := runeLen(u.text) // If this single unit exceeds absolute max, force split it further if uLen > absoluteMaxSize { // Flush current chunk if any if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) current = nil curLen = 0 } // Split this oversized unit into smaller chunks runes := []rune(u.text) offset := 0 for offset < len(runes) { chunkEnd := offset + absoluteMaxSize if chunkEnd > len(runes) { chunkEnd = len(runes) } else { // Try to break at a newline or space for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { if runes[i] == '\n' || runes[i] == ' ' { chunkEnd = i + 1 break } } } chunkText := string(runes[offset:chunkEnd]) chunks = append(chunks, Chunk{ Content: chunkText, Seq: len(chunks), Start: u.start + offset, End: u.start + chunkEnd, }) offset = chunkEnd } continue } // If adding this unit exceeds chunk size and we have content, flush if curLen+uLen > chunkSize && len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) // Keep overlap from the end of current current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen) } // Check if adding this unit would exceed absolute max if curLen+uLen > absoluteMaxSize { // Flush current and start fresh if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) current = nil curLen = 0 } } current = append(current, u) curLen += uLen } // Flush remaining if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) } return chunks } func buildChunk(units []splitUnit, seq int) Chunk { var sb strings.Builder for _, u := range units { sb.WriteString(u.text) } return Chunk{ Content: sb.String(), Seq: seq, Start: units[0].start, End: units[len(units)-1].end, } } // computeOverlap returns the units to keep for overlap and their total rune length. func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) { if chunkOverlap <= 0 { return nil, 0 } // Walk backward from end, accumulating overlap overlapLen := 0 startIdx := len(current) for i := len(current) - 1; i >= 0; i-- { uLen := runeLen(current[i].text) if overlapLen+uLen > chunkOverlap { break } // Check that overlap + next unit fits in chunk if overlapLen+uLen+nextLen > chunkSize { break } overlapLen += uLen startIdx = i } // Skip leading separators-only units in the overlap for startIdx < len(current) { u := current[startIdx] trimmed := strings.TrimSpace(u.text) if trimmed == "" || isSeparatorOnly(u.text) { overlapLen -= runeLen(u.text) startIdx++ } else { break } } if startIdx >= len(current) { return nil, 0 } overlap := make([]splitUnit, len(current)-startIdx) copy(overlap, current[startIdx:]) return overlap, overlapLen } func isSeparatorOnly(s string) bool { for _, r := range s { if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' { return false } } return true } // ParentChildResult holds the two-level chunking output. // Parent chunks provide context (large window), child chunks are used for // embedding/retrieval (small window). Each child carries its ParentIndex so // the caller can wire up ParentChunkID after DB insertion. type ParentChildResult struct { Parents []Chunk Children []ChildChunk } // ChildChunk extends Chunk with a reference to its parent. type ChildChunk struct { Chunk ParentIndex int // index into ParentChildResult.Parents } // SplitTextParentChild performs two-level chunking: // 1. Split text into large parent chunks (parentCfg). // 2. Split each parent into smaller child chunks (childCfg) for embedding. // // The child Seq is globally unique across the entire document. func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { parents := SplitText(text, parentCfg) if len(parents) == 0 { return ParentChildResult{} } var children []ChildChunk childSeq := 0 for pi, parent := range parents { subs := SplitText(parent.Content, childCfg) for _, sub := range subs { // Adjust offsets: sub positions are relative to parent content, // shift to document-level offsets. sub.Seq = childSeq sub.Start += parent.Start sub.End = sub.Start + runeLen(sub.Content) children = append(children, ChildChunk{ Chunk: sub, ParentIndex: pi, }) childSeq++ } } return ParentChildResult{Parents: parents, Children: children} } // ExtractImageRefs extracts markdown image references from text. var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) func ExtractImageRefs(text string) []ImageRef { matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1) var refs []ImageRef for _, m := range matches { refs = append(refs, ImageRef{ OriginalRef: text[m[4]:m[5]], // group 2: URL AltText: text[m[2]:m[3]], // group 1: alt text Start: m[0], End: m[1], }) } return refs }