// Package chunker implements text splitting for document chunking. // // Ported from the Python docreader/splitter/splitter.py recursive text splitter. package chunker import ( "regexp" "strings" "unicode/utf8" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" ) // Chunk represents a piece of split text with position tracking. type Chunk struct { Content string Seq int Start int End int } // ImageRef is an image reference found within a chunk's content. type ImageRef struct { OriginalRef string AltText string Start int // offset within the chunk content End int } // SplitterConfig configures the text splitter. type SplitterConfig struct { ChunkSize int ChunkOverlap int Separators []string } // DefaultConfig returns sensible defaults. func DefaultConfig() SplitterConfig { return SplitterConfig{ ChunkSize: 512, ChunkOverlap: 128, Separators: []string{"\n\n", "\n", "。"}, } } // protectedPatterns are regex patterns for content that must not be split. var protectedPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks } type span struct { start, end int } // protectedSpans finds all non-overlapping protected regions in text. func protectedSpans(text string) []span { type match struct { start, end int } var all []match for _, pat := range protectedPatterns { locs := pat.FindAllStringIndex(text, -1) for _, loc := range locs { if loc[1]-loc[0] > 0 { all = append(all, match{loc[0], loc[1]}) } } } if len(all) == 0 { return nil } // Sort by start, then by length descending for i := 1; i < len(all); i++ { for j := i; j > 0; j-- { if all[j].start < all[j-1].start || (all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) { all[j], all[j-1] = all[j-1], all[j] } else { break } } } // Remove overlaps var result []span lastEnd := 0 for _, m := range all { if m.start >= lastEnd { result = append(result, span{m.start, m.end}) lastEnd = m.end } } return result } // splitUnit is a piece of text with its original position. type splitUnit struct { text string start, end int } // splitBySeparators splits text by separators in priority order, keeping separators. func splitBySeparators(text string, separators []string) []string { if len(separators) == 0 || text == "" { return []string{text} } // Build regex that captures separators var parts []string for _, sep := range separators { parts = append(parts, regexp.QuoteMeta(sep)) } pattern := "(" + strings.Join(parts, "|") + ")" re := regexp.MustCompile(pattern) splits := re.Split(text, -1) matches := re.FindAllString(text, -1) var result []string for i, s := range splits { if s != "" { result = append(result, s) } if i < len(matches) && matches[i] != "" { result = append(result, matches[i]) } } return result } // runeLen returns the number of runes in s. func runeLen(s string) int { return utf8.RuneCountInString(s) } // SplitText splits text into chunks with overlap, respecting protected patterns. func SplitText(text string, cfg SplitterConfig) []Chunk { if text == "" { return nil } chunkSize := cfg.ChunkSize chunkOverlap := cfg.ChunkOverlap separators := cfg.Separators if chunkSize <= 0 { chunkSize = 512 } if chunkOverlap < 0 { chunkOverlap = 0 } // Step 1: Find protected spans protected := protectedSpans(text) // Step 2: Split non-protected regions by separators, keep protected as atomic units units := buildUnitsWithProtection(text, protected, separators) // Step 3: Merge units into chunks with overlap return mergeUnits(units, chunkSize, chunkOverlap) } // buildUnitsWithProtection splits text into units, preserving protected spans as atomic. // Start/End positions in the returned units are rune offsets (not byte offsets), // because downstream merge logic indexes content via []rune slicing. // If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent // creating chunks that are too large for downstream processing (e.g., embedding APIs). func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit { const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等) var units []splitUnit bytePos := 0 runePos := 0 for _, p := range protected { if p.start > bytePos { pre := text[bytePos:p.start] parts := splitBySeparators(pre, separators) runeOffset := runePos for _, part := range parts { partRuneLen := runeLen(part) units = append(units, splitUnit{ text: part, start: runeOffset, end: runeOffset + partRuneLen, }) runeOffset += partRuneLen } runePos += runeLen(pre) } protText := text[p.start:p.end] protRuneLen := runeLen(protText) // If protected content is too large, forcibly split it if protRuneLen > maxProtectedSize { // Split into smaller chunks at line breaks or spaces runes := []rune(protText) offset := 0 for offset < len(runes) { chunkEnd := offset + maxProtectedSize if chunkEnd > len(runes) { chunkEnd = len(runes) } else { // Try to break at a newline or space for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { if runes[i] == '\n' || runes[i] == ' ' { chunkEnd = i + 1 break } } } chunkText := string(runes[offset:chunkEnd]) chunkLen := chunkEnd - offset units = append(units, splitUnit{ text: chunkText, start: runePos + offset, end: runePos + offset + chunkLen, }) offset = chunkEnd } } else { // Normal case: keep protected content as a single unit units = append(units, splitUnit{ text: protText, start: runePos, end: runePos + protRuneLen, }) } runePos += protRuneLen bytePos = p.end } if bytePos < len(text) { remaining := text[bytePos:] parts := splitBySeparators(remaining, separators) runeOffset := runePos for _, part := range parts { partRuneLen := runeLen(part) units = append(units, splitUnit{ text: part, start: runeOffset, end: runeOffset + partRuneLen, }) runeOffset += partRuneLen } } return units } // mergeUnits combines split units into chunks with overlap tracking. // Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs). // Active contextual headers (e.g., Markdown table headers) are prepended to new // chunks so that every chunk carries its own header context. func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if len(units) == 0 { return nil } const absoluteMaxSize = 7500 ht := newHeaderTracker() var chunks []Chunk var current []splitUnit curLen := 0 for _, u := range units { uLen := runeLen(u.text) // If this single unit exceeds absolute max, force split it further if uLen > absoluteMaxSize { // Flush current chunk if any if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) current = nil curLen = 0 } // Update header state even for oversized units ht.update(u.text) // Split this oversized unit into smaller chunks runes := []rune(u.text) offset := 0 for offset < len(runes) { chunkEnd := offset + absoluteMaxSize if chunkEnd > len(runes) { chunkEnd = len(runes) } else { for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { if runes[i] == '\n' || runes[i] == ' ' { chunkEnd = i + 1 break } } } chunkText := string(runes[offset:chunkEnd]) chunks = append(chunks, Chunk{ Content: chunkText, Seq: len(chunks), Start: u.start + offset, End: u.start + chunkEnd, }) offset = chunkEnd } continue } // Update header tracking ht.update(u.text) headers := ht.getHeaders() headersLen := runeLen(headers) if headersLen > chunkSize { headers = "" headersLen = 0 } // If adding this unit (plus reserving space for headers in a potential // next chunk) would exceed chunk size, flush the current chunk. if curLen+uLen+headersLen > chunkSize && len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) // Keep overlap from the end of current current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen) // Shrink overlap further if needed to fit headers + next unit if headers != "" && headersLen+uLen <= chunkSize { for len(current) > 0 && curLen+uLen+headersLen > chunkSize { curLen -= runeLen(current[0].text) current = current[1:] } // Prepend headers if the column-name context is not already present // in the overlap or the next unit being added. overlapText := unitsText(current) if !headerAlreadyPresent(headers, overlapText, u.text) { startPos := u.start if len(current) > 0 { startPos = current[0].start } hUnit := splitUnit{text: headers, start: startPos, end: startPos} current = append([]splitUnit{hUnit}, current...) curLen += headersLen } } } // Check if adding this unit would exceed absolute max if curLen+uLen > absoluteMaxSize { if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) current = nil curLen = 0 } } current = append(current, u) curLen += uLen } // Flush remaining if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) } return chunks } // unitsText concatenates the text of all units. func unitsText(units []splitUnit) string { var sb strings.Builder for _, u := range units { sb.WriteString(u.text) } return sb.String() } // headerAlreadyPresent returns true if the column-name row from the header // is already present in the overlap or the next unit, preventing duplication. func headerAlreadyPresent(headers, overlapText, unitText string) bool { // Fast path: full header already in overlap or unit if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) { return true } // Extract the column-name row (first meaningful non-separator line). // For a rewritten header like "| col1 | col2 |\n| --- | --- |\n", // the first line is the column names. colRow := headerColumnRow(headers) if colRow == "" { return false } return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow) } // headerColumnRow extracts the column-name line from a header string. // Returns empty string if the header has no meaningful column names. func headerColumnRow(header string) string { for _, line := range strings.Split(header, "\n") { line = strings.TrimSpace(line) if line == "" || strings.Contains(line, "---") { continue } // Skip lines that are only pipes/whitespace (empty header rows) onlyPipes := true for _, r := range line { if r != '|' && r != ' ' && r != '\t' { onlyPipes = false break } } if !onlyPipes { return line } } return "" } func buildChunk(units []splitUnit, seq int) Chunk { var sb strings.Builder for _, u := range units { sb.WriteString(u.text) } return Chunk{ Content: sb.String(), Seq: seq, Start: units[0].start, End: units[len(units)-1].end, } } // computeOverlap returns the units to keep for overlap and their total rune length. func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) { if chunkOverlap <= 0 { return nil, 0 } // Walk backward from end, accumulating overlap overlapLen := 0 startIdx := len(current) for i := len(current) - 1; i >= 0; i-- { uLen := runeLen(current[i].text) if overlapLen+uLen > chunkOverlap { break } // Check that overlap + next unit fits in chunk if overlapLen+uLen+nextLen > chunkSize { break } overlapLen += uLen startIdx = i } // Skip leading separator-only and header-marker units in the overlap for startIdx < len(current) { u := current[startIdx] isHeaderMarker := u.start == u.end trimmed := strings.TrimSpace(u.text) if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) { overlapLen -= runeLen(u.text) startIdx++ } else { break } } if startIdx >= len(current) { return nil, 0 } overlap := make([]splitUnit, len(current)-startIdx) copy(overlap, current[startIdx:]) return overlap, overlapLen } func isSeparatorOnly(s string) bool { for _, r := range s { if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' { return false } } return true } // ParentChildResult holds the two-level chunking output. // Parent chunks provide context (large window), child chunks are used for // embedding/retrieval (small window). Each child carries its ParentIndex so // the caller can wire up ParentChunkID after DB insertion. type ParentChildResult struct { Parents []Chunk Children []ChildChunk } // ChildChunk extends Chunk with a reference to its parent. type ChildChunk struct { Chunk ParentIndex int // index into ParentChildResult.Parents } // SplitTextParentChild performs two-level chunking: // 1. Split text into large parent chunks (parentCfg). // 2. Split each parent into smaller child chunks (childCfg) for embedding. // // The child Seq is globally unique across the entire document. func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { parents := SplitText(text, parentCfg) if len(parents) == 0 { return ParentChildResult{} } var children []ChildChunk childSeq := 0 for pi, parent := range parents { subs := SplitText(parent.Content, childCfg) for _, sub := range subs { // Adjust offsets: sub positions are relative to parent content, // shift to document-level offsets. // Use additive shift (not Content-length based) so that chunks with // prepended context headers keep correct positional tracking. sub.Seq = childSeq sub.Start += parent.Start sub.End += parent.Start children = append(children, ChildChunk{ Chunk: sub, ParentIndex: pi, }) childSeq++ } } return ParentChildResult{Parents: parents, Children: children} } // ExtractImageRefs extracts markdown image references from text. // The URL group supports one level of balanced parentheses so that URLs // like https://example.com/item_(abc)/123 are captured in full. var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) func ExtractImageRefs(text string) []ImageRef { text = docparser.UnwrapLinkedImages(text) matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1) var refs []ImageRef for _, m := range matches { refs = append(refs, ImageRef{ OriginalRef: text[m[4]:m[5]], // group 2: URL AltText: text[m[2]:m[3]], // group 1: alt text Start: m[0], End: m[1], }) } return refs }