b333499078
- Added support for parent-child chunking in the knowledge base, allowing for a two-level chunking strategy that improves context retrieval. - Introduced new configuration options for enabling parent-child chunking, including parent and child chunk sizes. - Updated relevant components and services to handle parent chunk data, ensuring proper linking and retrieval of child chunks. - Enhanced user interface elements to reflect the new chunking options and provide better user guidance. These changes significantly improve the knowledge extraction process by providing richer context through hierarchical chunking.
381 lines
9.7 KiB
Go
381 lines
9.7 KiB
Go
// Package chunker implements text splitting for document chunking.
|
|
//
|
|
// Ported from the Python docreader/splitter/splitter.py recursive text splitter.
|
|
package chunker
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Chunk represents a piece of split text with position tracking.
|
|
type Chunk struct {
|
|
Content string
|
|
Seq int
|
|
Start int
|
|
End int
|
|
}
|
|
|
|
// ImageRef is an image reference found within a chunk's content.
|
|
type ImageRef struct {
|
|
OriginalRef string
|
|
AltText string
|
|
Start int // offset within the chunk content
|
|
End int
|
|
}
|
|
|
|
// SplitterConfig configures the text splitter.
|
|
type SplitterConfig struct {
|
|
ChunkSize int
|
|
ChunkOverlap int
|
|
Separators []string
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() SplitterConfig {
|
|
return SplitterConfig{
|
|
ChunkSize: 512,
|
|
ChunkOverlap: 128,
|
|
Separators: []string{"\n\n", "\n", "。"},
|
|
}
|
|
}
|
|
|
|
// protectedPatterns are regex patterns for content that must not be split.
|
|
var protectedPatterns = []*regexp.Regexp{
|
|
regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
|
|
regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
|
|
regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
|
|
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
|
|
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
|
|
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
|
|
}
|
|
|
|
type span struct {
|
|
start, end int
|
|
}
|
|
|
|
// protectedSpans finds all non-overlapping protected regions in text.
|
|
func protectedSpans(text string) []span {
|
|
type match struct {
|
|
start, end int
|
|
}
|
|
var all []match
|
|
for _, pat := range protectedPatterns {
|
|
locs := pat.FindAllStringIndex(text, -1)
|
|
for _, loc := range locs {
|
|
if loc[1]-loc[0] > 0 {
|
|
all = append(all, match{loc[0], loc[1]})
|
|
}
|
|
}
|
|
}
|
|
if len(all) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Sort by start, then by length descending
|
|
for i := 1; i < len(all); i++ {
|
|
for j := i; j > 0; j-- {
|
|
if all[j].start < all[j-1].start ||
|
|
(all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {
|
|
all[j], all[j-1] = all[j-1], all[j]
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove overlaps
|
|
var result []span
|
|
lastEnd := 0
|
|
for _, m := range all {
|
|
if m.start >= lastEnd {
|
|
result = append(result, span{m.start, m.end})
|
|
lastEnd = m.end
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// splitUnit is a piece of text with its original position.
|
|
type splitUnit struct {
|
|
text string
|
|
start, end int
|
|
}
|
|
|
|
// splitBySeparators splits text by separators in priority order, keeping separators.
|
|
func splitBySeparators(text string, separators []string) []string {
|
|
if len(separators) == 0 || text == "" {
|
|
return []string{text}
|
|
}
|
|
|
|
// Build regex that captures separators
|
|
var parts []string
|
|
for _, sep := range separators {
|
|
parts = append(parts, regexp.QuoteMeta(sep))
|
|
}
|
|
pattern := "(" + strings.Join(parts, "|") + ")"
|
|
re := regexp.MustCompile(pattern)
|
|
|
|
splits := re.Split(text, -1)
|
|
matches := re.FindAllString(text, -1)
|
|
|
|
var result []string
|
|
for i, s := range splits {
|
|
if s != "" {
|
|
result = append(result, s)
|
|
}
|
|
if i < len(matches) && matches[i] != "" {
|
|
result = append(result, matches[i])
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// runeLen returns the number of runes in s.
|
|
func runeLen(s string) int {
|
|
return utf8.RuneCountInString(s)
|
|
}
|
|
|
|
// SplitText splits text into chunks with overlap, respecting protected patterns.
|
|
func SplitText(text string, cfg SplitterConfig) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
|
|
chunkSize := cfg.ChunkSize
|
|
chunkOverlap := cfg.ChunkOverlap
|
|
separators := cfg.Separators
|
|
|
|
if chunkSize <= 0 {
|
|
chunkSize = 512
|
|
}
|
|
if chunkOverlap < 0 {
|
|
chunkOverlap = 0
|
|
}
|
|
|
|
// Step 1: Find protected spans
|
|
protected := protectedSpans(text)
|
|
|
|
// Step 2: Split non-protected regions by separators, keep protected as atomic units
|
|
units := buildUnitsWithProtection(text, protected, separators)
|
|
|
|
// Step 3: Merge units into chunks with overlap
|
|
return mergeUnits(units, chunkSize, chunkOverlap)
|
|
}
|
|
|
|
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
|
|
// Start/End positions in the returned units are rune offsets (not byte offsets),
|
|
// because downstream merge logic indexes content via []rune slicing.
|
|
func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit {
|
|
var units []splitUnit
|
|
bytePos := 0
|
|
runePos := 0
|
|
|
|
for _, p := range protected {
|
|
if p.start > bytePos {
|
|
pre := text[bytePos:p.start]
|
|
parts := splitBySeparators(pre, separators)
|
|
runeOffset := runePos
|
|
for _, part := range parts {
|
|
partRuneLen := runeLen(part)
|
|
units = append(units, splitUnit{
|
|
text: part,
|
|
start: runeOffset,
|
|
end: runeOffset + partRuneLen,
|
|
})
|
|
runeOffset += partRuneLen
|
|
}
|
|
runePos += runeLen(pre)
|
|
}
|
|
|
|
protText := text[p.start:p.end]
|
|
protRuneLen := runeLen(protText)
|
|
units = append(units, splitUnit{
|
|
text: protText,
|
|
start: runePos,
|
|
end: runePos + protRuneLen,
|
|
})
|
|
runePos += protRuneLen
|
|
bytePos = p.end
|
|
}
|
|
|
|
if bytePos < len(text) {
|
|
remaining := text[bytePos:]
|
|
parts := splitBySeparators(remaining, separators)
|
|
runeOffset := runePos
|
|
for _, part := range parts {
|
|
partRuneLen := runeLen(part)
|
|
units = append(units, splitUnit{
|
|
text: part,
|
|
start: runeOffset,
|
|
end: runeOffset + partRuneLen,
|
|
})
|
|
runeOffset += partRuneLen
|
|
}
|
|
}
|
|
|
|
return units
|
|
}
|
|
|
|
// mergeUnits combines split units into chunks with overlap tracking.
|
|
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
|
if len(units) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var chunks []Chunk
|
|
var current []splitUnit
|
|
curLen := 0
|
|
|
|
for _, u := range units {
|
|
uLen := runeLen(u.text)
|
|
|
|
// If adding this unit exceeds chunk size and we have content, flush
|
|
if curLen+uLen > chunkSize && len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
|
|
// Keep overlap from the end of current
|
|
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
|
|
}
|
|
|
|
current = append(current, u)
|
|
curLen += uLen
|
|
}
|
|
|
|
// Flush remaining
|
|
if len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
}
|
|
|
|
return chunks
|
|
}
|
|
|
|
func buildChunk(units []splitUnit, seq int) Chunk {
|
|
var sb strings.Builder
|
|
for _, u := range units {
|
|
sb.WriteString(u.text)
|
|
}
|
|
return Chunk{
|
|
Content: sb.String(),
|
|
Seq: seq,
|
|
Start: units[0].start,
|
|
End: units[len(units)-1].end,
|
|
}
|
|
}
|
|
|
|
// computeOverlap returns the units to keep for overlap and their total rune length.
|
|
func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) {
|
|
if chunkOverlap <= 0 {
|
|
return nil, 0
|
|
}
|
|
|
|
// Walk backward from end, accumulating overlap
|
|
overlapLen := 0
|
|
startIdx := len(current)
|
|
for i := len(current) - 1; i >= 0; i-- {
|
|
uLen := runeLen(current[i].text)
|
|
if overlapLen+uLen > chunkOverlap {
|
|
break
|
|
}
|
|
// Check that overlap + next unit fits in chunk
|
|
if overlapLen+uLen+nextLen > chunkSize {
|
|
break
|
|
}
|
|
overlapLen += uLen
|
|
startIdx = i
|
|
}
|
|
|
|
// Skip leading separators-only units in the overlap
|
|
for startIdx < len(current) {
|
|
u := current[startIdx]
|
|
trimmed := strings.TrimSpace(u.text)
|
|
if trimmed == "" || isSeparatorOnly(u.text) {
|
|
overlapLen -= runeLen(u.text)
|
|
startIdx++
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
|
|
if startIdx >= len(current) {
|
|
return nil, 0
|
|
}
|
|
|
|
overlap := make([]splitUnit, len(current)-startIdx)
|
|
copy(overlap, current[startIdx:])
|
|
return overlap, overlapLen
|
|
}
|
|
|
|
func isSeparatorOnly(s string) bool {
|
|
for _, r := range s {
|
|
if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ParentChildResult holds the two-level chunking output.
|
|
// Parent chunks provide context (large window), child chunks are used for
|
|
// embedding/retrieval (small window). Each child carries its ParentIndex so
|
|
// the caller can wire up ParentChunkID after DB insertion.
|
|
type ParentChildResult struct {
|
|
Parents []Chunk
|
|
Children []ChildChunk
|
|
}
|
|
|
|
// ChildChunk extends Chunk with a reference to its parent.
|
|
type ChildChunk struct {
|
|
Chunk
|
|
ParentIndex int // index into ParentChildResult.Parents
|
|
}
|
|
|
|
// SplitTextParentChild performs two-level chunking:
|
|
// 1. Split text into large parent chunks (parentCfg).
|
|
// 2. Split each parent into smaller child chunks (childCfg) for embedding.
|
|
//
|
|
// The child Seq is globally unique across the entire document.
|
|
func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
|
parents := SplitText(text, parentCfg)
|
|
if len(parents) == 0 {
|
|
return ParentChildResult{}
|
|
}
|
|
|
|
var children []ChildChunk
|
|
childSeq := 0
|
|
for pi, parent := range parents {
|
|
subs := SplitText(parent.Content, childCfg)
|
|
for _, sub := range subs {
|
|
// Adjust offsets: sub positions are relative to parent content,
|
|
// shift to document-level offsets.
|
|
sub.Seq = childSeq
|
|
sub.Start += parent.Start
|
|
sub.End = sub.Start + runeLen(sub.Content)
|
|
children = append(children, ChildChunk{
|
|
Chunk: sub,
|
|
ParentIndex: pi,
|
|
})
|
|
childSeq++
|
|
}
|
|
}
|
|
return ParentChildResult{Parents: parents, Children: children}
|
|
}
|
|
|
|
// ExtractImageRefs extracts markdown image references from text.
|
|
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
|
|
|
|
func ExtractImageRefs(text string) []ImageRef {
|
|
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
|
|
var refs []ImageRef
|
|
for _, m := range matches {
|
|
refs = append(refs, ImageRef{
|
|
OriginalRef: text[m[4]:m[5]], // group 2: URL
|
|
AltText: text[m[2]:m[3]], // group 1: alt text
|
|
Start: m[0],
|
|
End: m[1],
|
|
})
|
|
}
|
|
return refs
|
|
}
|