ce7df624fb
- Added headerTracker to preserve context of Markdown table headers across split chunks. - Implemented logic to prepend headers to subsequent chunks after the first. - Enhanced merging logic to ensure headers are not duplicated in overlapping content. - Added comprehensive tests for header tracking and chunking behavior with Markdown tables.
563 lines
15 KiB
Go
563 lines
15 KiB
Go
// Package chunker implements text splitting for document chunking.
|
|
//
|
|
// Ported from the Python docreader/splitter/splitter.py recursive text splitter.
|
|
package chunker
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
|
)
|
|
|
|
// Chunk represents a piece of split text with position tracking.
|
|
type Chunk struct {
|
|
Content string
|
|
Seq int
|
|
Start int
|
|
End int
|
|
}
|
|
|
|
// ImageRef is an image reference found within a chunk's content.
|
|
type ImageRef struct {
|
|
OriginalRef string
|
|
AltText string
|
|
Start int // offset within the chunk content
|
|
End int
|
|
}
|
|
|
|
// SplitterConfig configures the text splitter.
|
|
type SplitterConfig struct {
|
|
ChunkSize int
|
|
ChunkOverlap int
|
|
Separators []string
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() SplitterConfig {
|
|
return SplitterConfig{
|
|
ChunkSize: 512,
|
|
ChunkOverlap: 128,
|
|
Separators: []string{"\n\n", "\n", "。"},
|
|
}
|
|
}
|
|
|
|
// protectedPatterns are regex patterns for content that must not be split.
|
|
var protectedPatterns = []*regexp.Regexp{
|
|
regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
|
|
regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
|
|
regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
|
|
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
|
|
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
|
|
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
|
|
}
|
|
|
|
type span struct {
|
|
start, end int
|
|
}
|
|
|
|
// protectedSpans finds all non-overlapping protected regions in text.
|
|
func protectedSpans(text string) []span {
|
|
type match struct {
|
|
start, end int
|
|
}
|
|
var all []match
|
|
for _, pat := range protectedPatterns {
|
|
locs := pat.FindAllStringIndex(text, -1)
|
|
for _, loc := range locs {
|
|
if loc[1]-loc[0] > 0 {
|
|
all = append(all, match{loc[0], loc[1]})
|
|
}
|
|
}
|
|
}
|
|
if len(all) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Sort by start, then by length descending
|
|
for i := 1; i < len(all); i++ {
|
|
for j := i; j > 0; j-- {
|
|
if all[j].start < all[j-1].start ||
|
|
(all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {
|
|
all[j], all[j-1] = all[j-1], all[j]
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove overlaps
|
|
var result []span
|
|
lastEnd := 0
|
|
for _, m := range all {
|
|
if m.start >= lastEnd {
|
|
result = append(result, span{m.start, m.end})
|
|
lastEnd = m.end
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// splitUnit is a piece of text with its original position.
|
|
type splitUnit struct {
|
|
text string
|
|
start, end int
|
|
}
|
|
|
|
// splitBySeparators splits text by separators in priority order, keeping separators.
|
|
func splitBySeparators(text string, separators []string) []string {
|
|
if len(separators) == 0 || text == "" {
|
|
return []string{text}
|
|
}
|
|
|
|
// Build regex that captures separators
|
|
var parts []string
|
|
for _, sep := range separators {
|
|
parts = append(parts, regexp.QuoteMeta(sep))
|
|
}
|
|
pattern := "(" + strings.Join(parts, "|") + ")"
|
|
re := regexp.MustCompile(pattern)
|
|
|
|
splits := re.Split(text, -1)
|
|
matches := re.FindAllString(text, -1)
|
|
|
|
var result []string
|
|
for i, s := range splits {
|
|
if s != "" {
|
|
result = append(result, s)
|
|
}
|
|
if i < len(matches) && matches[i] != "" {
|
|
result = append(result, matches[i])
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// runeLen returns the number of runes in s.
|
|
func runeLen(s string) int {
|
|
return utf8.RuneCountInString(s)
|
|
}
|
|
|
|
// SplitText splits text into chunks with overlap, respecting protected patterns.
|
|
func SplitText(text string, cfg SplitterConfig) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
|
|
chunkSize := cfg.ChunkSize
|
|
chunkOverlap := cfg.ChunkOverlap
|
|
separators := cfg.Separators
|
|
|
|
if chunkSize <= 0 {
|
|
chunkSize = 512
|
|
}
|
|
if chunkOverlap < 0 {
|
|
chunkOverlap = 0
|
|
}
|
|
|
|
// Step 1: Find protected spans
|
|
protected := protectedSpans(text)
|
|
|
|
// Step 2: Split non-protected regions by separators, keep protected as atomic units
|
|
units := buildUnitsWithProtection(text, protected, separators)
|
|
|
|
// Step 3: Merge units into chunks with overlap
|
|
return mergeUnits(units, chunkSize, chunkOverlap)
|
|
}
|
|
|
|
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
|
|
// Start/End positions in the returned units are rune offsets (not byte offsets),
|
|
// because downstream merge logic indexes content via []rune slicing.
|
|
// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent
|
|
// creating chunks that are too large for downstream processing (e.g., embedding APIs).
|
|
func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit {
|
|
const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等)
|
|
|
|
var units []splitUnit
|
|
bytePos := 0
|
|
runePos := 0
|
|
|
|
for _, p := range protected {
|
|
if p.start > bytePos {
|
|
pre := text[bytePos:p.start]
|
|
parts := splitBySeparators(pre, separators)
|
|
runeOffset := runePos
|
|
for _, part := range parts {
|
|
partRuneLen := runeLen(part)
|
|
units = append(units, splitUnit{
|
|
text: part,
|
|
start: runeOffset,
|
|
end: runeOffset + partRuneLen,
|
|
})
|
|
runeOffset += partRuneLen
|
|
}
|
|
runePos += runeLen(pre)
|
|
}
|
|
|
|
protText := text[p.start:p.end]
|
|
protRuneLen := runeLen(protText)
|
|
|
|
// If protected content is too large, forcibly split it
|
|
if protRuneLen > maxProtectedSize {
|
|
// Split into smaller chunks at line breaks or spaces
|
|
runes := []rune(protText)
|
|
offset := 0
|
|
for offset < len(runes) {
|
|
chunkEnd := offset + maxProtectedSize
|
|
if chunkEnd > len(runes) {
|
|
chunkEnd = len(runes)
|
|
} else {
|
|
// Try to break at a newline or space
|
|
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
|
if runes[i] == '\n' || runes[i] == ' ' {
|
|
chunkEnd = i + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
chunkText := string(runes[offset:chunkEnd])
|
|
chunkLen := chunkEnd - offset
|
|
units = append(units, splitUnit{
|
|
text: chunkText,
|
|
start: runePos + offset,
|
|
end: runePos + offset + chunkLen,
|
|
})
|
|
offset = chunkEnd
|
|
}
|
|
} else {
|
|
// Normal case: keep protected content as a single unit
|
|
units = append(units, splitUnit{
|
|
text: protText,
|
|
start: runePos,
|
|
end: runePos + protRuneLen,
|
|
})
|
|
}
|
|
runePos += protRuneLen
|
|
bytePos = p.end
|
|
}
|
|
|
|
if bytePos < len(text) {
|
|
remaining := text[bytePos:]
|
|
parts := splitBySeparators(remaining, separators)
|
|
runeOffset := runePos
|
|
for _, part := range parts {
|
|
partRuneLen := runeLen(part)
|
|
units = append(units, splitUnit{
|
|
text: part,
|
|
start: runeOffset,
|
|
end: runeOffset + partRuneLen,
|
|
})
|
|
runeOffset += partRuneLen
|
|
}
|
|
}
|
|
|
|
return units
|
|
}
|
|
|
|
// mergeUnits combines split units into chunks with overlap tracking.
|
|
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
|
|
// Active contextual headers (e.g., Markdown table headers) are prepended to new
|
|
// chunks so that every chunk carries its own header context.
|
|
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
|
if len(units) == 0 {
|
|
return nil
|
|
}
|
|
|
|
const absoluteMaxSize = 7500
|
|
|
|
ht := newHeaderTracker()
|
|
|
|
var chunks []Chunk
|
|
var current []splitUnit
|
|
curLen := 0
|
|
|
|
for _, u := range units {
|
|
uLen := runeLen(u.text)
|
|
|
|
// If this single unit exceeds absolute max, force split it further
|
|
if uLen > absoluteMaxSize {
|
|
// Flush current chunk if any
|
|
if len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
current = nil
|
|
curLen = 0
|
|
}
|
|
|
|
// Update header state even for oversized units
|
|
ht.update(u.text)
|
|
|
|
// Split this oversized unit into smaller chunks
|
|
runes := []rune(u.text)
|
|
offset := 0
|
|
for offset < len(runes) {
|
|
chunkEnd := offset + absoluteMaxSize
|
|
if chunkEnd > len(runes) {
|
|
chunkEnd = len(runes)
|
|
} else {
|
|
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
|
if runes[i] == '\n' || runes[i] == ' ' {
|
|
chunkEnd = i + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
chunkText := string(runes[offset:chunkEnd])
|
|
chunks = append(chunks, Chunk{
|
|
Content: chunkText,
|
|
Seq: len(chunks),
|
|
Start: u.start + offset,
|
|
End: u.start + chunkEnd,
|
|
})
|
|
offset = chunkEnd
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Update header tracking
|
|
ht.update(u.text)
|
|
headers := ht.getHeaders()
|
|
headersLen := runeLen(headers)
|
|
if headersLen > chunkSize {
|
|
headers = ""
|
|
headersLen = 0
|
|
}
|
|
|
|
// If adding this unit (plus reserving space for headers in a potential
|
|
// next chunk) would exceed chunk size, flush the current chunk.
|
|
if curLen+uLen+headersLen > chunkSize && len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
|
|
// Keep overlap from the end of current
|
|
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
|
|
|
|
// Shrink overlap further if needed to fit headers + next unit
|
|
if headers != "" && headersLen+uLen <= chunkSize {
|
|
for len(current) > 0 && curLen+uLen+headersLen > chunkSize {
|
|
curLen -= runeLen(current[0].text)
|
|
current = current[1:]
|
|
}
|
|
|
|
// Prepend headers if the column-name context is not already present
|
|
// in the overlap or the next unit being added.
|
|
overlapText := unitsText(current)
|
|
if !headerAlreadyPresent(headers, overlapText, u.text) {
|
|
startPos := u.start
|
|
if len(current) > 0 {
|
|
startPos = current[0].start
|
|
}
|
|
hUnit := splitUnit{text: headers, start: startPos, end: startPos}
|
|
current = append([]splitUnit{hUnit}, current...)
|
|
curLen += headersLen
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if adding this unit would exceed absolute max
|
|
if curLen+uLen > absoluteMaxSize {
|
|
if len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
current = nil
|
|
curLen = 0
|
|
}
|
|
}
|
|
|
|
current = append(current, u)
|
|
curLen += uLen
|
|
}
|
|
|
|
// Flush remaining
|
|
if len(current) > 0 {
|
|
chunks = append(chunks, buildChunk(current, len(chunks)))
|
|
}
|
|
|
|
return chunks
|
|
}
|
|
|
|
// unitsText concatenates the text of all units.
|
|
func unitsText(units []splitUnit) string {
|
|
var sb strings.Builder
|
|
for _, u := range units {
|
|
sb.WriteString(u.text)
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
// headerAlreadyPresent returns true if the column-name row from the header
|
|
// is already present in the overlap or the next unit, preventing duplication.
|
|
func headerAlreadyPresent(headers, overlapText, unitText string) bool {
|
|
// Fast path: full header already in overlap or unit
|
|
if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) {
|
|
return true
|
|
}
|
|
|
|
// Extract the column-name row (first meaningful non-separator line).
|
|
// For a rewritten header like "| col1 | col2 |\n| --- | --- |\n",
|
|
// the first line is the column names.
|
|
colRow := headerColumnRow(headers)
|
|
if colRow == "" {
|
|
return false
|
|
}
|
|
|
|
return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow)
|
|
}
|
|
|
|
// headerColumnRow extracts the column-name line from a header string.
|
|
// Returns empty string if the header has no meaningful column names.
|
|
func headerColumnRow(header string) string {
|
|
for _, line := range strings.Split(header, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.Contains(line, "---") {
|
|
continue
|
|
}
|
|
// Skip lines that are only pipes/whitespace (empty header rows)
|
|
onlyPipes := true
|
|
for _, r := range line {
|
|
if r != '|' && r != ' ' && r != '\t' {
|
|
onlyPipes = false
|
|
break
|
|
}
|
|
}
|
|
if !onlyPipes {
|
|
return line
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func buildChunk(units []splitUnit, seq int) Chunk {
|
|
var sb strings.Builder
|
|
for _, u := range units {
|
|
sb.WriteString(u.text)
|
|
}
|
|
return Chunk{
|
|
Content: sb.String(),
|
|
Seq: seq,
|
|
Start: units[0].start,
|
|
End: units[len(units)-1].end,
|
|
}
|
|
}
|
|
|
|
// computeOverlap returns the units to keep for overlap and their total rune length.
|
|
func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) {
|
|
if chunkOverlap <= 0 {
|
|
return nil, 0
|
|
}
|
|
|
|
// Walk backward from end, accumulating overlap
|
|
overlapLen := 0
|
|
startIdx := len(current)
|
|
for i := len(current) - 1; i >= 0; i-- {
|
|
uLen := runeLen(current[i].text)
|
|
if overlapLen+uLen > chunkOverlap {
|
|
break
|
|
}
|
|
// Check that overlap + next unit fits in chunk
|
|
if overlapLen+uLen+nextLen > chunkSize {
|
|
break
|
|
}
|
|
overlapLen += uLen
|
|
startIdx = i
|
|
}
|
|
|
|
// Skip leading separator-only and header-marker units in the overlap
|
|
for startIdx < len(current) {
|
|
u := current[startIdx]
|
|
isHeaderMarker := u.start == u.end
|
|
trimmed := strings.TrimSpace(u.text)
|
|
if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) {
|
|
overlapLen -= runeLen(u.text)
|
|
startIdx++
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
|
|
if startIdx >= len(current) {
|
|
return nil, 0
|
|
}
|
|
|
|
overlap := make([]splitUnit, len(current)-startIdx)
|
|
copy(overlap, current[startIdx:])
|
|
return overlap, overlapLen
|
|
}
|
|
|
|
func isSeparatorOnly(s string) bool {
|
|
for _, r := range s {
|
|
if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ParentChildResult holds the two-level chunking output.
|
|
// Parent chunks provide context (large window), child chunks are used for
|
|
// embedding/retrieval (small window). Each child carries its ParentIndex so
|
|
// the caller can wire up ParentChunkID after DB insertion.
|
|
type ParentChildResult struct {
|
|
Parents []Chunk
|
|
Children []ChildChunk
|
|
}
|
|
|
|
// ChildChunk extends Chunk with a reference to its parent.
|
|
type ChildChunk struct {
|
|
Chunk
|
|
ParentIndex int // index into ParentChildResult.Parents
|
|
}
|
|
|
|
// SplitTextParentChild performs two-level chunking:
|
|
// 1. Split text into large parent chunks (parentCfg).
|
|
// 2. Split each parent into smaller child chunks (childCfg) for embedding.
|
|
//
|
|
// The child Seq is globally unique across the entire document.
|
|
func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
|
parents := SplitText(text, parentCfg)
|
|
if len(parents) == 0 {
|
|
return ParentChildResult{}
|
|
}
|
|
|
|
var children []ChildChunk
|
|
childSeq := 0
|
|
for pi, parent := range parents {
|
|
subs := SplitText(parent.Content, childCfg)
|
|
for _, sub := range subs {
|
|
// Adjust offsets: sub positions are relative to parent content,
|
|
// shift to document-level offsets.
|
|
// Use additive shift (not Content-length based) so that chunks with
|
|
// prepended context headers keep correct positional tracking.
|
|
sub.Seq = childSeq
|
|
sub.Start += parent.Start
|
|
sub.End += parent.Start
|
|
children = append(children, ChildChunk{
|
|
Chunk: sub,
|
|
ParentIndex: pi,
|
|
})
|
|
childSeq++
|
|
}
|
|
}
|
|
return ParentChildResult{Parents: parents, Children: children}
|
|
}
|
|
|
|
// ExtractImageRefs extracts markdown image references from text.
|
|
// The URL group supports one level of balanced parentheses so that URLs
|
|
// like https://example.com/item_(abc)/123 are captured in full.
|
|
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
|
|
|
func ExtractImageRefs(text string) []ImageRef {
|
|
text = docparser.UnwrapLinkedImages(text)
|
|
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
|
|
var refs []ImageRef
|
|
for _, m := range matches {
|
|
refs = append(refs, ImageRef{
|
|
OriginalRef: text[m[4]:m[5]], // group 2: URL
|
|
AltText: text[m[2]:m[3]], // group 1: alt text
|
|
Start: m[0],
|
|
End: m[1],
|
|
})
|
|
}
|
|
return refs
|
|
}
|