Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/splitter.go
T
wizardchen 397689d2f3 feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
2026-03-02 21:21:49 +08:00

335 lines
8.3 KiB
Go

// Package chunker implements text splitting for document chunking.
//
// Ported from the Python docreader/splitter/splitter.py recursive text splitter.
package chunker
import (
"regexp"
"strings"
"unicode/utf8"
)
// Chunk represents a piece of split text with position tracking.
type Chunk struct {
Content string
Seq int
Start int
End int
}
// ImageRef is an image reference found within a chunk's content.
type ImageRef struct {
OriginalRef string
AltText string
Start int // offset within the chunk content
End int
}
// SplitterConfig configures the text splitter.
type SplitterConfig struct {
ChunkSize int
ChunkOverlap int
Separators []string
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() SplitterConfig {
return SplitterConfig{
ChunkSize: 512,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
}
}
// protectedPatterns are regex patterns for content that must not be split.
var protectedPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
}
type span struct {
start, end int
}
// protectedSpans finds all non-overlapping protected regions in text.
func protectedSpans(text string) []span {
type match struct {
start, end int
}
var all []match
for _, pat := range protectedPatterns {
locs := pat.FindAllStringIndex(text, -1)
for _, loc := range locs {
if loc[1]-loc[0] > 0 {
all = append(all, match{loc[0], loc[1]})
}
}
}
if len(all) == 0 {
return nil
}
// Sort by start, then by length descending
for i := 1; i < len(all); i++ {
for j := i; j > 0; j-- {
if all[j].start < all[j-1].start ||
(all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {
all[j], all[j-1] = all[j-1], all[j]
} else {
break
}
}
}
// Remove overlaps
var result []span
lastEnd := 0
for _, m := range all {
if m.start >= lastEnd {
result = append(result, span{m.start, m.end})
lastEnd = m.end
}
}
return result
}
// splitUnit is a piece of text with its original position.
type splitUnit struct {
text string
start, end int
}
// splitBySeparators splits text by separators in priority order, keeping separators.
func splitBySeparators(text string, separators []string) []string {
if len(separators) == 0 || text == "" {
return []string{text}
}
// Build regex that captures separators
var parts []string
for _, sep := range separators {
parts = append(parts, regexp.QuoteMeta(sep))
}
pattern := "(" + strings.Join(parts, "|") + ")"
re := regexp.MustCompile(pattern)
splits := re.Split(text, -1)
matches := re.FindAllString(text, -1)
var result []string
for i, s := range splits {
if s != "" {
result = append(result, s)
}
if i < len(matches) && matches[i] != "" {
result = append(result, matches[i])
}
}
return result
}
// runeLen returns the number of runes in s.
func runeLen(s string) int {
return utf8.RuneCountInString(s)
}
// SplitText splits text into chunks with overlap, respecting protected patterns.
func SplitText(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
chunkSize := cfg.ChunkSize
chunkOverlap := cfg.ChunkOverlap
separators := cfg.Separators
if chunkSize <= 0 {
chunkSize = 512
}
if chunkOverlap < 0 {
chunkOverlap = 0
}
// Step 1: Find protected spans
protected := protectedSpans(text)
// Step 2: Split non-protected regions by separators, keep protected as atomic units
units := buildUnitsWithProtection(text, protected, separators)
// Step 3: Merge units into chunks with overlap
return mergeUnits(units, chunkSize, chunkOverlap)
}
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
// Start/End positions in the returned units are rune offsets (not byte offsets),
// because downstream merge logic indexes content via []rune slicing.
func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit {
var units []splitUnit
bytePos := 0
runePos := 0
for _, p := range protected {
if p.start > bytePos {
pre := text[bytePos:p.start]
parts := splitBySeparators(pre, separators)
runeOffset := runePos
for _, part := range parts {
partRuneLen := runeLen(part)
units = append(units, splitUnit{
text: part,
start: runeOffset,
end: runeOffset + partRuneLen,
})
runeOffset += partRuneLen
}
runePos += runeLen(pre)
}
protText := text[p.start:p.end]
protRuneLen := runeLen(protText)
units = append(units, splitUnit{
text: protText,
start: runePos,
end: runePos + protRuneLen,
})
runePos += protRuneLen
bytePos = p.end
}
if bytePos < len(text) {
remaining := text[bytePos:]
parts := splitBySeparators(remaining, separators)
runeOffset := runePos
for _, part := range parts {
partRuneLen := runeLen(part)
units = append(units, splitUnit{
text: part,
start: runeOffset,
end: runeOffset + partRuneLen,
})
runeOffset += partRuneLen
}
}
return units
}
// mergeUnits combines split units into chunks with overlap tracking.
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
if len(units) == 0 {
return nil
}
var chunks []Chunk
var current []splitUnit
curLen := 0
for _, u := range units {
uLen := runeLen(u.text)
// If adding this unit exceeds chunk size and we have content, flush
if curLen+uLen > chunkSize && len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
// Keep overlap from the end of current
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
}
current = append(current, u)
curLen += uLen
}
// Flush remaining
if len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
}
return chunks
}
func buildChunk(units []splitUnit, seq int) Chunk {
var sb strings.Builder
for _, u := range units {
sb.WriteString(u.text)
}
return Chunk{
Content: sb.String(),
Seq: seq,
Start: units[0].start,
End: units[len(units)-1].end,
}
}
// computeOverlap returns the units to keep for overlap and their total rune length.
func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) {
if chunkOverlap <= 0 {
return nil, 0
}
// Walk backward from end, accumulating overlap
overlapLen := 0
startIdx := len(current)
for i := len(current) - 1; i >= 0; i-- {
uLen := runeLen(current[i].text)
if overlapLen+uLen > chunkOverlap {
break
}
// Check that overlap + next unit fits in chunk
if overlapLen+uLen+nextLen > chunkSize {
break
}
overlapLen += uLen
startIdx = i
}
// Skip leading separators-only units in the overlap
for startIdx < len(current) {
u := current[startIdx]
trimmed := strings.TrimSpace(u.text)
if trimmed == "" || isSeparatorOnly(u.text) {
overlapLen -= runeLen(u.text)
startIdx++
} else {
break
}
}
if startIdx >= len(current) {
return nil, 0
}
overlap := make([]splitUnit, len(current)-startIdx)
copy(overlap, current[startIdx:])
return overlap, overlapLen
}
func isSeparatorOnly(s string) bool {
for _, r := range s {
if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' {
return false
}
}
return true
}
// ExtractImageRefs extracts markdown image references from text.
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
func ExtractImageRefs(text string) []ImageRef {
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
var refs []ImageRef
for _, m := range matches {
refs = append(refs, ImageRef{
OriginalRef: text[m[4]:m[5]], // group 2: URL
AltText: text[m[2]:m[3]], // group 1: alt text
Start: m[0],
End: m[1],
})
}
return refs
}