7b9a831db3
Heuristic and recursive splitters preserve original byte positions in Chunk.Content (the End-Start == RuneCountInString invariant is required by document-reconstruction code paths), which means a chunk sliced at a boundary often carries leading/trailing newlines from the boundary itself. Feeding that whitespace into the embedding model dilutes the vector and wastes tokens for no benefit. TrimSpace the body inside EmbeddingContent for both chunker.Chunk and the two types.Chunk / types.ParsedChunk mirrors. Inner whitespace is preserved; positions/Content are unchanged. Also clean up appendChunk: the previous code computed a trimmed string for the empty-check then stored a separate untrimmed copy, which read as if it might be intentionally divergent. Replace with an explicit 'raw text, skip if pure whitespace' shape and document the invariant.
703 lines
21 KiB
Go
703 lines
21 KiB
Go
// Package chunker implements text splitting for document chunking.
|
||
//
|
||
// Ported from the Python docreader/splitter/splitter.py recursive text splitter.
|
||
package chunker
|
||
|
||
import (
|
||
"regexp"
|
||
"strings"
|
||
"unicode/utf8"
|
||
|
||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||
)
|
||
|
||
// Chunk represents a piece of split text with position tracking.
|
||
//
|
||
// Content holds exactly the text from the original document between Start
|
||
// and End (rune offsets), so End-Start == utf8.RuneCountInString(Content).
|
||
// This invariant is relied on by document-reconstruction code paths
|
||
// (knowledge.go:2278+ for summary generation, UI highlighting, etc.).
|
||
//
|
||
// ContextHeader is a separately-tracked context string (e.g. a Markdown
|
||
// heading breadcrumb) that should be prepended at embedding/retrieval time
|
||
// but is NOT part of Content. Keeping the two apart preserves the
|
||
// position invariant while still letting embedding pipelines see the
|
||
// section context.
|
||
type Chunk struct {
|
||
Content string
|
||
ContextHeader string
|
||
Seq int
|
||
Start int
|
||
End int
|
||
}
|
||
|
||
// EmbeddingContent returns the text that should be fed to the embedding
|
||
// model — the ContextHeader prepended (when set) plus the chunk content.
|
||
// Use this where Content alone would lose semantic context (Tier-1 chunks).
|
||
//
|
||
// Content is returned verbatim from the source document (the End-Start
|
||
// rune-count invariant requires that), but for embedding we trim the
|
||
// surrounding whitespace so leading/trailing newlines from boundary slices
|
||
// don't dilute the embedded vector or waste tokens. Inner whitespace is
|
||
// preserved.
|
||
func (c Chunk) EmbeddingContent() string {
|
||
body := strings.TrimSpace(c.Content)
|
||
if c.ContextHeader == "" {
|
||
return body
|
||
}
|
||
return c.ContextHeader + "\n\n" + body
|
||
}
|
||
|
||
// ImageRef is an image reference found within a chunk's content.
|
||
type ImageRef struct {
|
||
OriginalRef string
|
||
AltText string
|
||
Start int // offset within the chunk content
|
||
End int
|
||
}
|
||
|
||
// SplitterConfig configures the text splitter. Strategy and TokenLimit are
|
||
// honored by the strategy entry point in strategy.go; the legacy SplitText
|
||
// path uses only ChunkSize/Overlap/Separators.
|
||
type SplitterConfig struct {
|
||
ChunkSize int
|
||
ChunkOverlap int
|
||
Separators []string
|
||
|
||
// Strategy selects an adaptive tier. Empty = legacy (backwards-compatible).
|
||
// See strategy.go for valid values.
|
||
Strategy string
|
||
// TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize chars.
|
||
TokenLimit int
|
||
// Languages hints multilingual heuristic patterns. Empty = auto-detect.
|
||
Languages []string
|
||
}
|
||
|
||
// Default chunk sizing constants. Single source of truth for the entire
|
||
// chunker package and (via knowledge.go::buildSplitterConfig) the
|
||
// knowledge service. The frontend KnowledgeBaseEditorModal mirrors these
|
||
// numbers in its initial form state — keep them in sync if you change
|
||
// either value here.
|
||
//
|
||
// DefaultChunkSize = 512 chars: ~100–130 English tokens / ~300 Chinese
|
||
// tokens. Validated as a strong baseline by the Vecta Feb-2026 benchmark
|
||
// across 50 academic papers. Use 200–400 for FAQ-style atomic content,
|
||
// 1000–2000 for narrative / argumentative documents.
|
||
//
|
||
// DefaultChunkOverlap = 80 chars (≈15% of DefaultChunkSize): community-
|
||
// recommended sweet spot between recall (an answer split across a
|
||
// boundary needs overlap to be retrievable) and storage cost. Use 0 for
|
||
// strictly atomic data (FAQ, JSON records), 150–200 for long narratives
|
||
// where reasoning crosses chunks.
|
||
//
|
||
// MIGRATION NOTE: Prior versions had three different overlap defaults
|
||
// (Go DefaultConfig: 64, knowledge.go buildSplitterConfig: 50, Python
|
||
// docreader: 100). All consolidated to 80 here.
|
||
//
|
||
// Existing knowledge bases that stored ChunkOverlap=0 in the DB pick
|
||
// this 80 up on next re-index; their previously-indexed embeddings will
|
||
// not match new ones bit-for-bit. Recall stays similar but search
|
||
// ranking can shift slightly. To freeze the old behavior on a per-KB
|
||
// basis, explicitly set ChunkingConfig.ChunkOverlap to 64 before
|
||
// re-indexing.
|
||
const (
|
||
DefaultChunkSize = 512
|
||
DefaultChunkOverlap = 80
|
||
)
|
||
|
||
// DefaultConfig returns sensible defaults.
|
||
func DefaultConfig() SplitterConfig {
|
||
return SplitterConfig{
|
||
ChunkSize: DefaultChunkSize,
|
||
ChunkOverlap: DefaultChunkOverlap,
|
||
Separators: []string{"\n\n", "\n", "。"},
|
||
}
|
||
}
|
||
|
||
// protectedPatterns are regex patterns for content that must not be split.
|
||
var protectedPatterns = []*regexp.Regexp{
|
||
regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
|
||
regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
|
||
regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
|
||
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
|
||
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
|
||
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
|
||
}
|
||
|
||
type span struct {
|
||
start, end int
|
||
}
|
||
|
||
// protectedSpansRune converts byte-offset protected spans to rune offsets
|
||
// in a single forward pass over text. Used by callers that work in rune
|
||
// space (e.g. the heuristic splitter) to avoid choosing chunk boundaries
|
||
// that cut through protected content. byteSpans must be sorted by start
|
||
// (protectedSpans guarantees this).
|
||
func protectedSpansRune(text string, byteSpans []span) []span {
|
||
if len(byteSpans) == 0 {
|
||
return nil
|
||
}
|
||
out := make([]span, 0, len(byteSpans))
|
||
runeIdx := 0
|
||
byteIdx := 0
|
||
for _, s := range byteSpans {
|
||
for byteIdx < s.start && byteIdx < len(text) {
|
||
_, size := utf8.DecodeRuneInString(text[byteIdx:])
|
||
byteIdx += size
|
||
runeIdx++
|
||
}
|
||
startRune := runeIdx
|
||
for byteIdx < s.end && byteIdx < len(text) {
|
||
_, size := utf8.DecodeRuneInString(text[byteIdx:])
|
||
byteIdx += size
|
||
runeIdx++
|
||
}
|
||
out = append(out, span{start: startRune, end: runeIdx})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// protectedSpans finds all non-overlapping protected regions in text.
|
||
func protectedSpans(text string) []span {
|
||
type match struct {
|
||
start, end int
|
||
}
|
||
var all []match
|
||
for _, pat := range protectedPatterns {
|
||
locs := pat.FindAllStringIndex(text, -1)
|
||
for _, loc := range locs {
|
||
if loc[1]-loc[0] > 0 {
|
||
all = append(all, match{loc[0], loc[1]})
|
||
}
|
||
}
|
||
}
|
||
if len(all) == 0 {
|
||
return nil
|
||
}
|
||
|
||
// Sort by start, then by length descending
|
||
for i := 1; i < len(all); i++ {
|
||
for j := i; j > 0; j-- {
|
||
if all[j].start < all[j-1].start ||
|
||
(all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {
|
||
all[j], all[j-1] = all[j-1], all[j]
|
||
} else {
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
// Remove overlaps
|
||
var result []span
|
||
lastEnd := 0
|
||
for _, m := range all {
|
||
if m.start >= lastEnd {
|
||
result = append(result, span{m.start, m.end})
|
||
lastEnd = m.end
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// splitUnit is a piece of text with its original position.
|
||
type splitUnit struct {
|
||
text string
|
||
start, end int
|
||
}
|
||
|
||
// splitBySeparators splits text by separators in priority order, recursively
|
||
// applying the next separator to any piece that is still larger than
|
||
// chunkSize. Mirrors the recursive priority semantics of the Python
|
||
// reference splitter (docreader/splitter/splitter.py:_split): if `\n\n`
|
||
// produces a piece that's still too big, `\n` (and subsequent separators)
|
||
// are applied within that piece — not to the whole text.
|
||
//
|
||
// chunkSize == 0 disables the recursion guard; callers that don't care
|
||
// about size budget (e.g. a final mergeUnits-style pass) pass 0.
|
||
func splitBySeparators(text string, separators []string, chunkSize int) []string {
|
||
if text == "" || len(separators) == 0 {
|
||
return []string{text}
|
||
}
|
||
if chunkSize > 0 && runeLen(text) <= chunkSize {
|
||
return []string{text}
|
||
}
|
||
|
||
for i, sep := range separators {
|
||
if sep == "" {
|
||
continue
|
||
}
|
||
re := regexp.MustCompile("(" + regexp.QuoteMeta(sep) + ")")
|
||
splits := re.Split(text, -1)
|
||
matches := re.FindAllString(text, -1)
|
||
if len(matches) == 0 {
|
||
continue
|
||
}
|
||
|
||
var pieces []string
|
||
for j, s := range splits {
|
||
if s != "" {
|
||
pieces = append(pieces, s)
|
||
}
|
||
if j < len(matches) && matches[j] != "" {
|
||
pieces = append(pieces, matches[j])
|
||
}
|
||
}
|
||
if len(pieces) <= 1 {
|
||
continue
|
||
}
|
||
|
||
// Recursively split any piece that is still too large with the
|
||
// remaining (lower-priority) separators.
|
||
var out []string
|
||
remaining := separators[i+1:]
|
||
for _, p := range pieces {
|
||
if chunkSize > 0 && runeLen(p) > chunkSize && len(remaining) > 0 {
|
||
out = append(out, splitBySeparators(p, remaining, chunkSize)...)
|
||
} else {
|
||
out = append(out, p)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
return []string{text}
|
||
}
|
||
|
||
// runeLen returns the number of runes in s.
|
||
func runeLen(s string) int {
|
||
return utf8.RuneCountInString(s)
|
||
}
|
||
|
||
// SplitText splits text into chunks with overlap, respecting protected patterns.
|
||
func SplitText(text string, cfg SplitterConfig) []Chunk {
|
||
if text == "" {
|
||
return nil
|
||
}
|
||
|
||
chunkSize := cfg.ChunkSize
|
||
chunkOverlap := cfg.ChunkOverlap
|
||
separators := cfg.Separators
|
||
|
||
if chunkSize <= 0 {
|
||
chunkSize = 512
|
||
}
|
||
if chunkOverlap < 0 {
|
||
chunkOverlap = 0
|
||
}
|
||
|
||
// Step 1: Find protected spans
|
||
protected := protectedSpans(text)
|
||
|
||
// Step 2: Split non-protected regions by separators, keep protected as atomic units.
|
||
// chunkSize is forwarded so splitBySeparators can recursively apply lower-priority
|
||
// separators to oversize pieces (Python-parity recursive split).
|
||
units := buildUnitsWithProtection(text, protected, separators, chunkSize)
|
||
|
||
// Step 3: Merge units into chunks with overlap
|
||
return mergeUnits(units, chunkSize, chunkOverlap)
|
||
}
|
||
|
||
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
|
||
// Start/End positions in the returned units are rune offsets (not byte offsets),
|
||
// because downstream merge logic indexes content via []rune slicing.
|
||
// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent
|
||
// creating chunks that are too large for downstream processing (e.g., embedding APIs).
|
||
// chunkSize is forwarded to splitBySeparators so recursive splitting can keep pieces
|
||
// under the budget when one separator alone leaves a piece oversize.
|
||
func buildUnitsWithProtection(text string, protected []span, separators []string, chunkSize int) []splitUnit {
|
||
const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等)
|
||
|
||
var units []splitUnit
|
||
bytePos := 0
|
||
runePos := 0
|
||
|
||
for _, p := range protected {
|
||
if p.start > bytePos {
|
||
pre := text[bytePos:p.start]
|
||
parts := splitBySeparators(pre, separators, chunkSize)
|
||
runeOffset := runePos
|
||
for _, part := range parts {
|
||
partRuneLen := runeLen(part)
|
||
units = append(units, splitUnit{
|
||
text: part,
|
||
start: runeOffset,
|
||
end: runeOffset + partRuneLen,
|
||
})
|
||
runeOffset += partRuneLen
|
||
}
|
||
runePos += runeLen(pre)
|
||
}
|
||
|
||
protText := text[p.start:p.end]
|
||
protRuneLen := runeLen(protText)
|
||
|
||
// If protected content is too large, forcibly split it
|
||
if protRuneLen > maxProtectedSize {
|
||
// Split into smaller chunks at line breaks or spaces
|
||
runes := []rune(protText)
|
||
offset := 0
|
||
for offset < len(runes) {
|
||
chunkEnd := offset + maxProtectedSize
|
||
if chunkEnd > len(runes) {
|
||
chunkEnd = len(runes)
|
||
} else {
|
||
// Try to break at a newline or space
|
||
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
||
if runes[i] == '\n' || runes[i] == ' ' {
|
||
chunkEnd = i + 1
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
chunkText := string(runes[offset:chunkEnd])
|
||
chunkLen := chunkEnd - offset
|
||
units = append(units, splitUnit{
|
||
text: chunkText,
|
||
start: runePos + offset,
|
||
end: runePos + offset + chunkLen,
|
||
})
|
||
offset = chunkEnd
|
||
}
|
||
} else {
|
||
// Normal case: keep protected content as a single unit
|
||
units = append(units, splitUnit{
|
||
text: protText,
|
||
start: runePos,
|
||
end: runePos + protRuneLen,
|
||
})
|
||
}
|
||
runePos += protRuneLen
|
||
bytePos = p.end
|
||
}
|
||
|
||
if bytePos < len(text) {
|
||
remaining := text[bytePos:]
|
||
parts := splitBySeparators(remaining, separators, chunkSize)
|
||
runeOffset := runePos
|
||
for _, part := range parts {
|
||
partRuneLen := runeLen(part)
|
||
units = append(units, splitUnit{
|
||
text: part,
|
||
start: runeOffset,
|
||
end: runeOffset + partRuneLen,
|
||
})
|
||
runeOffset += partRuneLen
|
||
}
|
||
}
|
||
|
||
return units
|
||
}
|
||
|
||
// mergeUnits combines split units into chunks with overlap tracking.
|
||
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
|
||
// Active contextual headers (e.g., Markdown table headers) are prepended to new
|
||
// chunks so that every chunk carries its own header context.
|
||
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||
if len(units) == 0 {
|
||
return nil
|
||
}
|
||
|
||
const absoluteMaxSize = 7500
|
||
|
||
ht := newHeaderTracker()
|
||
|
||
var chunks []Chunk
|
||
var current []splitUnit
|
||
curLen := 0
|
||
|
||
for _, u := range units {
|
||
uLen := runeLen(u.text)
|
||
|
||
// If this single unit exceeds absolute max, force split it further
|
||
if uLen > absoluteMaxSize {
|
||
// Flush current chunk if any
|
||
if len(current) > 0 {
|
||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||
current = nil
|
||
curLen = 0
|
||
}
|
||
|
||
// Update header state even for oversized units
|
||
ht.update(u.text)
|
||
|
||
// Split this oversized unit into smaller chunks
|
||
runes := []rune(u.text)
|
||
offset := 0
|
||
for offset < len(runes) {
|
||
chunkEnd := offset + absoluteMaxSize
|
||
if chunkEnd > len(runes) {
|
||
chunkEnd = len(runes)
|
||
} else {
|
||
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
||
if runes[i] == '\n' || runes[i] == ' ' {
|
||
chunkEnd = i + 1
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
chunkText := string(runes[offset:chunkEnd])
|
||
chunks = append(chunks, Chunk{
|
||
Content: chunkText,
|
||
Seq: len(chunks),
|
||
Start: u.start + offset,
|
||
End: u.start + chunkEnd,
|
||
})
|
||
offset = chunkEnd
|
||
}
|
||
continue
|
||
}
|
||
|
||
// Update header tracking
|
||
ht.update(u.text)
|
||
headers := ht.getHeaders()
|
||
headersLen := runeLen(headers)
|
||
if headersLen > chunkSize {
|
||
headers = ""
|
||
headersLen = 0
|
||
}
|
||
|
||
// If adding this unit (plus reserving space for headers in a potential
|
||
// next chunk) would exceed chunk size, flush the current chunk.
|
||
if curLen+uLen+headersLen > chunkSize && len(current) > 0 {
|
||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||
|
||
// Keep overlap from the end of current
|
||
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
|
||
|
||
// Shrink overlap further if needed to fit headers + next unit
|
||
if headers != "" && headersLen+uLen <= chunkSize {
|
||
for len(current) > 0 && curLen+uLen+headersLen > chunkSize {
|
||
curLen -= runeLen(current[0].text)
|
||
current = current[1:]
|
||
}
|
||
|
||
// Prepend headers if the column-name context is not already present
|
||
// in the overlap or the next unit being added.
|
||
overlapText := unitsText(current)
|
||
if !headerAlreadyPresent(headers, overlapText, u.text) {
|
||
startPos := u.start
|
||
if len(current) > 0 {
|
||
startPos = current[0].start
|
||
}
|
||
hUnit := splitUnit{text: headers, start: startPos, end: startPos}
|
||
current = append([]splitUnit{hUnit}, current...)
|
||
curLen += headersLen
|
||
}
|
||
}
|
||
}
|
||
|
||
// Check if adding this unit would exceed absolute max
|
||
if curLen+uLen > absoluteMaxSize {
|
||
if len(current) > 0 {
|
||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||
current = nil
|
||
curLen = 0
|
||
}
|
||
}
|
||
|
||
current = append(current, u)
|
||
curLen += uLen
|
||
}
|
||
|
||
// Flush remaining
|
||
if len(current) > 0 {
|
||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||
}
|
||
|
||
return chunks
|
||
}
|
||
|
||
// unitsText concatenates the text of all units.
|
||
func unitsText(units []splitUnit) string {
|
||
var sb strings.Builder
|
||
for _, u := range units {
|
||
sb.WriteString(u.text)
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
// headerAlreadyPresent returns true if the column-name row from the header
|
||
// is already present in the overlap or the next unit, preventing duplication.
|
||
func headerAlreadyPresent(headers, overlapText, unitText string) bool {
|
||
// Fast path: full header already in overlap or unit
|
||
if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) {
|
||
return true
|
||
}
|
||
|
||
// Extract the column-name row (first meaningful non-separator line).
|
||
// For a rewritten header like "| col1 | col2 |\n| --- | --- |\n",
|
||
// the first line is the column names.
|
||
colRow := headerColumnRow(headers)
|
||
if colRow == "" {
|
||
return false
|
||
}
|
||
|
||
return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow)
|
||
}
|
||
|
||
// headerColumnRow extracts the column-name line from a header string.
|
||
// Returns empty string if the header has no meaningful column names.
|
||
func headerColumnRow(header string) string {
|
||
for _, line := range strings.Split(header, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" || strings.Contains(line, "---") {
|
||
continue
|
||
}
|
||
// Skip lines that are only pipes/whitespace (empty header rows)
|
||
onlyPipes := true
|
||
for _, r := range line {
|
||
if r != '|' && r != ' ' && r != '\t' {
|
||
onlyPipes = false
|
||
break
|
||
}
|
||
}
|
||
if !onlyPipes {
|
||
return line
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func buildChunk(units []splitUnit, seq int) Chunk {
|
||
var sb strings.Builder
|
||
for _, u := range units {
|
||
sb.WriteString(u.text)
|
||
}
|
||
return Chunk{
|
||
Content: sb.String(),
|
||
Seq: seq,
|
||
Start: units[0].start,
|
||
End: units[len(units)-1].end,
|
||
}
|
||
}
|
||
|
||
// computeOverlap returns the units to keep for overlap and their total rune length.
|
||
func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) {
|
||
if chunkOverlap <= 0 {
|
||
return nil, 0
|
||
}
|
||
|
||
// Walk backward from end, accumulating overlap
|
||
overlapLen := 0
|
||
startIdx := len(current)
|
||
for i := len(current) - 1; i >= 0; i-- {
|
||
uLen := runeLen(current[i].text)
|
||
if overlapLen+uLen > chunkOverlap {
|
||
break
|
||
}
|
||
// Check that overlap + next unit fits in chunk
|
||
if overlapLen+uLen+nextLen > chunkSize {
|
||
break
|
||
}
|
||
overlapLen += uLen
|
||
startIdx = i
|
||
}
|
||
|
||
// Skip leading separator-only and header-marker units in the overlap
|
||
for startIdx < len(current) {
|
||
u := current[startIdx]
|
||
isHeaderMarker := u.start == u.end
|
||
trimmed := strings.TrimSpace(u.text)
|
||
if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) {
|
||
overlapLen -= runeLen(u.text)
|
||
startIdx++
|
||
} else {
|
||
break
|
||
}
|
||
}
|
||
|
||
if startIdx >= len(current) {
|
||
return nil, 0
|
||
}
|
||
|
||
overlap := make([]splitUnit, len(current)-startIdx)
|
||
copy(overlap, current[startIdx:])
|
||
return overlap, overlapLen
|
||
}
|
||
|
||
func isSeparatorOnly(s string) bool {
|
||
for _, r := range s {
|
||
if r != '\n' && r != '\r' && r != ' ' && r != '\t' && r != '。' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// ParentChildResult holds the two-level chunking output.
|
||
// Parent chunks provide context (large window), child chunks are used for
|
||
// embedding/retrieval (small window). Each child carries its ParentIndex so
|
||
// the caller can wire up ParentChunkID after DB insertion.
|
||
type ParentChildResult struct {
|
||
Parents []Chunk
|
||
Children []ChildChunk
|
||
}
|
||
|
||
// ChildChunk extends Chunk with a reference to its parent.
|
||
type ChildChunk struct {
|
||
Chunk
|
||
ParentIndex int // index into ParentChildResult.Parents
|
||
}
|
||
|
||
// SplitTextParentChild performs two-level chunking:
|
||
// 1. Split text into large parent chunks (parentCfg).
|
||
// 2. Split each parent into smaller child chunks (childCfg) for embedding.
|
||
//
|
||
// The child Seq is globally unique across the entire document.
|
||
func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
||
parents := SplitText(text, parentCfg)
|
||
if len(parents) == 0 {
|
||
return ParentChildResult{}
|
||
}
|
||
|
||
var newParents []Chunk
|
||
var children []ChildChunk
|
||
childSeq := 0
|
||
for _, parent := range parents {
|
||
subs := SplitText(parent.Content, childCfg)
|
||
|
||
parentIndex := -1
|
||
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
|
||
parentIndex = len(newParents)
|
||
newParents = append(newParents, parent)
|
||
}
|
||
|
||
for _, sub := range subs {
|
||
// Adjust offsets: sub positions are relative to parent content,
|
||
// shift to document-level offsets.
|
||
// Use additive shift (not Content-length based) so that chunks with
|
||
// prepended context headers keep correct positional tracking.
|
||
sub.Seq = childSeq
|
||
sub.Start += parent.Start
|
||
sub.End += parent.Start
|
||
children = append(children, ChildChunk{
|
||
Chunk: sub,
|
||
ParentIndex: parentIndex,
|
||
})
|
||
childSeq++
|
||
}
|
||
}
|
||
return ParentChildResult{Parents: newParents, Children: children}
|
||
}
|
||
|
||
// ExtractImageRefs extracts markdown image references from text.
|
||
// The URL group supports one level of balanced parentheses so that URLs
|
||
// like https://example.com/item_(abc)/123 are captured in full.
|
||
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||
|
||
func ExtractImageRefs(text string) []ImageRef {
|
||
text = docparser.UnwrapLinkedImages(text)
|
||
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
|
||
var refs []ImageRef
|
||
for _, m := range matches {
|
||
refs = append(refs, ImageRef{
|
||
OriginalRef: text[m[4]:m[5]], // group 2: URL
|
||
AltText: text[m[2]:m[3]], // group 1: alt text
|
||
Start: m[0],
|
||
End: m[1],
|
||
})
|
||
}
|
||
return refs
|
||
}
|