feat(chunker): implement header tracking for Markdown tables during chunking

- Added headerTracker to preserve context of Markdown table headers across split chunks.
- Implemented logic to prepend headers to subsequent chunks after the first.
- Enhanced merging logic to ensure headers are not duplicated in overlapping content.
- Added comprehensive tests for header tracking and chunking behavior with Markdown tables.
This commit is contained in:
wizardchen
2026-04-01 13:35:02 +08:00
committed by lyingbug
parent bc0c2d23d5
commit ce7df624fb
3 changed files with 878 additions and 8 deletions
@@ -0,0 +1,161 @@
// Package chunker - header_tracker.go implements context-preserving header
// tracking for document chunking, ported from docreader/splitter/header_hook.py.
//
// When a large Markdown table is split across multiple chunks, each chunk after
// the first would lose the table header context. The headerTracker detects table
// headers and signals the merge logic to prepend them to subsequent chunks.
package chunker
import (
"regexp"
"sort"
"strings"
)
// headerTrackerHook defines a pattern pair for detecting contextual headers.
// When startPattern matches a unit's text, that text becomes an "active header".
// The header stays active until endPattern matches a subsequent unit.
type headerTrackerHook struct {
startPattern *regexp.Regexp
endPattern *regexp.Regexp
priority int
}
// defaultHeaderHooks returns header tracking hooks matching the Python defaults
// in docreader/splitter/header_hook.py.
var defaultHeaderHooks = []headerTrackerHook{
{
// Markdown table: header row + separator row (e.g. "| A | B |\n| --- | --- |\n")
startPattern: regexp.MustCompile(`(?si)^\s*(?:\|[^|\n]*)+[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|?[\r\n]+$`),
// Empty/whitespace line or a line that doesn't start with | or whitespace
endPattern: regexp.MustCompile(`(?si)^\s*$|^\s*[^|\s].*$`),
priority: 15,
},
}
// tableRowPattern matches a single Markdown table row: "| cell | cell | ... |\n"
var tableRowPattern = regexp.MustCompile(`(?m)^\s*(?:\|[^|\n]*)+\|\s*$`)
// headerTracker maintains the state of active headers across split units.
type headerTracker struct {
hooks []headerTrackerHook
activeHeaders map[int]string // priority -> header text
endedHeaders map[int]bool // priorities that have been ended
pendingExtend map[int]bool // headers with empty column names awaiting first data row
}
func newHeaderTracker() *headerTracker {
return &headerTracker{
hooks: defaultHeaderHooks,
activeHeaders: make(map[int]string),
endedHeaders: make(map[int]bool),
pendingExtend: make(map[int]bool),
}
}
// update checks split text for header start/end markers and updates internal state.
func (ht *headerTracker) update(split string) {
// 1. Check for header-end markers among currently active headers
for _, hook := range ht.hooks {
if _, active := ht.activeHeaders[hook.priority]; active {
if hook.endPattern.MatchString(split) {
ht.endedHeaders[hook.priority] = true
delete(ht.activeHeaders, hook.priority)
delete(ht.pendingExtend, hook.priority)
}
}
}
// 2. If a header has an empty column-name row (e.g. "||"), replace it with
// a proper Markdown table header using the first data row as column names.
//
// Before: "||" + "| --- | --- |\n"
// After: "| col1 | col2 |\n" + "| --- | --- |\n"
for p := range ht.pendingExtend {
if _, active := ht.activeHeaders[p]; active && tableRowPattern.MatchString(split) {
sep := extractSeparatorLine(ht.activeHeaders[p])
ht.activeHeaders[p] = split + sep
}
delete(ht.pendingExtend, p)
}
// 3. Check for new header-start markers (only for hooks that are neither active nor ended)
for _, hook := range ht.hooks {
if _, active := ht.activeHeaders[hook.priority]; active {
continue
}
if ht.endedHeaders[hook.priority] {
continue
}
if loc := hook.startPattern.FindString(split); loc != "" {
ht.activeHeaders[hook.priority] = loc
if isEmptyTableHeaderRow(loc) {
ht.pendingExtend[hook.priority] = true
}
}
}
// 4. If all headers ended, clear the ended set so future tables can be tracked
if len(ht.activeHeaders) == 0 {
for k := range ht.endedHeaders {
delete(ht.endedHeaders, k)
}
}
}
// getHeaders returns all active headers concatenated, sorted by priority descending.
func (ht *headerTracker) getHeaders() string {
if len(ht.activeHeaders) == 0 {
return ""
}
type entry struct {
priority int
text string
}
entries := make([]entry, 0, len(ht.activeHeaders))
for p, t := range ht.activeHeaders {
entries = append(entries, entry{p, t})
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].priority > entries[j].priority
})
parts := make([]string, len(entries))
for i, e := range entries {
parts[i] = e.text
}
return strings.Join(parts, "\n")
}
// isEmptyTableHeaderRow checks if the header row (the line before the separator)
// contains only pipes and whitespace — meaning the column names are empty.
// This is common with MarkItDown and similar converters that produce tables like:
//
// ||
// | --- | --- |
// | real column A | real column B |
func isEmptyTableHeaderRow(header string) bool {
idx := strings.IndexByte(header, '\n')
if idx < 0 {
return false
}
row := strings.TrimSpace(header[:idx])
for _, r := range row {
if r != '|' && r != ' ' && r != '\t' {
return false
}
}
return true
}
// extractSeparatorLine returns the separator line (e.g. "| --- | --- |\n") from
// a table header string. It looks for the line containing "---".
func extractSeparatorLine(header string) string {
for _, line := range strings.Split(header, "\n") {
if strings.Contains(line, "---") {
return line + "\n"
}
}
return ""
}
+97 -8
View File
@@ -258,14 +258,17 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
// mergeUnits combines split units into chunks with overlap tracking.
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
// Active contextual headers (e.g., Markdown table headers) are prepended to new
// chunks so that every chunk carries its own header context.
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
if len(units) == 0 {
return nil
}
// Absolute maximum chunk size (留余量给标题等额外内容)
const absoluteMaxSize = 7500
ht := newHeaderTracker()
var chunks []Chunk
var current []splitUnit
curLen := 0
@@ -282,6 +285,9 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
curLen = 0
}
// Update header state even for oversized units
ht.update(u.text)
// Split this oversized unit into smaller chunks
runes := []rune(u.text)
offset := 0
@@ -290,7 +296,6 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
if chunkEnd > len(runes) {
chunkEnd = len(runes)
} else {
// Try to break at a newline or space
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
if runes[i] == '\n' || runes[i] == ' ' {
chunkEnd = i + 1
@@ -311,17 +316,47 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
continue
}
// If adding this unit exceeds chunk size and we have content, flush
if curLen+uLen > chunkSize && len(current) > 0 {
// Update header tracking
ht.update(u.text)
headers := ht.getHeaders()
headersLen := runeLen(headers)
if headersLen > chunkSize {
headers = ""
headersLen = 0
}
// If adding this unit (plus reserving space for headers in a potential
// next chunk) would exceed chunk size, flush the current chunk.
if curLen+uLen+headersLen > chunkSize && len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
// Keep overlap from the end of current
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
// Shrink overlap further if needed to fit headers + next unit
if headers != "" && headersLen+uLen <= chunkSize {
for len(current) > 0 && curLen+uLen+headersLen > chunkSize {
curLen -= runeLen(current[0].text)
current = current[1:]
}
// Prepend headers if the column-name context is not already present
// in the overlap or the next unit being added.
overlapText := unitsText(current)
if !headerAlreadyPresent(headers, overlapText, u.text) {
startPos := u.start
if len(current) > 0 {
startPos = current[0].start
}
hUnit := splitUnit{text: headers, start: startPos, end: startPos}
current = append([]splitUnit{hUnit}, current...)
curLen += headersLen
}
}
}
// Check if adding this unit would exceed absolute max
if curLen+uLen > absoluteMaxSize {
// Flush current and start fresh
if len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
current = nil
@@ -341,6 +376,57 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
return chunks
}
// unitsText concatenates the text of all units.
func unitsText(units []splitUnit) string {
var sb strings.Builder
for _, u := range units {
sb.WriteString(u.text)
}
return sb.String()
}
// headerAlreadyPresent returns true if the column-name row from the header
// is already present in the overlap or the next unit, preventing duplication.
func headerAlreadyPresent(headers, overlapText, unitText string) bool {
// Fast path: full header already in overlap or unit
if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) {
return true
}
// Extract the column-name row (first meaningful non-separator line).
// For a rewritten header like "| col1 | col2 |\n| --- | --- |\n",
// the first line is the column names.
colRow := headerColumnRow(headers)
if colRow == "" {
return false
}
return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow)
}
// headerColumnRow extracts the column-name line from a header string.
// Returns empty string if the header has no meaningful column names.
func headerColumnRow(header string) string {
for _, line := range strings.Split(header, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.Contains(line, "---") {
continue
}
// Skip lines that are only pipes/whitespace (empty header rows)
onlyPipes := true
for _, r := range line {
if r != '|' && r != ' ' && r != '\t' {
onlyPipes = false
break
}
}
if !onlyPipes {
return line
}
}
return ""
}
func buildChunk(units []splitUnit, seq int) Chunk {
var sb strings.Builder
for _, u := range units {
@@ -376,11 +462,12 @@ func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) (
startIdx = i
}
// Skip leading separators-only units in the overlap
// Skip leading separator-only and header-marker units in the overlap
for startIdx < len(current) {
u := current[startIdx]
isHeaderMarker := u.start == u.end
trimmed := strings.TrimSpace(u.text)
if trimmed == "" || isSeparatorOnly(u.text) {
if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) {
overlapLen -= runeLen(u.text)
startIdx++
} else {
@@ -439,9 +526,11 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
for _, sub := range subs {
// Adjust offsets: sub positions are relative to parent content,
// shift to document-level offsets.
// Use additive shift (not Content-length based) so that chunks with
// prepended context headers keep correct positional tracking.
sub.Seq = childSeq
sub.Start += parent.Start
sub.End = sub.Start + runeLen(sub.Content)
sub.End += parent.Start
children = append(children, ChildChunk{
Chunk: sub,
ParentIndex: pi,
@@ -364,3 +364,623 @@ func TestSplitText_LargeChineseDocument(t *testing.T) {
}
}
}
// ---------------------------------------------------------------------------
// Table header prepending tests
// ---------------------------------------------------------------------------
func TestSplitText_TableHeaderPrependedToChunks(t *testing.T) {
// A markdown table large enough to span multiple chunks.
// Each chunk after the first should have the header row + separator prepended.
text := "" +
"前面的文字\n\n" +
"| 姓名 | 年龄 | 城市 |\n" +
"| --- | --- | --- |\n" +
"| 张三 | 25 | 北京 |\n" +
"| 李四 | 30 | 上海 |\n" +
"| 王五 | 28 | 广州 |\n" +
"| 赵六 | 35 | 深圳 |\n" +
"| 孙七 | 22 | 杭州 |\n" +
"| 周八 | 40 | 成都 |\n" +
"\n后面的文字"
tableHeader := "| 姓名 | 年龄 | 城市 |\n| --- | --- | --- |\n"
cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
if len(chunks) < 3 {
t.Fatalf("expected at least 3 chunks, got %d", len(chunks))
}
// Find chunks that contain table row data but not the original header position.
// These should have the header prepended.
headerPrependCount := 0
for _, c := range chunks {
if strings.Contains(c.Content, "| 李四") || strings.Contains(c.Content, "| 王五") ||
strings.Contains(c.Content, "| 赵六") || strings.Contains(c.Content, "| 孙七") ||
strings.Contains(c.Content, "| 周八") {
if !strings.Contains(c.Content, "| 张三") {
// This is a chunk with table rows but not the first row;
// it should have the header prepended.
if !strings.HasPrefix(c.Content, tableHeader) {
t.Errorf("chunk (seq=%d) has table rows but is missing prepended header:\n%s",
c.Seq, c.Content)
} else {
headerPrependCount++
}
}
}
}
if headerPrependCount == 0 {
t.Error("expected at least one chunk to have prepended table header, found none")
for i, c := range chunks {
t.Logf("chunk[%d] (seq=%d, start=%d, end=%d):\n%s", i, c.Seq, c.Start, c.End, c.Content)
}
}
}
func TestSplitText_NoHeaderForNonTableContent(t *testing.T) {
// Ensure header prepending doesn't affect non-table content.
text := strings.Repeat("这是一段普通的中文文本,不包含任何表格。\n\n", 10)
cfg := SplitterConfig{ChunkSize: 30, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
textRunes := []rune(text)
for i, c := range chunks {
contentRuneLen := utf8.RuneCountInString(c.Content)
spanLen := c.End - c.Start
if spanLen != contentRuneLen {
t.Errorf("chunk[%d]: span %d != rune len %d (no table, should be exact)", i, spanLen, contentRuneLen)
}
if c.End > len(textRunes) {
t.Errorf("chunk[%d]: End %d exceeds total runes %d", i, c.End, len(textRunes))
}
}
}
func TestSplitText_TableHeaderEndedByEmptyLine(t *testing.T) {
// After the table ends (empty line), subsequent chunks should NOT have the header.
text := "" +
"| A | B |\n" +
"| --- | --- |\n" +
"| 1 | 2 |\n" +
"| 3 | 4 |\n" +
"\n" +
"这是表格之后的普通文本内容,不应该包含表头。\n" +
"更多的普通文本内容用于填充。"
cfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
for _, c := range chunks {
hasTableRow := strings.Contains(c.Content, "| A | B |") || strings.Contains(c.Content, "|")
hasPlainText := strings.Contains(c.Content, "这是表格之后") || strings.Contains(c.Content, "更多的普通")
if hasPlainText && !hasTableRow {
// This chunk is purely post-table text; should NOT have table header
if strings.Contains(c.Content, "| --- |") {
t.Errorf("post-table chunk should not contain table header:\n%s", c.Content)
}
}
}
}
func TestHeaderTracker_BasicLifecycle(t *testing.T) {
ht := newHeaderTracker()
// Before table: no headers
ht.update("Some regular text")
if h := ht.getHeaders(); h != "" {
t.Errorf("expected no headers before table, got %q", h)
}
// Table header unit
ht.update("| A | B |\n| --- | --- |\n")
if h := ht.getHeaders(); h == "" {
t.Error("expected active header after table header unit")
}
// Table row: header should stay active
ht.update("| 1 | 2 |\n")
if h := ht.getHeaders(); h == "" {
t.Error("header should remain active during table rows")
}
// Empty line: header should end
ht.update("\n")
if h := ht.getHeaders(); h != "" {
t.Errorf("header should be cleared after empty line, got %q", h)
}
// New table can be tracked after the old one ended
ht.update("| X | Y |\n| --- | --- |\n")
if h := ht.getHeaders(); h == "" {
t.Error("expected new header to be tracked after previous table ended")
}
}
func TestHeaderTracker_EmptyHeaderRowRewrite(t *testing.T) {
// Some converters (e.g., MarkItDown) produce tables with empty header rows:
// ||
// | --- | --- |
// | real col A | real col B |
// The tracker should rewrite the header to be a proper Markdown table header:
// | real col A | real col B |
// | --- | --- |
ht := newHeaderTracker()
// Empty header row + separator
ht.update("||\n| --- | --- | --- |\n")
h := ht.getHeaders()
if h == "" {
t.Fatal("expected active header after empty header unit")
}
t.Logf("after empty header unit (pending): %q", h)
// First data row → becomes the real column names
ht.update("| 测试用例 ID | 测试模块 | 备注 |\n")
h = ht.getHeaders()
t.Logf("after rewrite: %q", h)
if !strings.Contains(h, "测试用例 ID") {
t.Errorf("rewritten header should contain column names, got:\n%s", h)
}
if strings.Contains(h, "||") {
t.Errorf("rewritten header should NOT contain empty '||' row, got:\n%s", h)
}
if !strings.Contains(h, "---") {
t.Errorf("rewritten header should contain separator, got:\n%s", h)
}
// Column names should come BEFORE the separator
colIdx := strings.Index(h, "测试用例 ID")
sepIdx := strings.Index(h, "---")
if colIdx > sepIdx {
t.Errorf("column names should appear before separator in rewritten header:\n%s", h)
}
// Subsequent data rows should NOT be absorbed
ht.update("| TC-001 | 模块A | 备注1 |\n")
h2 := ht.getHeaders()
if strings.Contains(h2, "TC-001") {
t.Errorf("header should NOT include subsequent data rows, got:\n%s", h2)
}
// Table end
ht.update("\n")
if ht.getHeaders() != "" {
t.Error("header should be cleared after empty line")
}
}
func TestHeaderTracker_NormalHeaderNoExtension(t *testing.T) {
// A table with proper column names in the header row should NOT be extended.
ht := newHeaderTracker()
ht.update("| 姓名 | 年龄 |\n| --- | --- |\n")
h := ht.getHeaders()
if h == "" {
t.Fatal("expected active header")
}
// First data row should NOT be absorbed into the header
ht.update("| 张三 | 25 |\n")
h2 := ht.getHeaders()
if strings.Contains(h2, "张三") {
t.Errorf("normal header should not absorb data rows, got:\n%s", h2)
}
}
func TestSplitText_EmptyHeaderRowPrepend(t *testing.T) {
// Simulate MarkItDown output: empty header row, real column names in first data row.
text := "" +
"前言\n\n" +
"||\n" +
"| --- | --- | --- |\n" +
"| 用例ID | 模块 | 步骤 |\n" +
"| TC-001 | A | 步骤1 |\n" +
"| TC-002 | B | 步骤2 |\n" +
"| TC-003 | C | 步骤3 |\n" +
"| TC-004 | D | 步骤4 |\n" +
"\n" +
"结尾"
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
t.Logf("total chunks: %d", len(chunks))
for i, c := range chunks {
t.Logf("chunk[%d] seq=%d start=%d end=%d:\n%s", i, c.Seq, c.Start, c.End, c.Content)
}
for _, c := range chunks {
hasLaterRow := strings.Contains(c.Content, "TC-002") ||
strings.Contains(c.Content, "TC-003") ||
strings.Contains(c.Content, "TC-004")
if hasLaterRow && !strings.Contains(c.Content, "TC-001") {
// Should have column names prepended
if !strings.Contains(c.Content, "用例ID") {
t.Errorf("chunk with data rows should have real column names prepended:\n%s", c.Content)
}
// Should NOT have the empty || row
lines := strings.Split(c.Content, "\n")
for _, line := range lines {
trimmed := strings.TrimSpace(line)
isOnlyPipes := trimmed != "" && func() bool {
for _, r := range trimmed {
if r != '|' && r != ' ' {
return false
}
}
return true
}()
if isOnlyPipes {
t.Errorf("chunk should NOT contain empty pipe row %q:\n%s", trimmed, c.Content)
break
}
}
}
// No line should appear as a duplicate in any chunk
lines := strings.Split(strings.TrimRight(c.Content, "\n"), "\n")
seen := make(map[string]int)
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.Contains(trimmed, "---") {
continue
}
seen[trimmed]++
if seen[trimmed] > 1 {
t.Errorf("line appears %d times in chunk (seq=%d): %q", seen[trimmed], c.Seq, trimmed)
}
}
}
// Verify restoration still works
restored := restoreTextFromChunks(chunks)
if restored != text {
t.Errorf("restoration failed for empty-header table\n original: %q\n restored: %q", text, restored)
}
}
func TestSplitText_MultipleTablesInDocument(t *testing.T) {
text := "" +
"第一个表格:\n\n" +
"| 名称 | 值 |\n" +
"| --- | --- |\n" +
"| A | 1 |\n" +
"| B | 2 |\n" +
"| C | 3 |\n" +
"\n" +
"中间的文字\n\n" +
"| 项目 | 状态 |\n" +
"| --- | --- |\n" +
"| X | 完成 |\n" +
"| Y | 进行中 |\n" +
"| Z | 未开始 |\n" +
"\n" +
"结尾文字"
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
// Verify that if a chunk has rows from table 2, it has table 2's header, not table 1's.
for _, c := range chunks {
if strings.Contains(c.Content, "| Y |") && !strings.Contains(c.Content, "| X |") {
if !strings.Contains(c.Content, "| 项目 | 状态 |") {
t.Errorf("chunk with table-2 rows should have table-2 header:\n%s", c.Content)
}
if strings.Contains(c.Content, "| 名称 | 值 |") {
t.Errorf("chunk with table-2 rows should NOT have table-1 header:\n%s", c.Content)
}
}
}
}
// ---------------------------------------------------------------------------
// Start/End restoration tests — verify original text can be reconstructed
// ---------------------------------------------------------------------------
// restoreTextFromChunks reconstructs the original text using only chunk
// Start/End positions. For chunks with prepended headers, the header is a
// "virtual" prefix whose length = runeLen(Content) - (End - Start).
// The original text portion is the last (End-Start) runes of Content.
func restoreTextFromChunks(chunks []Chunk) string {
if len(chunks) == 0 {
return ""
}
// Sort by End (ascending), then Start (ascending) — same order as Python restore_text
sorted := make([]Chunk, len(chunks))
copy(sorted, chunks)
for i := 1; i < len(sorted); i++ {
for j := i; j > 0; j-- {
if sorted[j].End < sorted[j-1].End ||
(sorted[j].End == sorted[j-1].End && sorted[j].Start < sorted[j-1].Start) {
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
} else {
break
}
}
}
var result []rune
lastEnd := 0
for _, c := range sorted {
if c.End <= lastEnd {
continue // fully contained in a previously processed chunk
}
contentRunes := []rune(c.Content)
spanLen := c.End - c.Start
headerLen := len(contentRunes) - spanLen
if headerLen < 0 {
headerLen = 0
}
// originalPortion is the text[Start:End] part, excluding any prepended header
originalPortion := contentRunes[headerLen:]
// Only take the portion after lastEnd (skip overlap)
newStart := 0
if lastEnd > c.Start {
newStart = lastEnd - c.Start
}
if newStart < len(originalPortion) {
result = append(result, originalPortion[newStart:]...)
}
lastEnd = c.End
}
return string(result)
}
func TestSplitText_RestoreTextNoTable(t *testing.T) {
// Plain Chinese text without any tables — baseline restoration check.
var sb strings.Builder
for i := 0; i < 30; i++ {
sb.WriteString(fmt.Sprintf("第%d段:这是一段用于测试的中文内容,包含各种标点符号。", i))
sb.WriteString("\n\n")
}
text := sb.String()
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 10, Separators: []string{"\n\n", "\n", "。"}}
chunks := SplitText(text, cfg)
restored := restoreTextFromChunks(chunks)
if restored != text {
t.Errorf("restoration failed for plain text\n original len: %d\n restored len: %d",
len([]rune(text)), len([]rune(restored)))
// Find first difference
orig := []rune(text)
rest := []rune(restored)
minLen := len(orig)
if len(rest) < minLen {
minLen = len(rest)
}
for i := 0; i < minLen; i++ {
if orig[i] != rest[i] {
t.Errorf("first diff at rune %d: orig=%q rest=%q", i, string(orig[i]), string(rest[i]))
break
}
}
}
}
func TestSplitText_RestoreTextWithTable(t *testing.T) {
// Document with a table large enough to span multiple chunks.
// Use a 2-column table (shorter header ~24 runes) + ChunkSize 80 so the
// header can be prepended (header 24 + row ~16 = 40 < 80).
text := "" +
"这是文档前言部分的内容。\n\n" +
"| 姓名 | 城市 |\n" +
"| --- | --- |\n" +
"| 张三 | 北京 |\n" +
"| 李四 | 上海 |\n" +
"| 王五 | 广州 |\n" +
"| 赵六 | 深圳 |\n" +
"| 孙七 | 杭州 |\n" +
"| 周八 | 成都 |\n" +
"| 吴九 | 武汉 |\n" +
"| 郑十 | 南京 |\n" +
"\n" +
"这是表格之后的文字内容。\n" +
"这里还有更多的普通段落。"
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
t.Logf("total chunks: %d", len(chunks))
for i, c := range chunks {
contentRunes := []rune(c.Content)
spanLen := c.End - c.Start
headerLen := len(contentRunes) - spanLen
t.Logf("chunk[%d] seq=%d start=%d end=%d span=%d contentLen=%d headerPrepend=%d\n content: %q",
i, c.Seq, c.Start, c.End, spanLen, len(contentRunes), headerLen, c.Content)
}
// 1. Verify basic position invariants
textRunes := []rune(text)
for i, c := range chunks {
if c.Start < 0 {
t.Errorf("chunk[%d]: Start %d < 0", i, c.Start)
}
if c.End > len(textRunes) {
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
}
if c.End < c.Start {
t.Errorf("chunk[%d]: End %d < Start %d", i, c.End, c.Start)
}
}
// 2. Verify text[Start:End] matches the non-header portion of Content
for i, c := range chunks {
contentRunes := []rune(c.Content)
spanLen := c.End - c.Start
headerLen := len(contentRunes) - spanLen
if headerLen < 0 {
t.Errorf("chunk[%d]: Content rune len (%d) < span len (%d) — impossible",
i, len(contentRunes), spanLen)
continue
}
if c.Start >= 0 && c.End <= len(textRunes) {
originalSlice := string(textRunes[c.Start:c.End])
contentSuffix := string(contentRunes[headerLen:])
if originalSlice != contentSuffix {
t.Errorf("chunk[%d]: text[%d:%d] != Content[%d:]"+
"\n text slice: %q"+
"\n content suffix: %q",
i, c.Start, c.End, headerLen, originalSlice, contentSuffix)
}
}
}
// 3. Restore original text and compare
restored := restoreTextFromChunks(chunks)
if restored != text {
t.Errorf("restoration FAILED"+
"\n original rune len: %d"+
"\n restored rune len: %d",
len(textRunes), len([]rune(restored)))
orig := []rune(text)
rest := []rune(restored)
minLen := len(orig)
if len(rest) < minLen {
minLen = len(rest)
}
for i := 0; i < minLen; i++ {
if orig[i] != rest[i] {
lo := i - 20
if lo < 0 {
lo = 0
}
hi := i + 20
if hi > minLen {
hi = minLen
}
t.Errorf("first diff at rune %d:\n orig context: %q\n rest context: %q",
i, string(orig[lo:hi]), string(rest[lo:hi]))
break
}
}
} else {
t.Log("restoration OK — original text perfectly reconstructed from Start/End")
}
// 4. Verify full coverage — Start/End spans must cover [0, len(textRunes))
covered := make([]bool, len(textRunes))
for _, c := range chunks {
for p := c.Start; p < c.End && p < len(textRunes); p++ {
covered[p] = true
}
}
for i, v := range covered {
if !v {
t.Errorf("rune position %d is not covered by any chunk", i)
break
}
}
}
func TestSplitText_RestoreTextWithMultipleTables(t *testing.T) {
text := "" +
"前言\n\n" +
"| A | B |\n| --- | --- |\n" +
"| 1 | 2 |\n| 3 | 4 |\n| 5 | 6 |\n| 7 | 8 |\n" +
"\n中间文字\n\n" +
"| X | Y |\n| --- | --- |\n" +
"| a | b |\n| c | d |\n| e | f |\n" +
"\n结尾"
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
// Restore and compare
restored := restoreTextFromChunks(chunks)
if restored != text {
t.Errorf("multi-table restoration failed\n original: %q\n restored: %q", text, restored)
}
// Verify text[Start:End] matches content suffix for every chunk
textRunes := []rune(text)
for i, c := range chunks {
contentRunes := []rune(c.Content)
spanLen := c.End - c.Start
headerLen := len(contentRunes) - spanLen
if headerLen < 0 {
t.Errorf("chunk[%d]: headerLen < 0", i)
continue
}
if c.End <= len(textRunes) {
if string(textRunes[c.Start:c.End]) != string(contentRunes[headerLen:]) {
t.Errorf("chunk[%d]: text[Start:End] mismatch with content suffix", i)
}
}
}
}
func TestSplitText_RestoreTextWithOverlap(t *testing.T) {
// Larger overlap to stress the overlap+header interaction.
text := "" +
"| 列1 | 列2 | 列3 |\n" +
"| --- | --- | --- |\n" +
"| 数据A1 | 数据A2 | 数据A3 |\n" +
"| 数据B1 | 数据B2 | 数据B3 |\n" +
"| 数据C1 | 数据C2 | 数据C3 |\n" +
"| 数据D1 | 数据D2 | 数据D3 |\n" +
"| 数据E1 | 数据E2 | 数据E3 |\n" +
"| 数据F1 | 数据F2 | 数据F3 |\n" +
"\n" +
"表后文本。"
for _, overlap := range []int{0, 3, 10, 20} {
t.Run(fmt.Sprintf("overlap=%d", overlap), func(t *testing.T) {
cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: overlap, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
restored := restoreTextFromChunks(chunks)
if restored != text {
t.Errorf("restoration failed with overlap=%d\n orig len=%d rest len=%d",
overlap, len([]rune(text)), len([]rune(restored)))
for i, c := range chunks {
t.Logf(" chunk[%d] start=%d end=%d content=%q", i, c.Start, c.End, c.Content)
}
}
})
}
}
func TestSplitTextParentChild_WithTableHeaders(t *testing.T) {
text := "" +
"前言\n\n" +
"| 列A | 列B |\n" +
"| --- | --- |\n" +
"| 数据1 | 数据2 |\n" +
"| 数据3 | 数据4 |\n" +
"| 数据5 | 数据6 |\n" +
"| 数据7 | 数据8 |\n" +
"\n" +
"结尾"
parentCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n"}}
childCfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
result := SplitTextParentChild(text, parentCfg, childCfg)
if len(result.Children) == 0 {
t.Fatal("expected child chunks")
}
// Verify child chunk positions don't exceed parent document
textRunes := []rune(text)
for i, child := range result.Children {
if child.Start < 0 {
t.Errorf("child[%d]: negative Start %d", i, child.Start)
}
if child.End > len(textRunes) {
t.Errorf("child[%d]: End %d exceeds text rune count %d", i, child.End, len(textRunes))
}
}
}