feat(chunker): implement header tracking for Markdown tables during chunking
- Added headerTracker to preserve context of Markdown table headers across split chunks. - Implemented logic to prepend headers to subsequent chunks after the first. - Enhanced merging logic to ensure headers are not duplicated in overlapping content. - Added comprehensive tests for header tracking and chunking behavior with Markdown tables.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
// Package chunker - header_tracker.go implements context-preserving header
|
||||
// tracking for document chunking, ported from docreader/splitter/header_hook.py.
|
||||
//
|
||||
// When a large Markdown table is split across multiple chunks, each chunk after
|
||||
// the first would lose the table header context. The headerTracker detects table
|
||||
// headers and signals the merge logic to prepend them to subsequent chunks.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// headerTrackerHook defines a pattern pair for detecting contextual headers.
|
||||
// When startPattern matches a unit's text, that text becomes an "active header".
|
||||
// The header stays active until endPattern matches a subsequent unit.
|
||||
type headerTrackerHook struct {
|
||||
startPattern *regexp.Regexp
|
||||
endPattern *regexp.Regexp
|
||||
priority int
|
||||
}
|
||||
|
||||
// defaultHeaderHooks returns header tracking hooks matching the Python defaults
|
||||
// in docreader/splitter/header_hook.py.
|
||||
var defaultHeaderHooks = []headerTrackerHook{
|
||||
{
|
||||
// Markdown table: header row + separator row (e.g. "| A | B |\n| --- | --- |\n")
|
||||
startPattern: regexp.MustCompile(`(?si)^\s*(?:\|[^|\n]*)+[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|?[\r\n]+$`),
|
||||
// Empty/whitespace line or a line that doesn't start with | or whitespace
|
||||
endPattern: regexp.MustCompile(`(?si)^\s*$|^\s*[^|\s].*$`),
|
||||
priority: 15,
|
||||
},
|
||||
}
|
||||
|
||||
// tableRowPattern matches a single Markdown table row: "| cell | cell | ... |\n"
|
||||
var tableRowPattern = regexp.MustCompile(`(?m)^\s*(?:\|[^|\n]*)+\|\s*$`)
|
||||
|
||||
// headerTracker maintains the state of active headers across split units.
|
||||
type headerTracker struct {
|
||||
hooks []headerTrackerHook
|
||||
activeHeaders map[int]string // priority -> header text
|
||||
endedHeaders map[int]bool // priorities that have been ended
|
||||
pendingExtend map[int]bool // headers with empty column names awaiting first data row
|
||||
}
|
||||
|
||||
func newHeaderTracker() *headerTracker {
|
||||
return &headerTracker{
|
||||
hooks: defaultHeaderHooks,
|
||||
activeHeaders: make(map[int]string),
|
||||
endedHeaders: make(map[int]bool),
|
||||
pendingExtend: make(map[int]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// update checks split text for header start/end markers and updates internal state.
|
||||
func (ht *headerTracker) update(split string) {
|
||||
// 1. Check for header-end markers among currently active headers
|
||||
for _, hook := range ht.hooks {
|
||||
if _, active := ht.activeHeaders[hook.priority]; active {
|
||||
if hook.endPattern.MatchString(split) {
|
||||
ht.endedHeaders[hook.priority] = true
|
||||
delete(ht.activeHeaders, hook.priority)
|
||||
delete(ht.pendingExtend, hook.priority)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. If a header has an empty column-name row (e.g. "||"), replace it with
|
||||
// a proper Markdown table header using the first data row as column names.
|
||||
//
|
||||
// Before: "||" + "| --- | --- |\n"
|
||||
// After: "| col1 | col2 |\n" + "| --- | --- |\n"
|
||||
for p := range ht.pendingExtend {
|
||||
if _, active := ht.activeHeaders[p]; active && tableRowPattern.MatchString(split) {
|
||||
sep := extractSeparatorLine(ht.activeHeaders[p])
|
||||
ht.activeHeaders[p] = split + sep
|
||||
}
|
||||
delete(ht.pendingExtend, p)
|
||||
}
|
||||
|
||||
// 3. Check for new header-start markers (only for hooks that are neither active nor ended)
|
||||
for _, hook := range ht.hooks {
|
||||
if _, active := ht.activeHeaders[hook.priority]; active {
|
||||
continue
|
||||
}
|
||||
if ht.endedHeaders[hook.priority] {
|
||||
continue
|
||||
}
|
||||
if loc := hook.startPattern.FindString(split); loc != "" {
|
||||
ht.activeHeaders[hook.priority] = loc
|
||||
if isEmptyTableHeaderRow(loc) {
|
||||
ht.pendingExtend[hook.priority] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. If all headers ended, clear the ended set so future tables can be tracked
|
||||
if len(ht.activeHeaders) == 0 {
|
||||
for k := range ht.endedHeaders {
|
||||
delete(ht.endedHeaders, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getHeaders returns all active headers concatenated, sorted by priority descending.
|
||||
func (ht *headerTracker) getHeaders() string {
|
||||
if len(ht.activeHeaders) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
priority int
|
||||
text string
|
||||
}
|
||||
entries := make([]entry, 0, len(ht.activeHeaders))
|
||||
for p, t := range ht.activeHeaders {
|
||||
entries = append(entries, entry{p, t})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].priority > entries[j].priority
|
||||
})
|
||||
|
||||
parts := make([]string, len(entries))
|
||||
for i, e := range entries {
|
||||
parts[i] = e.text
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// isEmptyTableHeaderRow checks if the header row (the line before the separator)
|
||||
// contains only pipes and whitespace — meaning the column names are empty.
|
||||
// This is common with MarkItDown and similar converters that produce tables like:
|
||||
//
|
||||
// ||
|
||||
// | --- | --- |
|
||||
// | real column A | real column B |
|
||||
func isEmptyTableHeaderRow(header string) bool {
|
||||
idx := strings.IndexByte(header, '\n')
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
row := strings.TrimSpace(header[:idx])
|
||||
for _, r := range row {
|
||||
if r != '|' && r != ' ' && r != '\t' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// extractSeparatorLine returns the separator line (e.g. "| --- | --- |\n") from
|
||||
// a table header string. It looks for the line containing "---".
|
||||
func extractSeparatorLine(header string) string {
|
||||
for _, line := range strings.Split(header, "\n") {
|
||||
if strings.Contains(line, "---") {
|
||||
return line + "\n"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -258,14 +258,17 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
|
||||
|
||||
// mergeUnits combines split units into chunks with overlap tracking.
|
||||
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
|
||||
// Active contextual headers (e.g., Markdown table headers) are prepended to new
|
||||
// chunks so that every chunk carries its own header context.
|
||||
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
if len(units) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Absolute maximum chunk size (留余量给标题等额外内容)
|
||||
const absoluteMaxSize = 7500
|
||||
|
||||
ht := newHeaderTracker()
|
||||
|
||||
var chunks []Chunk
|
||||
var current []splitUnit
|
||||
curLen := 0
|
||||
@@ -282,6 +285,9 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
curLen = 0
|
||||
}
|
||||
|
||||
// Update header state even for oversized units
|
||||
ht.update(u.text)
|
||||
|
||||
// Split this oversized unit into smaller chunks
|
||||
runes := []rune(u.text)
|
||||
offset := 0
|
||||
@@ -290,7 +296,6 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
if chunkEnd > len(runes) {
|
||||
chunkEnd = len(runes)
|
||||
} else {
|
||||
// Try to break at a newline or space
|
||||
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
||||
if runes[i] == '\n' || runes[i] == ' ' {
|
||||
chunkEnd = i + 1
|
||||
@@ -311,17 +316,47 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding this unit exceeds chunk size and we have content, flush
|
||||
if curLen+uLen > chunkSize && len(current) > 0 {
|
||||
// Update header tracking
|
||||
ht.update(u.text)
|
||||
headers := ht.getHeaders()
|
||||
headersLen := runeLen(headers)
|
||||
if headersLen > chunkSize {
|
||||
headers = ""
|
||||
headersLen = 0
|
||||
}
|
||||
|
||||
// If adding this unit (plus reserving space for headers in a potential
|
||||
// next chunk) would exceed chunk size, flush the current chunk.
|
||||
if curLen+uLen+headersLen > chunkSize && len(current) > 0 {
|
||||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||||
|
||||
// Keep overlap from the end of current
|
||||
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
|
||||
|
||||
// Shrink overlap further if needed to fit headers + next unit
|
||||
if headers != "" && headersLen+uLen <= chunkSize {
|
||||
for len(current) > 0 && curLen+uLen+headersLen > chunkSize {
|
||||
curLen -= runeLen(current[0].text)
|
||||
current = current[1:]
|
||||
}
|
||||
|
||||
// Prepend headers if the column-name context is not already present
|
||||
// in the overlap or the next unit being added.
|
||||
overlapText := unitsText(current)
|
||||
if !headerAlreadyPresent(headers, overlapText, u.text) {
|
||||
startPos := u.start
|
||||
if len(current) > 0 {
|
||||
startPos = current[0].start
|
||||
}
|
||||
hUnit := splitUnit{text: headers, start: startPos, end: startPos}
|
||||
current = append([]splitUnit{hUnit}, current...)
|
||||
curLen += headersLen
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if adding this unit would exceed absolute max
|
||||
if curLen+uLen > absoluteMaxSize {
|
||||
// Flush current and start fresh
|
||||
if len(current) > 0 {
|
||||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||||
current = nil
|
||||
@@ -341,6 +376,57 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
return chunks
|
||||
}
|
||||
|
||||
// unitsText concatenates the text of all units.
|
||||
func unitsText(units []splitUnit) string {
|
||||
var sb strings.Builder
|
||||
for _, u := range units {
|
||||
sb.WriteString(u.text)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// headerAlreadyPresent returns true if the column-name row from the header
|
||||
// is already present in the overlap or the next unit, preventing duplication.
|
||||
func headerAlreadyPresent(headers, overlapText, unitText string) bool {
|
||||
// Fast path: full header already in overlap or unit
|
||||
if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Extract the column-name row (first meaningful non-separator line).
|
||||
// For a rewritten header like "| col1 | col2 |\n| --- | --- |\n",
|
||||
// the first line is the column names.
|
||||
colRow := headerColumnRow(headers)
|
||||
if colRow == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow)
|
||||
}
|
||||
|
||||
// headerColumnRow extracts the column-name line from a header string.
|
||||
// Returns empty string if the header has no meaningful column names.
|
||||
func headerColumnRow(header string) string {
|
||||
for _, line := range strings.Split(header, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.Contains(line, "---") {
|
||||
continue
|
||||
}
|
||||
// Skip lines that are only pipes/whitespace (empty header rows)
|
||||
onlyPipes := true
|
||||
for _, r := range line {
|
||||
if r != '|' && r != ' ' && r != '\t' {
|
||||
onlyPipes = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !onlyPipes {
|
||||
return line
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func buildChunk(units []splitUnit, seq int) Chunk {
|
||||
var sb strings.Builder
|
||||
for _, u := range units {
|
||||
@@ -376,11 +462,12 @@ func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) (
|
||||
startIdx = i
|
||||
}
|
||||
|
||||
// Skip leading separators-only units in the overlap
|
||||
// Skip leading separator-only and header-marker units in the overlap
|
||||
for startIdx < len(current) {
|
||||
u := current[startIdx]
|
||||
isHeaderMarker := u.start == u.end
|
||||
trimmed := strings.TrimSpace(u.text)
|
||||
if trimmed == "" || isSeparatorOnly(u.text) {
|
||||
if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) {
|
||||
overlapLen -= runeLen(u.text)
|
||||
startIdx++
|
||||
} else {
|
||||
@@ -439,9 +526,11 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
|
||||
for _, sub := range subs {
|
||||
// Adjust offsets: sub positions are relative to parent content,
|
||||
// shift to document-level offsets.
|
||||
// Use additive shift (not Content-length based) so that chunks with
|
||||
// prepended context headers keep correct positional tracking.
|
||||
sub.Seq = childSeq
|
||||
sub.Start += parent.Start
|
||||
sub.End = sub.Start + runeLen(sub.Content)
|
||||
sub.End += parent.Start
|
||||
children = append(children, ChildChunk{
|
||||
Chunk: sub,
|
||||
ParentIndex: pi,
|
||||
|
||||
@@ -364,3 +364,623 @@ func TestSplitText_LargeChineseDocument(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Table header prepending tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestSplitText_TableHeaderPrependedToChunks(t *testing.T) {
|
||||
// A markdown table large enough to span multiple chunks.
|
||||
// Each chunk after the first should have the header row + separator prepended.
|
||||
text := "" +
|
||||
"前面的文字\n\n" +
|
||||
"| 姓名 | 年龄 | 城市 |\n" +
|
||||
"| --- | --- | --- |\n" +
|
||||
"| 张三 | 25 | 北京 |\n" +
|
||||
"| 李四 | 30 | 上海 |\n" +
|
||||
"| 王五 | 28 | 广州 |\n" +
|
||||
"| 赵六 | 35 | 深圳 |\n" +
|
||||
"| 孙七 | 22 | 杭州 |\n" +
|
||||
"| 周八 | 40 | 成都 |\n" +
|
||||
"\n后面的文字"
|
||||
|
||||
tableHeader := "| 姓名 | 年龄 | 城市 |\n| --- | --- | --- |\n"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("expected at least 3 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Find chunks that contain table row data but not the original header position.
|
||||
// These should have the header prepended.
|
||||
headerPrependCount := 0
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.Content, "| 李四") || strings.Contains(c.Content, "| 王五") ||
|
||||
strings.Contains(c.Content, "| 赵六") || strings.Contains(c.Content, "| 孙七") ||
|
||||
strings.Contains(c.Content, "| 周八") {
|
||||
if !strings.Contains(c.Content, "| 张三") {
|
||||
// This is a chunk with table rows but not the first row;
|
||||
// it should have the header prepended.
|
||||
if !strings.HasPrefix(c.Content, tableHeader) {
|
||||
t.Errorf("chunk (seq=%d) has table rows but is missing prepended header:\n%s",
|
||||
c.Seq, c.Content)
|
||||
} else {
|
||||
headerPrependCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if headerPrependCount == 0 {
|
||||
t.Error("expected at least one chunk to have prepended table header, found none")
|
||||
for i, c := range chunks {
|
||||
t.Logf("chunk[%d] (seq=%d, start=%d, end=%d):\n%s", i, c.Seq, c.Start, c.End, c.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_NoHeaderForNonTableContent(t *testing.T) {
|
||||
// Ensure header prepending doesn't affect non-table content.
|
||||
text := strings.Repeat("这是一段普通的中文文本,不包含任何表格。\n\n", 10)
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 30, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
textRunes := []rune(text)
|
||||
for i, c := range chunks {
|
||||
contentRuneLen := utf8.RuneCountInString(c.Content)
|
||||
spanLen := c.End - c.Start
|
||||
if spanLen != contentRuneLen {
|
||||
t.Errorf("chunk[%d]: span %d != rune len %d (no table, should be exact)", i, spanLen, contentRuneLen)
|
||||
}
|
||||
if c.End > len(textRunes) {
|
||||
t.Errorf("chunk[%d]: End %d exceeds total runes %d", i, c.End, len(textRunes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_TableHeaderEndedByEmptyLine(t *testing.T) {
|
||||
// After the table ends (empty line), subsequent chunks should NOT have the header.
|
||||
text := "" +
|
||||
"| A | B |\n" +
|
||||
"| --- | --- |\n" +
|
||||
"| 1 | 2 |\n" +
|
||||
"| 3 | 4 |\n" +
|
||||
"\n" +
|
||||
"这是表格之后的普通文本内容,不应该包含表头。\n" +
|
||||
"更多的普通文本内容用于填充。"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
for _, c := range chunks {
|
||||
hasTableRow := strings.Contains(c.Content, "| A | B |") || strings.Contains(c.Content, "|")
|
||||
hasPlainText := strings.Contains(c.Content, "这是表格之后") || strings.Contains(c.Content, "更多的普通")
|
||||
if hasPlainText && !hasTableRow {
|
||||
// This chunk is purely post-table text; should NOT have table header
|
||||
if strings.Contains(c.Content, "| --- |") {
|
||||
t.Errorf("post-table chunk should not contain table header:\n%s", c.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderTracker_BasicLifecycle(t *testing.T) {
|
||||
ht := newHeaderTracker()
|
||||
|
||||
// Before table: no headers
|
||||
ht.update("Some regular text")
|
||||
if h := ht.getHeaders(); h != "" {
|
||||
t.Errorf("expected no headers before table, got %q", h)
|
||||
}
|
||||
|
||||
// Table header unit
|
||||
ht.update("| A | B |\n| --- | --- |\n")
|
||||
if h := ht.getHeaders(); h == "" {
|
||||
t.Error("expected active header after table header unit")
|
||||
}
|
||||
|
||||
// Table row: header should stay active
|
||||
ht.update("| 1 | 2 |\n")
|
||||
if h := ht.getHeaders(); h == "" {
|
||||
t.Error("header should remain active during table rows")
|
||||
}
|
||||
|
||||
// Empty line: header should end
|
||||
ht.update("\n")
|
||||
if h := ht.getHeaders(); h != "" {
|
||||
t.Errorf("header should be cleared after empty line, got %q", h)
|
||||
}
|
||||
|
||||
// New table can be tracked after the old one ended
|
||||
ht.update("| X | Y |\n| --- | --- |\n")
|
||||
if h := ht.getHeaders(); h == "" {
|
||||
t.Error("expected new header to be tracked after previous table ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderTracker_EmptyHeaderRowRewrite(t *testing.T) {
|
||||
// Some converters (e.g., MarkItDown) produce tables with empty header rows:
|
||||
// ||
|
||||
// | --- | --- |
|
||||
// | real col A | real col B |
|
||||
// The tracker should rewrite the header to be a proper Markdown table header:
|
||||
// | real col A | real col B |
|
||||
// | --- | --- |
|
||||
ht := newHeaderTracker()
|
||||
|
||||
// Empty header row + separator
|
||||
ht.update("||\n| --- | --- | --- |\n")
|
||||
h := ht.getHeaders()
|
||||
if h == "" {
|
||||
t.Fatal("expected active header after empty header unit")
|
||||
}
|
||||
t.Logf("after empty header unit (pending): %q", h)
|
||||
|
||||
// First data row → becomes the real column names
|
||||
ht.update("| 测试用例 ID | 测试模块 | 备注 |\n")
|
||||
h = ht.getHeaders()
|
||||
t.Logf("after rewrite: %q", h)
|
||||
|
||||
if !strings.Contains(h, "测试用例 ID") {
|
||||
t.Errorf("rewritten header should contain column names, got:\n%s", h)
|
||||
}
|
||||
if strings.Contains(h, "||") {
|
||||
t.Errorf("rewritten header should NOT contain empty '||' row, got:\n%s", h)
|
||||
}
|
||||
if !strings.Contains(h, "---") {
|
||||
t.Errorf("rewritten header should contain separator, got:\n%s", h)
|
||||
}
|
||||
// Column names should come BEFORE the separator
|
||||
colIdx := strings.Index(h, "测试用例 ID")
|
||||
sepIdx := strings.Index(h, "---")
|
||||
if colIdx > sepIdx {
|
||||
t.Errorf("column names should appear before separator in rewritten header:\n%s", h)
|
||||
}
|
||||
|
||||
// Subsequent data rows should NOT be absorbed
|
||||
ht.update("| TC-001 | 模块A | 备注1 |\n")
|
||||
h2 := ht.getHeaders()
|
||||
if strings.Contains(h2, "TC-001") {
|
||||
t.Errorf("header should NOT include subsequent data rows, got:\n%s", h2)
|
||||
}
|
||||
|
||||
// Table end
|
||||
ht.update("\n")
|
||||
if ht.getHeaders() != "" {
|
||||
t.Error("header should be cleared after empty line")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderTracker_NormalHeaderNoExtension(t *testing.T) {
|
||||
// A table with proper column names in the header row should NOT be extended.
|
||||
ht := newHeaderTracker()
|
||||
|
||||
ht.update("| 姓名 | 年龄 |\n| --- | --- |\n")
|
||||
h := ht.getHeaders()
|
||||
if h == "" {
|
||||
t.Fatal("expected active header")
|
||||
}
|
||||
|
||||
// First data row should NOT be absorbed into the header
|
||||
ht.update("| 张三 | 25 |\n")
|
||||
h2 := ht.getHeaders()
|
||||
if strings.Contains(h2, "张三") {
|
||||
t.Errorf("normal header should not absorb data rows, got:\n%s", h2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_EmptyHeaderRowPrepend(t *testing.T) {
|
||||
// Simulate MarkItDown output: empty header row, real column names in first data row.
|
||||
text := "" +
|
||||
"前言\n\n" +
|
||||
"||\n" +
|
||||
"| --- | --- | --- |\n" +
|
||||
"| 用例ID | 模块 | 步骤 |\n" +
|
||||
"| TC-001 | A | 步骤1 |\n" +
|
||||
"| TC-002 | B | 步骤2 |\n" +
|
||||
"| TC-003 | C | 步骤3 |\n" +
|
||||
"| TC-004 | D | 步骤4 |\n" +
|
||||
"\n" +
|
||||
"结尾"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
t.Logf("total chunks: %d", len(chunks))
|
||||
for i, c := range chunks {
|
||||
t.Logf("chunk[%d] seq=%d start=%d end=%d:\n%s", i, c.Seq, c.Start, c.End, c.Content)
|
||||
}
|
||||
|
||||
for _, c := range chunks {
|
||||
hasLaterRow := strings.Contains(c.Content, "TC-002") ||
|
||||
strings.Contains(c.Content, "TC-003") ||
|
||||
strings.Contains(c.Content, "TC-004")
|
||||
if hasLaterRow && !strings.Contains(c.Content, "TC-001") {
|
||||
// Should have column names prepended
|
||||
if !strings.Contains(c.Content, "用例ID") {
|
||||
t.Errorf("chunk with data rows should have real column names prepended:\n%s", c.Content)
|
||||
}
|
||||
// Should NOT have the empty || row
|
||||
lines := strings.Split(c.Content, "\n")
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
isOnlyPipes := trimmed != "" && func() bool {
|
||||
for _, r := range trimmed {
|
||||
if r != '|' && r != ' ' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}()
|
||||
if isOnlyPipes {
|
||||
t.Errorf("chunk should NOT contain empty pipe row %q:\n%s", trimmed, c.Content)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No line should appear as a duplicate in any chunk
|
||||
lines := strings.Split(strings.TrimRight(c.Content, "\n"), "\n")
|
||||
seen := make(map[string]int)
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.Contains(trimmed, "---") {
|
||||
continue
|
||||
}
|
||||
seen[trimmed]++
|
||||
if seen[trimmed] > 1 {
|
||||
t.Errorf("line appears %d times in chunk (seq=%d): %q", seen[trimmed], c.Seq, trimmed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify restoration still works
|
||||
restored := restoreTextFromChunks(chunks)
|
||||
if restored != text {
|
||||
t.Errorf("restoration failed for empty-header table\n original: %q\n restored: %q", text, restored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_MultipleTablesInDocument(t *testing.T) {
|
||||
text := "" +
|
||||
"第一个表格:\n\n" +
|
||||
"| 名称 | 值 |\n" +
|
||||
"| --- | --- |\n" +
|
||||
"| A | 1 |\n" +
|
||||
"| B | 2 |\n" +
|
||||
"| C | 3 |\n" +
|
||||
"\n" +
|
||||
"中间的文字\n\n" +
|
||||
"| 项目 | 状态 |\n" +
|
||||
"| --- | --- |\n" +
|
||||
"| X | 完成 |\n" +
|
||||
"| Y | 进行中 |\n" +
|
||||
"| Z | 未开始 |\n" +
|
||||
"\n" +
|
||||
"结尾文字"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
// Verify that if a chunk has rows from table 2, it has table 2's header, not table 1's.
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.Content, "| Y |") && !strings.Contains(c.Content, "| X |") {
|
||||
if !strings.Contains(c.Content, "| 项目 | 状态 |") {
|
||||
t.Errorf("chunk with table-2 rows should have table-2 header:\n%s", c.Content)
|
||||
}
|
||||
if strings.Contains(c.Content, "| 名称 | 值 |") {
|
||||
t.Errorf("chunk with table-2 rows should NOT have table-1 header:\n%s", c.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Start/End restoration tests — verify original text can be reconstructed
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// restoreTextFromChunks reconstructs the original text using only chunk
|
||||
// Start/End positions. For chunks with prepended headers, the header is a
|
||||
// "virtual" prefix whose length = runeLen(Content) - (End - Start).
|
||||
// The original text portion is the last (End-Start) runes of Content.
|
||||
func restoreTextFromChunks(chunks []Chunk) string {
|
||||
if len(chunks) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Sort by End (ascending), then Start (ascending) — same order as Python restore_text
|
||||
sorted := make([]Chunk, len(chunks))
|
||||
copy(sorted, chunks)
|
||||
for i := 1; i < len(sorted); i++ {
|
||||
for j := i; j > 0; j-- {
|
||||
if sorted[j].End < sorted[j-1].End ||
|
||||
(sorted[j].End == sorted[j-1].End && sorted[j].Start < sorted[j-1].Start) {
|
||||
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var result []rune
|
||||
lastEnd := 0
|
||||
|
||||
for _, c := range sorted {
|
||||
if c.End <= lastEnd {
|
||||
continue // fully contained in a previously processed chunk
|
||||
}
|
||||
|
||||
contentRunes := []rune(c.Content)
|
||||
spanLen := c.End - c.Start
|
||||
headerLen := len(contentRunes) - spanLen
|
||||
if headerLen < 0 {
|
||||
headerLen = 0
|
||||
}
|
||||
// originalPortion is the text[Start:End] part, excluding any prepended header
|
||||
originalPortion := contentRunes[headerLen:]
|
||||
|
||||
// Only take the portion after lastEnd (skip overlap)
|
||||
newStart := 0
|
||||
if lastEnd > c.Start {
|
||||
newStart = lastEnd - c.Start
|
||||
}
|
||||
if newStart < len(originalPortion) {
|
||||
result = append(result, originalPortion[newStart:]...)
|
||||
}
|
||||
|
||||
lastEnd = c.End
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func TestSplitText_RestoreTextNoTable(t *testing.T) {
|
||||
// Plain Chinese text without any tables — baseline restoration check.
|
||||
var sb strings.Builder
|
||||
for i := 0; i < 30; i++ {
|
||||
sb.WriteString(fmt.Sprintf("第%d段:这是一段用于测试的中文内容,包含各种标点符号。", i))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
text := sb.String()
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 10, Separators: []string{"\n\n", "\n", "。"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
restored := restoreTextFromChunks(chunks)
|
||||
if restored != text {
|
||||
t.Errorf("restoration failed for plain text\n original len: %d\n restored len: %d",
|
||||
len([]rune(text)), len([]rune(restored)))
|
||||
// Find first difference
|
||||
orig := []rune(text)
|
||||
rest := []rune(restored)
|
||||
minLen := len(orig)
|
||||
if len(rest) < minLen {
|
||||
minLen = len(rest)
|
||||
}
|
||||
for i := 0; i < minLen; i++ {
|
||||
if orig[i] != rest[i] {
|
||||
t.Errorf("first diff at rune %d: orig=%q rest=%q", i, string(orig[i]), string(rest[i]))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_RestoreTextWithTable(t *testing.T) {
|
||||
// Document with a table large enough to span multiple chunks.
|
||||
// Use a 2-column table (shorter header ~24 runes) + ChunkSize 80 so the
|
||||
// header can be prepended (header 24 + row ~16 = 40 < 80).
|
||||
text := "" +
|
||||
"这是文档前言部分的内容。\n\n" +
|
||||
"| 姓名 | 城市 |\n" +
|
||||
"| --- | --- |\n" +
|
||||
"| 张三 | 北京 |\n" +
|
||||
"| 李四 | 上海 |\n" +
|
||||
"| 王五 | 广州 |\n" +
|
||||
"| 赵六 | 深圳 |\n" +
|
||||
"| 孙七 | 杭州 |\n" +
|
||||
"| 周八 | 成都 |\n" +
|
||||
"| 吴九 | 武汉 |\n" +
|
||||
"| 郑十 | 南京 |\n" +
|
||||
"\n" +
|
||||
"这是表格之后的文字内容。\n" +
|
||||
"这里还有更多的普通段落。"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
t.Logf("total chunks: %d", len(chunks))
|
||||
for i, c := range chunks {
|
||||
contentRunes := []rune(c.Content)
|
||||
spanLen := c.End - c.Start
|
||||
headerLen := len(contentRunes) - spanLen
|
||||
t.Logf("chunk[%d] seq=%d start=%d end=%d span=%d contentLen=%d headerPrepend=%d\n content: %q",
|
||||
i, c.Seq, c.Start, c.End, spanLen, len(contentRunes), headerLen, c.Content)
|
||||
}
|
||||
|
||||
// 1. Verify basic position invariants
|
||||
textRunes := []rune(text)
|
||||
for i, c := range chunks {
|
||||
if c.Start < 0 {
|
||||
t.Errorf("chunk[%d]: Start %d < 0", i, c.Start)
|
||||
}
|
||||
if c.End > len(textRunes) {
|
||||
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
|
||||
}
|
||||
if c.End < c.Start {
|
||||
t.Errorf("chunk[%d]: End %d < Start %d", i, c.End, c.Start)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Verify text[Start:End] matches the non-header portion of Content
|
||||
for i, c := range chunks {
|
||||
contentRunes := []rune(c.Content)
|
||||
spanLen := c.End - c.Start
|
||||
headerLen := len(contentRunes) - spanLen
|
||||
if headerLen < 0 {
|
||||
t.Errorf("chunk[%d]: Content rune len (%d) < span len (%d) — impossible",
|
||||
i, len(contentRunes), spanLen)
|
||||
continue
|
||||
}
|
||||
|
||||
if c.Start >= 0 && c.End <= len(textRunes) {
|
||||
originalSlice := string(textRunes[c.Start:c.End])
|
||||
contentSuffix := string(contentRunes[headerLen:])
|
||||
if originalSlice != contentSuffix {
|
||||
t.Errorf("chunk[%d]: text[%d:%d] != Content[%d:]"+
|
||||
"\n text slice: %q"+
|
||||
"\n content suffix: %q",
|
||||
i, c.Start, c.End, headerLen, originalSlice, contentSuffix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Restore original text and compare
|
||||
restored := restoreTextFromChunks(chunks)
|
||||
if restored != text {
|
||||
t.Errorf("restoration FAILED"+
|
||||
"\n original rune len: %d"+
|
||||
"\n restored rune len: %d",
|
||||
len(textRunes), len([]rune(restored)))
|
||||
orig := []rune(text)
|
||||
rest := []rune(restored)
|
||||
minLen := len(orig)
|
||||
if len(rest) < minLen {
|
||||
minLen = len(rest)
|
||||
}
|
||||
for i := 0; i < minLen; i++ {
|
||||
if orig[i] != rest[i] {
|
||||
lo := i - 20
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
hi := i + 20
|
||||
if hi > minLen {
|
||||
hi = minLen
|
||||
}
|
||||
t.Errorf("first diff at rune %d:\n orig context: %q\n rest context: %q",
|
||||
i, string(orig[lo:hi]), string(rest[lo:hi]))
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Log("restoration OK — original text perfectly reconstructed from Start/End")
|
||||
}
|
||||
|
||||
// 4. Verify full coverage — Start/End spans must cover [0, len(textRunes))
|
||||
covered := make([]bool, len(textRunes))
|
||||
for _, c := range chunks {
|
||||
for p := c.Start; p < c.End && p < len(textRunes); p++ {
|
||||
covered[p] = true
|
||||
}
|
||||
}
|
||||
for i, v := range covered {
|
||||
if !v {
|
||||
t.Errorf("rune position %d is not covered by any chunk", i)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_RestoreTextWithMultipleTables(t *testing.T) {
|
||||
text := "" +
|
||||
"前言\n\n" +
|
||||
"| A | B |\n| --- | --- |\n" +
|
||||
"| 1 | 2 |\n| 3 | 4 |\n| 5 | 6 |\n| 7 | 8 |\n" +
|
||||
"\n中间文字\n\n" +
|
||||
"| X | Y |\n| --- | --- |\n" +
|
||||
"| a | b |\n| c | d |\n| e | f |\n" +
|
||||
"\n结尾"
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
// Restore and compare
|
||||
restored := restoreTextFromChunks(chunks)
|
||||
if restored != text {
|
||||
t.Errorf("multi-table restoration failed\n original: %q\n restored: %q", text, restored)
|
||||
}
|
||||
|
||||
// Verify text[Start:End] matches content suffix for every chunk
|
||||
textRunes := []rune(text)
|
||||
for i, c := range chunks {
|
||||
contentRunes := []rune(c.Content)
|
||||
spanLen := c.End - c.Start
|
||||
headerLen := len(contentRunes) - spanLen
|
||||
if headerLen < 0 {
|
||||
t.Errorf("chunk[%d]: headerLen < 0", i)
|
||||
continue
|
||||
}
|
||||
if c.End <= len(textRunes) {
|
||||
if string(textRunes[c.Start:c.End]) != string(contentRunes[headerLen:]) {
|
||||
t.Errorf("chunk[%d]: text[Start:End] mismatch with content suffix", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText_RestoreTextWithOverlap(t *testing.T) {
|
||||
// Larger overlap to stress the overlap+header interaction.
|
||||
text := "" +
|
||||
"| 列1 | 列2 | 列3 |\n" +
|
||||
"| --- | --- | --- |\n" +
|
||||
"| 数据A1 | 数据A2 | 数据A3 |\n" +
|
||||
"| 数据B1 | 数据B2 | 数据B3 |\n" +
|
||||
"| 数据C1 | 数据C2 | 数据C3 |\n" +
|
||||
"| 数据D1 | 数据D2 | 数据D3 |\n" +
|
||||
"| 数据E1 | 数据E2 | 数据E3 |\n" +
|
||||
"| 数据F1 | 数据F2 | 数据F3 |\n" +
|
||||
"\n" +
|
||||
"表后文本。"
|
||||
|
||||
for _, overlap := range []int{0, 3, 10, 20} {
|
||||
t.Run(fmt.Sprintf("overlap=%d", overlap), func(t *testing.T) {
|
||||
cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: overlap, Separators: []string{"\n\n", "\n"}}
|
||||
chunks := SplitText(text, cfg)
|
||||
|
||||
restored := restoreTextFromChunks(chunks)
|
||||
if restored != text {
|
||||
t.Errorf("restoration failed with overlap=%d\n orig len=%d rest len=%d",
|
||||
overlap, len([]rune(text)), len([]rune(restored)))
|
||||
for i, c := range chunks {
|
||||
t.Logf(" chunk[%d] start=%d end=%d content=%q", i, c.Start, c.End, c.Content)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitTextParentChild_WithTableHeaders(t *testing.T) {
|
||||
text := "" +
|
||||
"前言\n\n" +
|
||||
"| 列A | 列B |\n" +
|
||||
"| --- | --- |\n" +
|
||||
"| 数据1 | 数据2 |\n" +
|
||||
"| 数据3 | 数据4 |\n" +
|
||||
"| 数据5 | 数据6 |\n" +
|
||||
"| 数据7 | 数据8 |\n" +
|
||||
"\n" +
|
||||
"结尾"
|
||||
|
||||
parentCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n"}}
|
||||
childCfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||||
result := SplitTextParentChild(text, parentCfg, childCfg)
|
||||
|
||||
if len(result.Children) == 0 {
|
||||
t.Fatal("expected child chunks")
|
||||
}
|
||||
|
||||
// Verify child chunk positions don't exceed parent document
|
||||
textRunes := []rune(text)
|
||||
for i, child := range result.Children {
|
||||
if child.Start < 0 {
|
||||
t.Errorf("child[%d]: negative Start %d", i, child.Start)
|
||||
}
|
||||
if child.End > len(textRunes) {
|
||||
t.Errorf("child[%d]: End %d exceeds text rune count %d", i, child.End, len(textRunes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user