diff --git a/internal/infrastructure/chunker/header_tracker.go b/internal/infrastructure/chunker/header_tracker.go new file mode 100644 index 00000000..cf1b59da --- /dev/null +++ b/internal/infrastructure/chunker/header_tracker.go @@ -0,0 +1,161 @@ +// Package chunker - header_tracker.go implements context-preserving header +// tracking for document chunking, ported from docreader/splitter/header_hook.py. +// +// When a large Markdown table is split across multiple chunks, each chunk after +// the first would lose the table header context. The headerTracker detects table +// headers and signals the merge logic to prepend them to subsequent chunks. +package chunker + +import ( + "regexp" + "sort" + "strings" +) + +// headerTrackerHook defines a pattern pair for detecting contextual headers. +// When startPattern matches a unit's text, that text becomes an "active header". +// The header stays active until endPattern matches a subsequent unit. +type headerTrackerHook struct { + startPattern *regexp.Regexp + endPattern *regexp.Regexp + priority int +} + +// defaultHeaderHooks returns header tracking hooks matching the Python defaults +// in docreader/splitter/header_hook.py. +var defaultHeaderHooks = []headerTrackerHook{ + { + // Markdown table: header row + separator row (e.g. "| A | B |\n| --- | --- |\n") + startPattern: regexp.MustCompile(`(?si)^\s*(?:\|[^|\n]*)+[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|?[\r\n]+$`), + // Empty/whitespace line or a line that doesn't start with | or whitespace + endPattern: regexp.MustCompile(`(?si)^\s*$|^\s*[^|\s].*$`), + priority: 15, + }, +} + +// tableRowPattern matches a single Markdown table row: "| cell | cell | ... |\n" +var tableRowPattern = regexp.MustCompile(`(?m)^\s*(?:\|[^|\n]*)+\|\s*$`) + +// headerTracker maintains the state of active headers across split units. +type headerTracker struct { + hooks []headerTrackerHook + activeHeaders map[int]string // priority -> header text + endedHeaders map[int]bool // priorities that have been ended + pendingExtend map[int]bool // headers with empty column names awaiting first data row +} + +func newHeaderTracker() *headerTracker { + return &headerTracker{ + hooks: defaultHeaderHooks, + activeHeaders: make(map[int]string), + endedHeaders: make(map[int]bool), + pendingExtend: make(map[int]bool), + } +} + +// update checks split text for header start/end markers and updates internal state. +func (ht *headerTracker) update(split string) { + // 1. Check for header-end markers among currently active headers + for _, hook := range ht.hooks { + if _, active := ht.activeHeaders[hook.priority]; active { + if hook.endPattern.MatchString(split) { + ht.endedHeaders[hook.priority] = true + delete(ht.activeHeaders, hook.priority) + delete(ht.pendingExtend, hook.priority) + } + } + } + + // 2. If a header has an empty column-name row (e.g. "||"), replace it with + // a proper Markdown table header using the first data row as column names. + // + // Before: "||" + "| --- | --- |\n" + // After: "| col1 | col2 |\n" + "| --- | --- |\n" + for p := range ht.pendingExtend { + if _, active := ht.activeHeaders[p]; active && tableRowPattern.MatchString(split) { + sep := extractSeparatorLine(ht.activeHeaders[p]) + ht.activeHeaders[p] = split + sep + } + delete(ht.pendingExtend, p) + } + + // 3. Check for new header-start markers (only for hooks that are neither active nor ended) + for _, hook := range ht.hooks { + if _, active := ht.activeHeaders[hook.priority]; active { + continue + } + if ht.endedHeaders[hook.priority] { + continue + } + if loc := hook.startPattern.FindString(split); loc != "" { + ht.activeHeaders[hook.priority] = loc + if isEmptyTableHeaderRow(loc) { + ht.pendingExtend[hook.priority] = true + } + } + } + + // 4. If all headers ended, clear the ended set so future tables can be tracked + if len(ht.activeHeaders) == 0 { + for k := range ht.endedHeaders { + delete(ht.endedHeaders, k) + } + } +} + +// getHeaders returns all active headers concatenated, sorted by priority descending. +func (ht *headerTracker) getHeaders() string { + if len(ht.activeHeaders) == 0 { + return "" + } + + type entry struct { + priority int + text string + } + entries := make([]entry, 0, len(ht.activeHeaders)) + for p, t := range ht.activeHeaders { + entries = append(entries, entry{p, t}) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].priority > entries[j].priority + }) + + parts := make([]string, len(entries)) + for i, e := range entries { + parts[i] = e.text + } + return strings.Join(parts, "\n") +} + +// isEmptyTableHeaderRow checks if the header row (the line before the separator) +// contains only pipes and whitespace — meaning the column names are empty. +// This is common with MarkItDown and similar converters that produce tables like: +// +// || +// | --- | --- | +// | real column A | real column B | +func isEmptyTableHeaderRow(header string) bool { + idx := strings.IndexByte(header, '\n') + if idx < 0 { + return false + } + row := strings.TrimSpace(header[:idx]) + for _, r := range row { + if r != '|' && r != ' ' && r != '\t' { + return false + } + } + return true +} + +// extractSeparatorLine returns the separator line (e.g. "| --- | --- |\n") from +// a table header string. It looks for the line containing "---". +func extractSeparatorLine(header string) string { + for _, line := range strings.Split(header, "\n") { + if strings.Contains(line, "---") { + return line + "\n" + } + } + return "" +} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 4518e7b0..78e4b82e 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -258,14 +258,17 @@ func buildUnitsWithProtection(text string, protected []span, separators []string // mergeUnits combines split units into chunks with overlap tracking. // Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs). +// Active contextual headers (e.g., Markdown table headers) are prepended to new +// chunks so that every chunk carries its own header context. func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if len(units) == 0 { return nil } - // Absolute maximum chunk size (留余量给标题等额外内容) const absoluteMaxSize = 7500 + ht := newHeaderTracker() + var chunks []Chunk var current []splitUnit curLen := 0 @@ -282,6 +285,9 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { curLen = 0 } + // Update header state even for oversized units + ht.update(u.text) + // Split this oversized unit into smaller chunks runes := []rune(u.text) offset := 0 @@ -290,7 +296,6 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if chunkEnd > len(runes) { chunkEnd = len(runes) } else { - // Try to break at a newline or space for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { if runes[i] == '\n' || runes[i] == ' ' { chunkEnd = i + 1 @@ -311,17 +316,47 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { continue } - // If adding this unit exceeds chunk size and we have content, flush - if curLen+uLen > chunkSize && len(current) > 0 { + // Update header tracking + ht.update(u.text) + headers := ht.getHeaders() + headersLen := runeLen(headers) + if headersLen > chunkSize { + headers = "" + headersLen = 0 + } + + // If adding this unit (plus reserving space for headers in a potential + // next chunk) would exceed chunk size, flush the current chunk. + if curLen+uLen+headersLen > chunkSize && len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) // Keep overlap from the end of current current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen) + + // Shrink overlap further if needed to fit headers + next unit + if headers != "" && headersLen+uLen <= chunkSize { + for len(current) > 0 && curLen+uLen+headersLen > chunkSize { + curLen -= runeLen(current[0].text) + current = current[1:] + } + + // Prepend headers if the column-name context is not already present + // in the overlap or the next unit being added. + overlapText := unitsText(current) + if !headerAlreadyPresent(headers, overlapText, u.text) { + startPos := u.start + if len(current) > 0 { + startPos = current[0].start + } + hUnit := splitUnit{text: headers, start: startPos, end: startPos} + current = append([]splitUnit{hUnit}, current...) + curLen += headersLen + } + } } // Check if adding this unit would exceed absolute max if curLen+uLen > absoluteMaxSize { - // Flush current and start fresh if len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) current = nil @@ -341,6 +376,57 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { return chunks } +// unitsText concatenates the text of all units. +func unitsText(units []splitUnit) string { + var sb strings.Builder + for _, u := range units { + sb.WriteString(u.text) + } + return sb.String() +} + +// headerAlreadyPresent returns true if the column-name row from the header +// is already present in the overlap or the next unit, preventing duplication. +func headerAlreadyPresent(headers, overlapText, unitText string) bool { + // Fast path: full header already in overlap or unit + if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) { + return true + } + + // Extract the column-name row (first meaningful non-separator line). + // For a rewritten header like "| col1 | col2 |\n| --- | --- |\n", + // the first line is the column names. + colRow := headerColumnRow(headers) + if colRow == "" { + return false + } + + return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow) +} + +// headerColumnRow extracts the column-name line from a header string. +// Returns empty string if the header has no meaningful column names. +func headerColumnRow(header string) string { + for _, line := range strings.Split(header, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.Contains(line, "---") { + continue + } + // Skip lines that are only pipes/whitespace (empty header rows) + onlyPipes := true + for _, r := range line { + if r != '|' && r != ' ' && r != '\t' { + onlyPipes = false + break + } + } + if !onlyPipes { + return line + } + } + return "" +} + func buildChunk(units []splitUnit, seq int) Chunk { var sb strings.Builder for _, u := range units { @@ -376,11 +462,12 @@ func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ( startIdx = i } - // Skip leading separators-only units in the overlap + // Skip leading separator-only and header-marker units in the overlap for startIdx < len(current) { u := current[startIdx] + isHeaderMarker := u.start == u.end trimmed := strings.TrimSpace(u.text) - if trimmed == "" || isSeparatorOnly(u.text) { + if isHeaderMarker || trimmed == "" || isSeparatorOnly(u.text) { overlapLen -= runeLen(u.text) startIdx++ } else { @@ -439,9 +526,11 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren for _, sub := range subs { // Adjust offsets: sub positions are relative to parent content, // shift to document-level offsets. + // Use additive shift (not Content-length based) so that chunks with + // prepended context headers keep correct positional tracking. sub.Seq = childSeq sub.Start += parent.Start - sub.End = sub.Start + runeLen(sub.Content) + sub.End += parent.Start children = append(children, ChildChunk{ Chunk: sub, ParentIndex: pi, diff --git a/internal/infrastructure/chunker/splitter_test.go b/internal/infrastructure/chunker/splitter_test.go index 29b57d95..b0e9d438 100644 --- a/internal/infrastructure/chunker/splitter_test.go +++ b/internal/infrastructure/chunker/splitter_test.go @@ -364,3 +364,623 @@ func TestSplitText_LargeChineseDocument(t *testing.T) { } } } + +// --------------------------------------------------------------------------- +// Table header prepending tests +// --------------------------------------------------------------------------- + +func TestSplitText_TableHeaderPrependedToChunks(t *testing.T) { + // A markdown table large enough to span multiple chunks. + // Each chunk after the first should have the header row + separator prepended. + text := "" + + "前面的文字\n\n" + + "| 姓名 | 年龄 | 城市 |\n" + + "| --- | --- | --- |\n" + + "| 张三 | 25 | 北京 |\n" + + "| 李四 | 30 | 上海 |\n" + + "| 王五 | 28 | 广州 |\n" + + "| 赵六 | 35 | 深圳 |\n" + + "| 孙七 | 22 | 杭州 |\n" + + "| 周八 | 40 | 成都 |\n" + + "\n后面的文字" + + tableHeader := "| 姓名 | 年龄 | 城市 |\n| --- | --- | --- |\n" + + cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + if len(chunks) < 3 { + t.Fatalf("expected at least 3 chunks, got %d", len(chunks)) + } + + // Find chunks that contain table row data but not the original header position. + // These should have the header prepended. + headerPrependCount := 0 + for _, c := range chunks { + if strings.Contains(c.Content, "| 李四") || strings.Contains(c.Content, "| 王五") || + strings.Contains(c.Content, "| 赵六") || strings.Contains(c.Content, "| 孙七") || + strings.Contains(c.Content, "| 周八") { + if !strings.Contains(c.Content, "| 张三") { + // This is a chunk with table rows but not the first row; + // it should have the header prepended. + if !strings.HasPrefix(c.Content, tableHeader) { + t.Errorf("chunk (seq=%d) has table rows but is missing prepended header:\n%s", + c.Seq, c.Content) + } else { + headerPrependCount++ + } + } + } + } + + if headerPrependCount == 0 { + t.Error("expected at least one chunk to have prepended table header, found none") + for i, c := range chunks { + t.Logf("chunk[%d] (seq=%d, start=%d, end=%d):\n%s", i, c.Seq, c.Start, c.End, c.Content) + } + } +} + +func TestSplitText_NoHeaderForNonTableContent(t *testing.T) { + // Ensure header prepending doesn't affect non-table content. + text := strings.Repeat("这是一段普通的中文文本,不包含任何表格。\n\n", 10) + + cfg := SplitterConfig{ChunkSize: 30, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + textRunes := []rune(text) + for i, c := range chunks { + contentRuneLen := utf8.RuneCountInString(c.Content) + spanLen := c.End - c.Start + if spanLen != contentRuneLen { + t.Errorf("chunk[%d]: span %d != rune len %d (no table, should be exact)", i, spanLen, contentRuneLen) + } + if c.End > len(textRunes) { + t.Errorf("chunk[%d]: End %d exceeds total runes %d", i, c.End, len(textRunes)) + } + } +} + +func TestSplitText_TableHeaderEndedByEmptyLine(t *testing.T) { + // After the table ends (empty line), subsequent chunks should NOT have the header. + text := "" + + "| A | B |\n" + + "| --- | --- |\n" + + "| 1 | 2 |\n" + + "| 3 | 4 |\n" + + "\n" + + "这是表格之后的普通文本内容,不应该包含表头。\n" + + "更多的普通文本内容用于填充。" + + cfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + for _, c := range chunks { + hasTableRow := strings.Contains(c.Content, "| A | B |") || strings.Contains(c.Content, "|") + hasPlainText := strings.Contains(c.Content, "这是表格之后") || strings.Contains(c.Content, "更多的普通") + if hasPlainText && !hasTableRow { + // This chunk is purely post-table text; should NOT have table header + if strings.Contains(c.Content, "| --- |") { + t.Errorf("post-table chunk should not contain table header:\n%s", c.Content) + } + } + } +} + +func TestHeaderTracker_BasicLifecycle(t *testing.T) { + ht := newHeaderTracker() + + // Before table: no headers + ht.update("Some regular text") + if h := ht.getHeaders(); h != "" { + t.Errorf("expected no headers before table, got %q", h) + } + + // Table header unit + ht.update("| A | B |\n| --- | --- |\n") + if h := ht.getHeaders(); h == "" { + t.Error("expected active header after table header unit") + } + + // Table row: header should stay active + ht.update("| 1 | 2 |\n") + if h := ht.getHeaders(); h == "" { + t.Error("header should remain active during table rows") + } + + // Empty line: header should end + ht.update("\n") + if h := ht.getHeaders(); h != "" { + t.Errorf("header should be cleared after empty line, got %q", h) + } + + // New table can be tracked after the old one ended + ht.update("| X | Y |\n| --- | --- |\n") + if h := ht.getHeaders(); h == "" { + t.Error("expected new header to be tracked after previous table ended") + } +} + +func TestHeaderTracker_EmptyHeaderRowRewrite(t *testing.T) { + // Some converters (e.g., MarkItDown) produce tables with empty header rows: + // || + // | --- | --- | + // | real col A | real col B | + // The tracker should rewrite the header to be a proper Markdown table header: + // | real col A | real col B | + // | --- | --- | + ht := newHeaderTracker() + + // Empty header row + separator + ht.update("||\n| --- | --- | --- |\n") + h := ht.getHeaders() + if h == "" { + t.Fatal("expected active header after empty header unit") + } + t.Logf("after empty header unit (pending): %q", h) + + // First data row → becomes the real column names + ht.update("| 测试用例 ID | 测试模块 | 备注 |\n") + h = ht.getHeaders() + t.Logf("after rewrite: %q", h) + + if !strings.Contains(h, "测试用例 ID") { + t.Errorf("rewritten header should contain column names, got:\n%s", h) + } + if strings.Contains(h, "||") { + t.Errorf("rewritten header should NOT contain empty '||' row, got:\n%s", h) + } + if !strings.Contains(h, "---") { + t.Errorf("rewritten header should contain separator, got:\n%s", h) + } + // Column names should come BEFORE the separator + colIdx := strings.Index(h, "测试用例 ID") + sepIdx := strings.Index(h, "---") + if colIdx > sepIdx { + t.Errorf("column names should appear before separator in rewritten header:\n%s", h) + } + + // Subsequent data rows should NOT be absorbed + ht.update("| TC-001 | 模块A | 备注1 |\n") + h2 := ht.getHeaders() + if strings.Contains(h2, "TC-001") { + t.Errorf("header should NOT include subsequent data rows, got:\n%s", h2) + } + + // Table end + ht.update("\n") + if ht.getHeaders() != "" { + t.Error("header should be cleared after empty line") + } +} + +func TestHeaderTracker_NormalHeaderNoExtension(t *testing.T) { + // A table with proper column names in the header row should NOT be extended. + ht := newHeaderTracker() + + ht.update("| 姓名 | 年龄 |\n| --- | --- |\n") + h := ht.getHeaders() + if h == "" { + t.Fatal("expected active header") + } + + // First data row should NOT be absorbed into the header + ht.update("| 张三 | 25 |\n") + h2 := ht.getHeaders() + if strings.Contains(h2, "张三") { + t.Errorf("normal header should not absorb data rows, got:\n%s", h2) + } +} + +func TestSplitText_EmptyHeaderRowPrepend(t *testing.T) { + // Simulate MarkItDown output: empty header row, real column names in first data row. + text := "" + + "前言\n\n" + + "||\n" + + "| --- | --- | --- |\n" + + "| 用例ID | 模块 | 步骤 |\n" + + "| TC-001 | A | 步骤1 |\n" + + "| TC-002 | B | 步骤2 |\n" + + "| TC-003 | C | 步骤3 |\n" + + "| TC-004 | D | 步骤4 |\n" + + "\n" + + "结尾" + + cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + t.Logf("total chunks: %d", len(chunks)) + for i, c := range chunks { + t.Logf("chunk[%d] seq=%d start=%d end=%d:\n%s", i, c.Seq, c.Start, c.End, c.Content) + } + + for _, c := range chunks { + hasLaterRow := strings.Contains(c.Content, "TC-002") || + strings.Contains(c.Content, "TC-003") || + strings.Contains(c.Content, "TC-004") + if hasLaterRow && !strings.Contains(c.Content, "TC-001") { + // Should have column names prepended + if !strings.Contains(c.Content, "用例ID") { + t.Errorf("chunk with data rows should have real column names prepended:\n%s", c.Content) + } + // Should NOT have the empty || row + lines := strings.Split(c.Content, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + isOnlyPipes := trimmed != "" && func() bool { + for _, r := range trimmed { + if r != '|' && r != ' ' { + return false + } + } + return true + }() + if isOnlyPipes { + t.Errorf("chunk should NOT contain empty pipe row %q:\n%s", trimmed, c.Content) + break + } + } + } + + // No line should appear as a duplicate in any chunk + lines := strings.Split(strings.TrimRight(c.Content, "\n"), "\n") + seen := make(map[string]int) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.Contains(trimmed, "---") { + continue + } + seen[trimmed]++ + if seen[trimmed] > 1 { + t.Errorf("line appears %d times in chunk (seq=%d): %q", seen[trimmed], c.Seq, trimmed) + } + } + } + + // Verify restoration still works + restored := restoreTextFromChunks(chunks) + if restored != text { + t.Errorf("restoration failed for empty-header table\n original: %q\n restored: %q", text, restored) + } +} + +func TestSplitText_MultipleTablesInDocument(t *testing.T) { + text := "" + + "第一个表格:\n\n" + + "| 名称 | 值 |\n" + + "| --- | --- |\n" + + "| A | 1 |\n" + + "| B | 2 |\n" + + "| C | 3 |\n" + + "\n" + + "中间的文字\n\n" + + "| 项目 | 状态 |\n" + + "| --- | --- |\n" + + "| X | 完成 |\n" + + "| Y | 进行中 |\n" + + "| Z | 未开始 |\n" + + "\n" + + "结尾文字" + + cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + // Verify that if a chunk has rows from table 2, it has table 2's header, not table 1's. + for _, c := range chunks { + if strings.Contains(c.Content, "| Y |") && !strings.Contains(c.Content, "| X |") { + if !strings.Contains(c.Content, "| 项目 | 状态 |") { + t.Errorf("chunk with table-2 rows should have table-2 header:\n%s", c.Content) + } + if strings.Contains(c.Content, "| 名称 | 值 |") { + t.Errorf("chunk with table-2 rows should NOT have table-1 header:\n%s", c.Content) + } + } + } +} + +// --------------------------------------------------------------------------- +// Start/End restoration tests — verify original text can be reconstructed +// --------------------------------------------------------------------------- + +// restoreTextFromChunks reconstructs the original text using only chunk +// Start/End positions. For chunks with prepended headers, the header is a +// "virtual" prefix whose length = runeLen(Content) - (End - Start). +// The original text portion is the last (End-Start) runes of Content. +func restoreTextFromChunks(chunks []Chunk) string { + if len(chunks) == 0 { + return "" + } + + // Sort by End (ascending), then Start (ascending) — same order as Python restore_text + sorted := make([]Chunk, len(chunks)) + copy(sorted, chunks) + for i := 1; i < len(sorted); i++ { + for j := i; j > 0; j-- { + if sorted[j].End < sorted[j-1].End || + (sorted[j].End == sorted[j-1].End && sorted[j].Start < sorted[j-1].Start) { + sorted[j], sorted[j-1] = sorted[j-1], sorted[j] + } else { + break + } + } + } + + var result []rune + lastEnd := 0 + + for _, c := range sorted { + if c.End <= lastEnd { + continue // fully contained in a previously processed chunk + } + + contentRunes := []rune(c.Content) + spanLen := c.End - c.Start + headerLen := len(contentRunes) - spanLen + if headerLen < 0 { + headerLen = 0 + } + // originalPortion is the text[Start:End] part, excluding any prepended header + originalPortion := contentRunes[headerLen:] + + // Only take the portion after lastEnd (skip overlap) + newStart := 0 + if lastEnd > c.Start { + newStart = lastEnd - c.Start + } + if newStart < len(originalPortion) { + result = append(result, originalPortion[newStart:]...) + } + + lastEnd = c.End + } + + return string(result) +} + +func TestSplitText_RestoreTextNoTable(t *testing.T) { + // Plain Chinese text without any tables — baseline restoration check. + var sb strings.Builder + for i := 0; i < 30; i++ { + sb.WriteString(fmt.Sprintf("第%d段:这是一段用于测试的中文内容,包含各种标点符号。", i)) + sb.WriteString("\n\n") + } + text := sb.String() + + cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 10, Separators: []string{"\n\n", "\n", "。"}} + chunks := SplitText(text, cfg) + + restored := restoreTextFromChunks(chunks) + if restored != text { + t.Errorf("restoration failed for plain text\n original len: %d\n restored len: %d", + len([]rune(text)), len([]rune(restored))) + // Find first difference + orig := []rune(text) + rest := []rune(restored) + minLen := len(orig) + if len(rest) < minLen { + minLen = len(rest) + } + for i := 0; i < minLen; i++ { + if orig[i] != rest[i] { + t.Errorf("first diff at rune %d: orig=%q rest=%q", i, string(orig[i]), string(rest[i])) + break + } + } + } +} + +func TestSplitText_RestoreTextWithTable(t *testing.T) { + // Document with a table large enough to span multiple chunks. + // Use a 2-column table (shorter header ~24 runes) + ChunkSize 80 so the + // header can be prepended (header 24 + row ~16 = 40 < 80). + text := "" + + "这是文档前言部分的内容。\n\n" + + "| 姓名 | 城市 |\n" + + "| --- | --- |\n" + + "| 张三 | 北京 |\n" + + "| 李四 | 上海 |\n" + + "| 王五 | 广州 |\n" + + "| 赵六 | 深圳 |\n" + + "| 孙七 | 杭州 |\n" + + "| 周八 | 成都 |\n" + + "| 吴九 | 武汉 |\n" + + "| 郑十 | 南京 |\n" + + "\n" + + "这是表格之后的文字内容。\n" + + "这里还有更多的普通段落。" + + cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + t.Logf("total chunks: %d", len(chunks)) + for i, c := range chunks { + contentRunes := []rune(c.Content) + spanLen := c.End - c.Start + headerLen := len(contentRunes) - spanLen + t.Logf("chunk[%d] seq=%d start=%d end=%d span=%d contentLen=%d headerPrepend=%d\n content: %q", + i, c.Seq, c.Start, c.End, spanLen, len(contentRunes), headerLen, c.Content) + } + + // 1. Verify basic position invariants + textRunes := []rune(text) + for i, c := range chunks { + if c.Start < 0 { + t.Errorf("chunk[%d]: Start %d < 0", i, c.Start) + } + if c.End > len(textRunes) { + t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes)) + } + if c.End < c.Start { + t.Errorf("chunk[%d]: End %d < Start %d", i, c.End, c.Start) + } + } + + // 2. Verify text[Start:End] matches the non-header portion of Content + for i, c := range chunks { + contentRunes := []rune(c.Content) + spanLen := c.End - c.Start + headerLen := len(contentRunes) - spanLen + if headerLen < 0 { + t.Errorf("chunk[%d]: Content rune len (%d) < span len (%d) — impossible", + i, len(contentRunes), spanLen) + continue + } + + if c.Start >= 0 && c.End <= len(textRunes) { + originalSlice := string(textRunes[c.Start:c.End]) + contentSuffix := string(contentRunes[headerLen:]) + if originalSlice != contentSuffix { + t.Errorf("chunk[%d]: text[%d:%d] != Content[%d:]"+ + "\n text slice: %q"+ + "\n content suffix: %q", + i, c.Start, c.End, headerLen, originalSlice, contentSuffix) + } + } + } + + // 3. Restore original text and compare + restored := restoreTextFromChunks(chunks) + if restored != text { + t.Errorf("restoration FAILED"+ + "\n original rune len: %d"+ + "\n restored rune len: %d", + len(textRunes), len([]rune(restored))) + orig := []rune(text) + rest := []rune(restored) + minLen := len(orig) + if len(rest) < minLen { + minLen = len(rest) + } + for i := 0; i < minLen; i++ { + if orig[i] != rest[i] { + lo := i - 20 + if lo < 0 { + lo = 0 + } + hi := i + 20 + if hi > minLen { + hi = minLen + } + t.Errorf("first diff at rune %d:\n orig context: %q\n rest context: %q", + i, string(orig[lo:hi]), string(rest[lo:hi])) + break + } + } + } else { + t.Log("restoration OK — original text perfectly reconstructed from Start/End") + } + + // 4. Verify full coverage — Start/End spans must cover [0, len(textRunes)) + covered := make([]bool, len(textRunes)) + for _, c := range chunks { + for p := c.Start; p < c.End && p < len(textRunes); p++ { + covered[p] = true + } + } + for i, v := range covered { + if !v { + t.Errorf("rune position %d is not covered by any chunk", i) + break + } + } +} + +func TestSplitText_RestoreTextWithMultipleTables(t *testing.T) { + text := "" + + "前言\n\n" + + "| A | B |\n| --- | --- |\n" + + "| 1 | 2 |\n| 3 | 4 |\n| 5 | 6 |\n| 7 | 8 |\n" + + "\n中间文字\n\n" + + "| X | Y |\n| --- | --- |\n" + + "| a | b |\n| c | d |\n| e | f |\n" + + "\n结尾" + + cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + // Restore and compare + restored := restoreTextFromChunks(chunks) + if restored != text { + t.Errorf("multi-table restoration failed\n original: %q\n restored: %q", text, restored) + } + + // Verify text[Start:End] matches content suffix for every chunk + textRunes := []rune(text) + for i, c := range chunks { + contentRunes := []rune(c.Content) + spanLen := c.End - c.Start + headerLen := len(contentRunes) - spanLen + if headerLen < 0 { + t.Errorf("chunk[%d]: headerLen < 0", i) + continue + } + if c.End <= len(textRunes) { + if string(textRunes[c.Start:c.End]) != string(contentRunes[headerLen:]) { + t.Errorf("chunk[%d]: text[Start:End] mismatch with content suffix", i) + } + } + } +} + +func TestSplitText_RestoreTextWithOverlap(t *testing.T) { + // Larger overlap to stress the overlap+header interaction. + text := "" + + "| 列1 | 列2 | 列3 |\n" + + "| --- | --- | --- |\n" + + "| 数据A1 | 数据A2 | 数据A3 |\n" + + "| 数据B1 | 数据B2 | 数据B3 |\n" + + "| 数据C1 | 数据C2 | 数据C3 |\n" + + "| 数据D1 | 数据D2 | 数据D3 |\n" + + "| 数据E1 | 数据E2 | 数据E3 |\n" + + "| 数据F1 | 数据F2 | 数据F3 |\n" + + "\n" + + "表后文本。" + + for _, overlap := range []int{0, 3, 10, 20} { + t.Run(fmt.Sprintf("overlap=%d", overlap), func(t *testing.T) { + cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: overlap, Separators: []string{"\n\n", "\n"}} + chunks := SplitText(text, cfg) + + restored := restoreTextFromChunks(chunks) + if restored != text { + t.Errorf("restoration failed with overlap=%d\n orig len=%d rest len=%d", + overlap, len([]rune(text)), len([]rune(restored))) + for i, c := range chunks { + t.Logf(" chunk[%d] start=%d end=%d content=%q", i, c.Start, c.End, c.Content) + } + } + }) + } +} + +func TestSplitTextParentChild_WithTableHeaders(t *testing.T) { + text := "" + + "前言\n\n" + + "| 列A | 列B |\n" + + "| --- | --- |\n" + + "| 数据1 | 数据2 |\n" + + "| 数据3 | 数据4 |\n" + + "| 数据5 | 数据6 |\n" + + "| 数据7 | 数据8 |\n" + + "\n" + + "结尾" + + parentCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n"}} + childCfg := SplitterConfig{ChunkSize: 40, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}} + result := SplitTextParentChild(text, parentCfg, childCfg) + + if len(result.Children) == 0 { + t.Fatal("expected child chunks") + } + + // Verify child chunk positions don't exceed parent document + textRunes := []rune(text) + for i, child := range result.Children { + if child.Start < 0 { + t.Errorf("child[%d]: negative Start %d", i, child.Start) + } + if child.End > len(textRunes) { + t.Errorf("child[%d]: End %d exceeds text rune count %d", i, child.End, len(textRunes)) + } + } +}