Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/splitter_test.go
T
wizardchen 397689d2f3 feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
2026-03-02 21:21:49 +08:00

367 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package chunker
import (
"fmt"
"strings"
"testing"
"unicode/utf8"
)
func TestSplitText_BasicASCII(t *testing.T) {
text := "Hello world. This is a test."
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{". "}}
chunks := SplitText(text, cfg)
if len(chunks) == 0 {
t.Fatal("expected at least one chunk")
}
combined := ""
for _, c := range chunks {
combined += c.Content
}
if combined != text {
t.Errorf("combined content mismatch:\n got: %q\n want: %q", combined, text)
}
}
func TestSplitText_ChineseText_StartEndAreRuneOffsets(t *testing.T) {
// Each Chinese character is 3 bytes in UTF-8 but 1 rune.
// This test ensures Start/End are rune offsets, not byte offsets.
text := "你好世界这是一个测试文本用于检验分割位置"
runeCount := utf8.RuneCountInString(text)
byteCount := len(text)
if runeCount == byteCount {
t.Fatal("test requires multi-byte characters")
}
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{"\n"}}
chunks := SplitText(text, cfg)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
c := chunks[0]
if c.Start != 0 {
t.Errorf("Start: got %d, want 0", c.Start)
}
if c.End != runeCount {
t.Errorf("End: got %d, want %d (runeCount); byteCount would be %d",
c.End, runeCount, byteCount)
}
}
func TestSplitText_ChineseMultiChunk_StartEndConsistency(t *testing.T) {
// Build a long Chinese text that will be split into multiple chunks.
line := "这是一段中文内容用于测试分割功能是否正确。"
text := strings.Repeat(line+"\n\n", 20)
text = strings.TrimRight(text, "\n")
cfg := SplitterConfig{ChunkSize: 30, ChunkOverlap: 5, Separators: []string{"\n\n", "\n", "。"}}
chunks := SplitText(text, cfg)
if len(chunks) < 2 {
t.Fatalf("expected multiple chunks, got %d", len(chunks))
}
textRunes := []rune(text)
for i, c := range chunks {
contentRunes := []rune(c.Content)
contentRuneLen := len(contentRunes)
// End - Start must equal the rune length of the content
spanLen := c.End - c.Start
if spanLen != contentRuneLen {
t.Errorf("chunk[%d]: End(%d) - Start(%d) = %d, but rune len of content = %d",
i, c.End, c.Start, spanLen, contentRuneLen)
}
// Start must be non-negative and End must not exceed total rune count
if c.Start < 0 {
t.Errorf("chunk[%d]: Start is negative: %d", i, c.Start)
}
if c.End > len(textRunes) {
t.Errorf("chunk[%d]: End %d exceeds total rune count %d", i, c.End, len(textRunes))
}
// Content from rune slice must match the chunk content
if c.Start >= 0 && c.End <= len(textRunes) {
sliced := string(textRunes[c.Start:c.End])
if sliced != c.Content {
t.Errorf("chunk[%d]: content mismatch via rune slice:\n got: %q\n want: %q",
i, sliced, c.Content)
}
}
}
}
func TestSplitText_MixedChineseAndASCII(t *testing.T) {
text := "Hello你好World世界Test测试"
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{"\n"}}
chunks := SplitText(text, cfg)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
c := chunks[0]
expectedRuneLen := utf8.RuneCountInString(text)
if c.End-c.Start != expectedRuneLen {
t.Errorf("End(%d) - Start(%d) = %d, want rune len %d (byte len would be %d)",
c.End, c.Start, c.End-c.Start, expectedRuneLen, len(text))
}
}
func TestSplitText_ProtectedPattern_ChineseContext(t *testing.T) {
// Test protected markdown images in Chinese context.
text := "这是前面的中文内容。![图片描述](http://example.com/img.png)这是后面的中文内容。"
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"。"}}
chunks := SplitText(text, cfg)
textRunes := []rune(text)
for i, c := range chunks {
if c.Start < 0 || c.End > len(textRunes) {
t.Errorf("chunk[%d]: out of rune range [%d, %d), total runes %d",
i, c.Start, c.End, len(textRunes))
continue
}
sliced := string(textRunes[c.Start:c.End])
if sliced != c.Content {
t.Errorf("chunk[%d]: rune-slice mismatch:\n sliced: %q\n content: %q",
i, sliced, c.Content)
}
}
}
func TestSplitText_SimulateMergeSlicing(t *testing.T) {
// Simulate what merge.go:104-106 does to ensure it won't panic.
// This is the exact pattern that caused the production crash.
line := "这是第一段内容用于模拟知识库问答的文本"
text := line + "\n\n" + line + "\n\n" + line
cfg := SplitterConfig{ChunkSize: 25, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
if len(chunks) < 2 {
t.Fatalf("need at least 2 chunks for overlap test, got %d", len(chunks))
}
for i := 1; i < len(chunks); i++ {
prev := chunks[i-1]
curr := chunks[i]
if curr.Start > prev.End {
continue // non-overlapping, no merge needed
}
// This is the exact merge.go logic:
contentRunes := []rune(curr.Content)
offset := len(contentRunes) - (curr.End - prev.End)
if offset < 0 {
t.Fatalf("chunk[%d] merge panic: offset=%d < 0 (contentRunes=%d, curr.End=%d, prev.End=%d)",
i, offset, len(contentRunes), curr.End, prev.End)
}
if offset > len(contentRunes) {
t.Fatalf("chunk[%d] merge panic: offset=%d > len(contentRunes)=%d",
i, offset, len(contentRunes))
}
_ = string(contentRunes[offset:])
}
}
func TestSplitText_Empty(t *testing.T) {
chunks := SplitText("", DefaultConfig())
if len(chunks) != 0 {
t.Errorf("expected 0 chunks for empty text, got %d", len(chunks))
}
}
func TestSplitText_SingleCharChinese(t *testing.T) {
text := "你"
cfg := SplitterConfig{ChunkSize: 10, ChunkOverlap: 0, Separators: []string{"\n"}}
chunks := SplitText(text, cfg)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
if chunks[0].Start != 0 || chunks[0].End != 1 {
t.Errorf("expected [0,1), got [%d,%d)", chunks[0].Start, chunks[0].End)
}
}
func TestSplitText_LaTeXBlockInChinese(t *testing.T) {
text := "前面的文字$$E=mc^2$$后面的文字"
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n"}}
chunks := SplitText(text, cfg)
textRunes := []rune(text)
for i, c := range chunks {
spanLen := c.End - c.Start
contentRuneLen := utf8.RuneCountInString(c.Content)
if spanLen != contentRuneLen {
t.Errorf("chunk[%d]: span %d != rune len %d", i, spanLen, contentRuneLen)
}
if c.End > len(textRunes) {
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
}
}
}
func TestSplitText_CodeBlockInChinese(t *testing.T) {
text := "中文描述\n```python\nprint('hello')\n```\n继续中文"
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n"}}
chunks := SplitText(text, cfg)
textRunes := []rune(text)
for i, c := range chunks {
if c.Start < 0 || c.End > len(textRunes) {
t.Errorf("chunk[%d]: out of range [%d,%d), total %d", i, c.Start, c.End, len(textRunes))
continue
}
sliced := string(textRunes[c.Start:c.End])
if sliced != c.Content {
t.Errorf("chunk[%d]: rune-slice mismatch:\n sliced: %q\n content: %q",
i, sliced, c.Content)
}
}
}
func TestSplitText_OverlapChunks_NonNegativeStart(t *testing.T) {
// When overlap is used, start of the next chunk could go before 0 if broken.
text := strings.Repeat("中文测试内容,", 50)
cfg := SplitterConfig{ChunkSize: 20, ChunkOverlap: 5, Separators: []string{","}}
chunks := SplitText(text, cfg)
for i, c := range chunks {
if c.Start < 0 {
t.Errorf("chunk[%d]: negative Start %d", i, c.Start)
}
if c.End < c.Start {
t.Errorf("chunk[%d]: End %d < Start %d", i, c.End, c.Start)
}
}
}
func TestBuildUnitsWithProtection_RuneOffsets(t *testing.T) {
text := "你好世界"
units := buildUnitsWithProtection(text, nil, []string{"\n"})
if len(units) != 1 {
t.Fatalf("expected 1 unit, got %d", len(units))
}
u := units[0]
expectedRuneLen := 4 // 4 Chinese characters
byteLen := len(text) // 12 bytes
if u.start != 0 {
t.Errorf("start: got %d, want 0", u.start)
}
if u.end != expectedRuneLen {
t.Errorf("end: got %d, want %d (rune len); byte len is %d", u.end, expectedRuneLen, byteLen)
}
}
func TestBuildUnitsWithProtection_WithProtectedSpan(t *testing.T) {
text := "前面![alt](url)后面"
protected := protectedSpans(text)
units := buildUnitsWithProtection(text, protected, []string{"\n"})
textRunes := []rune(text)
for i, u := range units {
contentRuneLen := utf8.RuneCountInString(u.text)
spanLen := u.end - u.start
if spanLen != contentRuneLen {
t.Errorf("unit[%d] %q: span %d != rune len %d (byte len %d)",
i, u.text, spanLen, contentRuneLen, len(u.text))
}
if u.start < 0 || u.end > len(textRunes) {
t.Errorf("unit[%d]: out of range [%d,%d), total runes %d",
i, u.start, u.end, len(textRunes))
}
}
}
func TestSplitBySeparators(t *testing.T) {
tests := []struct {
text string
separators []string
wantParts int
}{
{"a\n\nb\n\nc", []string{"\n\n"}, 5},
{"abc", []string{"\n"}, 1},
{"a\nb\nc", []string{"\n"}, 5},
{"", []string{"\n"}, 1},
}
for _, tt := range tests {
parts := splitBySeparators(tt.text, tt.separators)
if len(parts) != tt.wantParts {
t.Errorf("splitBySeparators(%q, %v): got %d parts %v, want %d",
tt.text, tt.separators, len(parts), parts, tt.wantParts)
}
}
}
func TestExtractImageRefs(t *testing.T) {
text := "hello ![alt1](url1) world ![alt2](url2) end"
refs := ExtractImageRefs(text)
if len(refs) != 2 {
t.Fatalf("expected 2 refs, got %d", len(refs))
}
if refs[0].OriginalRef != "url1" || refs[0].AltText != "alt1" {
t.Errorf("ref[0] mismatch: %+v", refs[0])
}
if refs[1].OriginalRef != "url2" || refs[1].AltText != "alt2" {
t.Errorf("ref[1] mismatch: %+v", refs[1])
}
}
func TestSplitText_LargeChineseDocument(t *testing.T) {
// Simulate a real document with paragraphs of Chinese text.
var sb strings.Builder
for i := 0; i < 100; i++ {
sb.WriteString(fmt.Sprintf("第%d段:这是一段用于测试的中文内容,包含各种常见的汉字和标点符号。", i))
sb.WriteString("\n\n")
}
text := sb.String()
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 10, Separators: []string{"\n\n", "\n", "。"}}
chunks := SplitText(text, cfg)
textRunes := []rune(text)
for i, c := range chunks {
contentRuneLen := utf8.RuneCountInString(c.Content)
spanLen := c.End - c.Start
if spanLen != contentRuneLen {
t.Errorf("chunk[%d]: End(%d)-Start(%d)=%d != runeLen(%d)",
i, c.End, c.Start, spanLen, contentRuneLen)
}
if c.Start < 0 {
t.Errorf("chunk[%d]: negative Start %d", i, c.Start)
}
if c.End > len(textRunes) {
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
}
if c.Start >= 0 && c.End <= len(textRunes) {
sliced := string(textRunes[c.Start:c.End])
if sliced != c.Content {
t.Errorf("chunk[%d]: content mismatch via rune-slice", i)
}
}
}
// Simulate merge.go logic on all overlapping chunk pairs
for i := 1; i < len(chunks); i++ {
prev := chunks[i-1]
curr := chunks[i]
if curr.Start > prev.End {
continue
}
contentRunes := []rune(curr.Content)
offset := len(contentRunes) - (curr.End - prev.End)
if offset < 0 || offset > len(contentRunes) {
t.Fatalf("chunk[%d] merge would panic: offset=%d, contentRunes=%d, curr.End=%d, prev.End=%d",
i, offset, len(contentRunes), curr.End, prev.End)
}
}
}