397689d2f3
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
367 lines
11 KiB
Go
367 lines
11 KiB
Go
package chunker
|
||
|
||
import (
|
||
"fmt"
|
||
"strings"
|
||
"testing"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
func TestSplitText_BasicASCII(t *testing.T) {
|
||
text := "Hello world. This is a test."
|
||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{". "}}
|
||
chunks := SplitText(text, cfg)
|
||
if len(chunks) == 0 {
|
||
t.Fatal("expected at least one chunk")
|
||
}
|
||
combined := ""
|
||
for _, c := range chunks {
|
||
combined += c.Content
|
||
}
|
||
if combined != text {
|
||
t.Errorf("combined content mismatch:\n got: %q\n want: %q", combined, text)
|
||
}
|
||
}
|
||
|
||
func TestSplitText_ChineseText_StartEndAreRuneOffsets(t *testing.T) {
|
||
// Each Chinese character is 3 bytes in UTF-8 but 1 rune.
|
||
// This test ensures Start/End are rune offsets, not byte offsets.
|
||
text := "你好世界这是一个测试文本用于检验分割位置"
|
||
runeCount := utf8.RuneCountInString(text)
|
||
byteCount := len(text)
|
||
if runeCount == byteCount {
|
||
t.Fatal("test requires multi-byte characters")
|
||
}
|
||
|
||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{"\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
if len(chunks) != 1 {
|
||
t.Fatalf("expected 1 chunk, got %d", len(chunks))
|
||
}
|
||
|
||
c := chunks[0]
|
||
if c.Start != 0 {
|
||
t.Errorf("Start: got %d, want 0", c.Start)
|
||
}
|
||
if c.End != runeCount {
|
||
t.Errorf("End: got %d, want %d (runeCount); byteCount would be %d",
|
||
c.End, runeCount, byteCount)
|
||
}
|
||
}
|
||
|
||
func TestSplitText_ChineseMultiChunk_StartEndConsistency(t *testing.T) {
|
||
// Build a long Chinese text that will be split into multiple chunks.
|
||
line := "这是一段中文内容用于测试分割功能是否正确。"
|
||
text := strings.Repeat(line+"\n\n", 20)
|
||
text = strings.TrimRight(text, "\n")
|
||
|
||
cfg := SplitterConfig{ChunkSize: 30, ChunkOverlap: 5, Separators: []string{"\n\n", "\n", "。"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
if len(chunks) < 2 {
|
||
t.Fatalf("expected multiple chunks, got %d", len(chunks))
|
||
}
|
||
|
||
textRunes := []rune(text)
|
||
for i, c := range chunks {
|
||
contentRunes := []rune(c.Content)
|
||
contentRuneLen := len(contentRunes)
|
||
|
||
// End - Start must equal the rune length of the content
|
||
spanLen := c.End - c.Start
|
||
if spanLen != contentRuneLen {
|
||
t.Errorf("chunk[%d]: End(%d) - Start(%d) = %d, but rune len of content = %d",
|
||
i, c.End, c.Start, spanLen, contentRuneLen)
|
||
}
|
||
|
||
// Start must be non-negative and End must not exceed total rune count
|
||
if c.Start < 0 {
|
||
t.Errorf("chunk[%d]: Start is negative: %d", i, c.Start)
|
||
}
|
||
if c.End > len(textRunes) {
|
||
t.Errorf("chunk[%d]: End %d exceeds total rune count %d", i, c.End, len(textRunes))
|
||
}
|
||
|
||
// Content from rune slice must match the chunk content
|
||
if c.Start >= 0 && c.End <= len(textRunes) {
|
||
sliced := string(textRunes[c.Start:c.End])
|
||
if sliced != c.Content {
|
||
t.Errorf("chunk[%d]: content mismatch via rune slice:\n got: %q\n want: %q",
|
||
i, sliced, c.Content)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSplitText_MixedChineseAndASCII(t *testing.T) {
|
||
text := "Hello你好World世界Test测试"
|
||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 0, Separators: []string{"\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
if len(chunks) != 1 {
|
||
t.Fatalf("expected 1 chunk, got %d", len(chunks))
|
||
}
|
||
c := chunks[0]
|
||
expectedRuneLen := utf8.RuneCountInString(text)
|
||
if c.End-c.Start != expectedRuneLen {
|
||
t.Errorf("End(%d) - Start(%d) = %d, want rune len %d (byte len would be %d)",
|
||
c.End, c.Start, c.End-c.Start, expectedRuneLen, len(text))
|
||
}
|
||
}
|
||
|
||
func TestSplitText_ProtectedPattern_ChineseContext(t *testing.T) {
|
||
// Test protected markdown images in Chinese context.
|
||
text := "这是前面的中文内容。这是后面的中文内容。"
|
||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"。"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
textRunes := []rune(text)
|
||
for i, c := range chunks {
|
||
if c.Start < 0 || c.End > len(textRunes) {
|
||
t.Errorf("chunk[%d]: out of rune range [%d, %d), total runes %d",
|
||
i, c.Start, c.End, len(textRunes))
|
||
continue
|
||
}
|
||
sliced := string(textRunes[c.Start:c.End])
|
||
if sliced != c.Content {
|
||
t.Errorf("chunk[%d]: rune-slice mismatch:\n sliced: %q\n content: %q",
|
||
i, sliced, c.Content)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSplitText_SimulateMergeSlicing(t *testing.T) {
|
||
// Simulate what merge.go:104-106 does to ensure it won't panic.
|
||
// This is the exact pattern that caused the production crash.
|
||
line := "这是第一段内容用于模拟知识库问答的文本"
|
||
text := line + "\n\n" + line + "\n\n" + line
|
||
|
||
cfg := SplitterConfig{ChunkSize: 25, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
if len(chunks) < 2 {
|
||
t.Fatalf("need at least 2 chunks for overlap test, got %d", len(chunks))
|
||
}
|
||
|
||
for i := 1; i < len(chunks); i++ {
|
||
prev := chunks[i-1]
|
||
curr := chunks[i]
|
||
|
||
if curr.Start > prev.End {
|
||
continue // non-overlapping, no merge needed
|
||
}
|
||
|
||
// This is the exact merge.go logic:
|
||
contentRunes := []rune(curr.Content)
|
||
offset := len(contentRunes) - (curr.End - prev.End)
|
||
|
||
if offset < 0 {
|
||
t.Fatalf("chunk[%d] merge panic: offset=%d < 0 (contentRunes=%d, curr.End=%d, prev.End=%d)",
|
||
i, offset, len(contentRunes), curr.End, prev.End)
|
||
}
|
||
if offset > len(contentRunes) {
|
||
t.Fatalf("chunk[%d] merge panic: offset=%d > len(contentRunes)=%d",
|
||
i, offset, len(contentRunes))
|
||
}
|
||
|
||
_ = string(contentRunes[offset:])
|
||
}
|
||
}
|
||
|
||
func TestSplitText_Empty(t *testing.T) {
|
||
chunks := SplitText("", DefaultConfig())
|
||
if len(chunks) != 0 {
|
||
t.Errorf("expected 0 chunks for empty text, got %d", len(chunks))
|
||
}
|
||
}
|
||
|
||
func TestSplitText_SingleCharChinese(t *testing.T) {
|
||
text := "你"
|
||
cfg := SplitterConfig{ChunkSize: 10, ChunkOverlap: 0, Separators: []string{"\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
if len(chunks) != 1 {
|
||
t.Fatalf("expected 1 chunk, got %d", len(chunks))
|
||
}
|
||
if chunks[0].Start != 0 || chunks[0].End != 1 {
|
||
t.Errorf("expected [0,1), got [%d,%d)", chunks[0].Start, chunks[0].End)
|
||
}
|
||
}
|
||
|
||
func TestSplitText_LaTeXBlockInChinese(t *testing.T) {
|
||
text := "前面的文字$$E=mc^2$$后面的文字"
|
||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
textRunes := []rune(text)
|
||
for i, c := range chunks {
|
||
spanLen := c.End - c.Start
|
||
contentRuneLen := utf8.RuneCountInString(c.Content)
|
||
if spanLen != contentRuneLen {
|
||
t.Errorf("chunk[%d]: span %d != rune len %d", i, spanLen, contentRuneLen)
|
||
}
|
||
if c.End > len(textRunes) {
|
||
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSplitText_CodeBlockInChinese(t *testing.T) {
|
||
text := "中文描述\n```python\nprint('hello')\n```\n继续中文"
|
||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
textRunes := []rune(text)
|
||
for i, c := range chunks {
|
||
if c.Start < 0 || c.End > len(textRunes) {
|
||
t.Errorf("chunk[%d]: out of range [%d,%d), total %d", i, c.Start, c.End, len(textRunes))
|
||
continue
|
||
}
|
||
sliced := string(textRunes[c.Start:c.End])
|
||
if sliced != c.Content {
|
||
t.Errorf("chunk[%d]: rune-slice mismatch:\n sliced: %q\n content: %q",
|
||
i, sliced, c.Content)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSplitText_OverlapChunks_NonNegativeStart(t *testing.T) {
|
||
// When overlap is used, start of the next chunk could go before 0 if broken.
|
||
text := strings.Repeat("中文测试内容,", 50)
|
||
cfg := SplitterConfig{ChunkSize: 20, ChunkOverlap: 5, Separators: []string{","}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
for i, c := range chunks {
|
||
if c.Start < 0 {
|
||
t.Errorf("chunk[%d]: negative Start %d", i, c.Start)
|
||
}
|
||
if c.End < c.Start {
|
||
t.Errorf("chunk[%d]: End %d < Start %d", i, c.End, c.Start)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestBuildUnitsWithProtection_RuneOffsets(t *testing.T) {
|
||
text := "你好世界"
|
||
units := buildUnitsWithProtection(text, nil, []string{"\n"})
|
||
|
||
if len(units) != 1 {
|
||
t.Fatalf("expected 1 unit, got %d", len(units))
|
||
}
|
||
|
||
u := units[0]
|
||
expectedRuneLen := 4 // 4 Chinese characters
|
||
byteLen := len(text) // 12 bytes
|
||
|
||
if u.start != 0 {
|
||
t.Errorf("start: got %d, want 0", u.start)
|
||
}
|
||
if u.end != expectedRuneLen {
|
||
t.Errorf("end: got %d, want %d (rune len); byte len is %d", u.end, expectedRuneLen, byteLen)
|
||
}
|
||
}
|
||
|
||
func TestBuildUnitsWithProtection_WithProtectedSpan(t *testing.T) {
|
||
text := "前面后面"
|
||
protected := protectedSpans(text)
|
||
units := buildUnitsWithProtection(text, protected, []string{"\n"})
|
||
|
||
textRunes := []rune(text)
|
||
for i, u := range units {
|
||
contentRuneLen := utf8.RuneCountInString(u.text)
|
||
spanLen := u.end - u.start
|
||
if spanLen != contentRuneLen {
|
||
t.Errorf("unit[%d] %q: span %d != rune len %d (byte len %d)",
|
||
i, u.text, spanLen, contentRuneLen, len(u.text))
|
||
}
|
||
if u.start < 0 || u.end > len(textRunes) {
|
||
t.Errorf("unit[%d]: out of range [%d,%d), total runes %d",
|
||
i, u.start, u.end, len(textRunes))
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSplitBySeparators(t *testing.T) {
|
||
tests := []struct {
|
||
text string
|
||
separators []string
|
||
wantParts int
|
||
}{
|
||
{"a\n\nb\n\nc", []string{"\n\n"}, 5},
|
||
{"abc", []string{"\n"}, 1},
|
||
{"a\nb\nc", []string{"\n"}, 5},
|
||
{"", []string{"\n"}, 1},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
parts := splitBySeparators(tt.text, tt.separators)
|
||
if len(parts) != tt.wantParts {
|
||
t.Errorf("splitBySeparators(%q, %v): got %d parts %v, want %d",
|
||
tt.text, tt.separators, len(parts), parts, tt.wantParts)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestExtractImageRefs(t *testing.T) {
|
||
text := "hello  world  end"
|
||
refs := ExtractImageRefs(text)
|
||
if len(refs) != 2 {
|
||
t.Fatalf("expected 2 refs, got %d", len(refs))
|
||
}
|
||
if refs[0].OriginalRef != "url1" || refs[0].AltText != "alt1" {
|
||
t.Errorf("ref[0] mismatch: %+v", refs[0])
|
||
}
|
||
if refs[1].OriginalRef != "url2" || refs[1].AltText != "alt2" {
|
||
t.Errorf("ref[1] mismatch: %+v", refs[1])
|
||
}
|
||
}
|
||
|
||
func TestSplitText_LargeChineseDocument(t *testing.T) {
|
||
// Simulate a real document with paragraphs of Chinese text.
|
||
var sb strings.Builder
|
||
for i := 0; i < 100; i++ {
|
||
sb.WriteString(fmt.Sprintf("第%d段:这是一段用于测试的中文内容,包含各种常见的汉字和标点符号。", i))
|
||
sb.WriteString("\n\n")
|
||
}
|
||
text := sb.String()
|
||
|
||
cfg := SplitterConfig{ChunkSize: 50, ChunkOverlap: 10, Separators: []string{"\n\n", "\n", "。"}}
|
||
chunks := SplitText(text, cfg)
|
||
|
||
textRunes := []rune(text)
|
||
for i, c := range chunks {
|
||
contentRuneLen := utf8.RuneCountInString(c.Content)
|
||
spanLen := c.End - c.Start
|
||
if spanLen != contentRuneLen {
|
||
t.Errorf("chunk[%d]: End(%d)-Start(%d)=%d != runeLen(%d)",
|
||
i, c.End, c.Start, spanLen, contentRuneLen)
|
||
}
|
||
if c.Start < 0 {
|
||
t.Errorf("chunk[%d]: negative Start %d", i, c.Start)
|
||
}
|
||
if c.End > len(textRunes) {
|
||
t.Errorf("chunk[%d]: End %d > total runes %d", i, c.End, len(textRunes))
|
||
}
|
||
if c.Start >= 0 && c.End <= len(textRunes) {
|
||
sliced := string(textRunes[c.Start:c.End])
|
||
if sliced != c.Content {
|
||
t.Errorf("chunk[%d]: content mismatch via rune-slice", i)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Simulate merge.go logic on all overlapping chunk pairs
|
||
for i := 1; i < len(chunks); i++ {
|
||
prev := chunks[i-1]
|
||
curr := chunks[i]
|
||
if curr.Start > prev.End {
|
||
continue
|
||
}
|
||
contentRunes := []rune(curr.Content)
|
||
offset := len(contentRunes) - (curr.End - prev.End)
|
||
if offset < 0 || offset > len(contentRunes) {
|
||
t.Fatalf("chunk[%d] merge would panic: offset=%d, contentRunes=%d, curr.End=%d, prev.End=%d",
|
||
i, offset, len(contentRunes), curr.End, prev.End)
|
||
}
|
||
}
|
||
}
|