6744e1cdd5
Self-review of the prior commit surfaced one critical regression and
several smaller issues. This change addresses them.
Critical: the old stripImageMarkup stripped the WHOLE <image>...</image>
block, but searchutil.EnrichContentWithImageInfo wraps successful VLM OCR
output in exactly that shape — so a scanned PDF where OCR worked was
being rejected by hasSufficientTextContent and routed to the same
"failed" path as the OCR-empty case. The fix splits stripping into:
* <image_original>...</image_original> blocks removed entirely
(they only carry the redundant Markdown ref already stripped above)
* <image>, <images>, <image_caption>, <image_ocr> tags removed at
the tag level, INNER TEXT PRESERVED, so OCR / caption text reaches
the threshold check and the LLM call.
A regression test pins this down with realistic enriched-image XML that
the previous code failed silently on.
Other fixes:
* Threshold lowered from 50 to 10 runes — short legitimate documents
(brief memos, single-line notes) must still pass.
* Summary chunk content no longer prefixes "# Document\n<filename>";
only the LLM-generated summary is indexed, so filenames cannot
leak back into RAG retrieval context.
* Wiki summary slug is now derived from the knowledge ID rather than
the upload filename, eliminating filename-based slugs like
"summary/mx5280-pdf" that downstream wiki-edit prompts can read.
* Citation sourceRef likewise drops the "|<docTitle>" suffix.
* Dead docTitle parameter removed from extractCandidateSlugs,
extractEntitiesAndConceptsNoUpsert, classifyChunkCitations.
* The redundant stripImageMarkup pass in the warning log is gone;
realTextRuneCount caches the count once and uses
utf8.RuneCountInString to avoid the rune-slice allocation.
Wiki-page display titles still flow through DocTitle (filename) into
later wiki-edit / index-rebuild prompts. That is a larger refactor of
the wiki pipeline and is intentionally left for a separate PR.
204 lines
5.5 KiB
Go
204 lines
5.5 KiB
Go
package service
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
func TestSlugify(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
want string
|
|
}{
|
|
{"Hello World", "hello-world"},
|
|
{"Acme Corp", "acme-corp"},
|
|
{" spaces ", "spaces"},
|
|
{"under_score", "under-score"},
|
|
{"Already-Good", "already-good"},
|
|
{"Special!@#Chars", "specialchars"},
|
|
{"CamelCase", "camelcase"},
|
|
{"", ""},
|
|
{"a/b/c", "a/b/c"}, // preserve slashes for hierarchical slugs
|
|
{"中文标题", "中文标题"}, // preserve CJK
|
|
{"Mix 中英文 Test", "mix-中英文-test"}, // mixed
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.input, func(t *testing.T) {
|
|
got := slugify(tt.input)
|
|
if got != tt.want {
|
|
t.Errorf("slugify(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTruncateString(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
maxLen int
|
|
want string
|
|
}{
|
|
{"hello world", 20, "hello world"},
|
|
{"hello world", 5, "hello..."},
|
|
{"", 10, ""},
|
|
{"abc", 3, "abc"},
|
|
{"abcd", 3, "abc..."},
|
|
{"中文测试", 2, "中文..."},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.input, func(t *testing.T) {
|
|
got := truncateString(tt.input, tt.maxLen)
|
|
if got != tt.want {
|
|
t.Errorf("truncateString(%q, %d) = %q, want %q", tt.input, tt.maxLen, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAppendUnique(t *testing.T) {
|
|
arr := types.StringArray{"a", "b"}
|
|
|
|
// Add new
|
|
result := appendUnique(arr, "c")
|
|
if len(result) != 3 {
|
|
t.Errorf("Expected 3 items, got %d", len(result))
|
|
}
|
|
|
|
// Add duplicate
|
|
result = appendUnique(result, "b")
|
|
if len(result) != 3 {
|
|
t.Errorf("Expected 3 items (no dup), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestReconstructContent(t *testing.T) {
|
|
chunks := []*types.Chunk{
|
|
{ChunkIndex: 2, ChunkType: types.ChunkTypeText, Content: "Third paragraph."},
|
|
{ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "First paragraph."},
|
|
{ChunkIndex: 1, ChunkType: types.ChunkTypeText, Content: "Second paragraph."},
|
|
{ChunkIndex: 3, ChunkType: types.ChunkTypeImageOCR, Content: "OCR text should be excluded."},
|
|
}
|
|
|
|
content := reconstructContent(chunks)
|
|
|
|
// Should be sorted by ChunkIndex and exclude non-text chunks
|
|
if content == "" {
|
|
t.Fatal("reconstructContent should not be empty")
|
|
}
|
|
|
|
// Verify order: first, second, third
|
|
if len(content) == 0 {
|
|
t.Fatal("empty content")
|
|
}
|
|
|
|
// The first characters should be "First"
|
|
if content[:5] != "First" {
|
|
t.Errorf("Expected content to start with 'First', got: %s", content[:20])
|
|
}
|
|
}
|
|
|
|
func TestReconstructContentEmpty(t *testing.T) {
|
|
content := reconstructContent(nil)
|
|
if content != "" {
|
|
t.Errorf("Empty chunks should produce empty content, got %q", content)
|
|
}
|
|
}
|
|
|
|
func TestStripImageMarkup(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
}{
|
|
{"plain text untouched", "Hello world.", "Hello world."},
|
|
{"single markdown image removed", "", ""},
|
|
{
|
|
"scanned-pdf style page references all stripped",
|
|
"\n\n",
|
|
"\n\n",
|
|
},
|
|
{"mixed text and image keeps text", "Intro paragraph.\n\nConclusion.", "Intro paragraph.\n\nConclusion."},
|
|
{"html img tag stripped", `Before <img src="x.png" alt="y"/> after`, "Before after"},
|
|
{
|
|
// Regression guard: an earlier version stripped the WHOLE
|
|
// <image>...</image> block (including <image_ocr> content),
|
|
// silently destroying successful VLM OCR results. The fix must
|
|
// preserve the inner OCR / caption text.
|
|
"enriched <image> block keeps inner OCR + caption text",
|
|
`<image url="images/page_1.png">
|
|
<image_original></image_original>
|
|
<image_caption>scanned letter on letterhead</image_caption>
|
|
<image_ocr>SEHR GEEHRTER HERR MUSTERMANN, ...</image_ocr>
|
|
</image>`,
|
|
"\n\nscanned letter on letterhead\nSEHR GEEHRTER HERR MUSTERMANN, ...\n",
|
|
},
|
|
{
|
|
"empty <image> block (OCR failed) reduces to whitespace",
|
|
`<image url="x"><image_original></image_original></image>`,
|
|
"",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := stripImageMarkup(tt.input)
|
|
if got != tt.want {
|
|
t.Errorf("stripImageMarkup(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHasSufficientTextContent(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want bool
|
|
}{
|
|
{"empty string", "", false},
|
|
{"only whitespace", " \n\n\t ", false},
|
|
{
|
|
"only image references (scanned PDF without OCR)",
|
|
"\n",
|
|
false,
|
|
},
|
|
{"too-short text below 10-rune threshold", "hi", false},
|
|
{
|
|
"short legitimate note above threshold",
|
|
"Meeting at 3pm tomorrow.",
|
|
true,
|
|
},
|
|
{
|
|
"image-only with successful VLM OCR (the fix)",
|
|
`<image url="images/p1.png">
|
|
<image_original></image_original>
|
|
<image_caption>scanned letter</image_caption>
|
|
<image_ocr>Sehr geehrter Herr Mustermann, in der Sache 4711/2024 ...</image_ocr>
|
|
</image>`,
|
|
true,
|
|
},
|
|
{
|
|
"image-only with failed VLM OCR (still rejected)",
|
|
`<image url="images/p1.png">
|
|
<image_original></image_original>
|
|
</image>`,
|
|
false,
|
|
},
|
|
{
|
|
"sufficient text mixed with images still passes",
|
|
"\nDie Beklagte hat die Klage anerkannt.\n",
|
|
true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := hasSufficientTextContent(tt.input)
|
|
if got != tt.want {
|
|
t.Errorf("hasSufficientTextContent(%q) = %v, want %v", tt.input, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|