7aca1017db
Three follow-up fixes on top of the MinerU markdown preservation work: - Stop applying normalizeMinerUMarkdown inside ResolveAndStore. The helper is already called by MinerUReader.Read, and ResolveAndStore is shared by every parser (docreader, session attachments, ...). Running the heading/image unescape regexes globally would silently rewrite content (including inside fenced code blocks) for non-MinerU sources. - Recognize MinerU image references whose path contains spaces, e.g. "images/第 1 页.jpg". The previous regex used in extractImageRefsFromContent disallowed whitespace in the URL group, so such images were never matched and never persisted. Use a whitespace-tolerant pattern aligned with ResolveAndStore's own imgPattern. - Deduplicate uploads when the same MinerU image is referenced under multiple path forms (e.g. "images/foo.png" vs "./images/foo.png"). saveReferencedImage now caches by ref.Filename in addition to the raw ref path, so the second variant reuses the previously stored ServingURL instead of writing the same bytes to object storage again. Tests added: - TestProcessImagesMatchesPathsWithSpaces - TestResolveAndStoreDedupsSameImageRefVariants
90 lines
2.7 KiB
Go
90 lines
2.7 KiB
Go
package docparser
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestNormalizeMinerUMarkdownPreservesMarkdownAndHTML(t *testing.T) {
|
|
input := strings.Join([]string{
|
|
"# Heading",
|
|
"",
|
|
"",
|
|
"",
|
|
`<details><summary>text_image</summary>caption</details>`,
|
|
"",
|
|
`<table><tr><td><img src="images/profile.jpg"/></td></tr></table>`,
|
|
}, "\n")
|
|
|
|
got := normalizeMinerUMarkdown(input)
|
|
|
|
if !strings.Contains(got, "# Heading") {
|
|
t.Fatalf("expected heading to stay intact, got: %q", got)
|
|
}
|
|
if strings.Contains(got, `\# Heading`) {
|
|
t.Fatalf("expected heading to avoid escaped form, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, "") {
|
|
t.Fatalf("expected markdown image syntax to stay intact, got: %q", got)
|
|
}
|
|
if strings.Contains(got, `!\[](images/cover.jpg)`) {
|
|
t.Fatalf("expected markdown image syntax to avoid escaped form, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, `<details><summary>text_image</summary>caption</details>`) {
|
|
t.Fatalf("expected details/summary block to be preserved, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, `<img src="images/profile.jpg"/>`) {
|
|
t.Fatalf("expected html img tag to be preserved, got: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestProcessImagesKeepsReferencedVariants(t *testing.T) {
|
|
reader := &MinerUReader{}
|
|
mdContent := strings.Join([]string{
|
|
"",
|
|
`<img src="./images/profile.jpg"/>`,
|
|
``,
|
|
}, "\n")
|
|
|
|
png := createTestPNG(200, 150)
|
|
b64 := base64.StdEncoding.EncodeToString(png)
|
|
images := map[string]string{
|
|
"cover.jpg": "data:image/png;base64," + b64,
|
|
"profile.jpg": "data:image/png;base64," + b64,
|
|
"plain.jpg": "data:image/png;base64," + b64,
|
|
}
|
|
|
|
refs, gotMarkdown := reader.processImages(mdContent, images)
|
|
|
|
if gotMarkdown != mdContent {
|
|
t.Fatalf("processImages should not rewrite markdown content")
|
|
}
|
|
if len(refs) != 3 {
|
|
t.Fatalf("expected 3 image refs, got %d", len(refs))
|
|
}
|
|
}
|
|
|
|
// TestProcessImagesMatchesPathsWithSpaces guards against a regression where
|
|
// MinerU image filenames containing spaces (common on Chinese documents,
|
|
// e.g. "images/第 1 页.jpg") would be silently dropped because the markdown
|
|
// regex used to extract refs disallowed whitespace inside the URL group.
|
|
func TestProcessImagesMatchesPathsWithSpaces(t *testing.T) {
|
|
reader := &MinerUReader{}
|
|
mdContent := ""
|
|
|
|
png := createTestPNG(200, 150)
|
|
b64 := base64.StdEncoding.EncodeToString(png)
|
|
images := map[string]string{
|
|
"第 1 页.jpg": "data:image/png;base64," + b64,
|
|
}
|
|
|
|
refs, _ := reader.processImages(mdContent, images)
|
|
if len(refs) != 1 {
|
|
t.Fatalf("expected 1 image ref for path with spaces, got %d", len(refs))
|
|
}
|
|
if refs[0].OriginalRef != "images/第 1 页.jpg" {
|
|
t.Fatalf("unexpected OriginalRef: %q", refs[0].OriginalRef)
|
|
}
|
|
}
|