6210f44fb6
MinerU already returns markdown with embedded HTML blocks, but the current\nreader runs the whole document back through html-to-markdown. That\nsecond conversion escapes valid headings and image syntax, so chunk\nprofiling sees plain text instead of markdown structure and relative\nimage references stop matching the storage pipeline.\n\nKeep MinerU output in its original markdown form and only apply narrow\ncompatibility normalization for the specific over-escaped patterns we\nactually need to recover. The converter now matches image refs by the\npaths that are really present in markdown or embedded HTML instead of\nassuming a single images/<name> form.\n\nExtend ImageResolver so relative HTML <img src=...> references share the\nsame storage rewrite path as markdown images, deduplicate repeated saves,\nand keep the frontend sanitizer compatible with MinerU's details/summary\nblocks. Add focused docparser tests that cover escaped markdown repair,\nvariant image path matching, and relative HTML image persistence.
67 lines
1.9 KiB
Go
67 lines
1.9 KiB
Go
package docparser
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestNormalizeMinerUMarkdownPreservesMarkdownAndHTML(t *testing.T) {
|
|
input := strings.Join([]string{
|
|
"# Heading",
|
|
"",
|
|
"",
|
|
"",
|
|
`<details><summary>text_image</summary>caption</details>`,
|
|
"",
|
|
`<table><tr><td><img src="images/profile.jpg"/></td></tr></table>`,
|
|
}, "\n")
|
|
|
|
got := normalizeMinerUMarkdown(input)
|
|
|
|
if !strings.Contains(got, "# Heading") {
|
|
t.Fatalf("expected heading to stay intact, got: %q", got)
|
|
}
|
|
if strings.Contains(got, `\# Heading`) {
|
|
t.Fatalf("expected heading to avoid escaped form, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, "") {
|
|
t.Fatalf("expected markdown image syntax to stay intact, got: %q", got)
|
|
}
|
|
if strings.Contains(got, `!\[](images/cover.jpg)`) {
|
|
t.Fatalf("expected markdown image syntax to avoid escaped form, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, `<details><summary>text_image</summary>caption</details>`) {
|
|
t.Fatalf("expected details/summary block to be preserved, got: %q", got)
|
|
}
|
|
if !strings.Contains(got, `<img src="images/profile.jpg"/>`) {
|
|
t.Fatalf("expected html img tag to be preserved, got: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestProcessImagesKeepsReferencedVariants(t *testing.T) {
|
|
reader := &MinerUReader{}
|
|
mdContent := strings.Join([]string{
|
|
"",
|
|
`<img src="./images/profile.jpg"/>`,
|
|
``,
|
|
}, "\n")
|
|
|
|
png := createTestPNG(200, 150)
|
|
b64 := base64.StdEncoding.EncodeToString(png)
|
|
images := map[string]string{
|
|
"cover.jpg": "data:image/png;base64," + b64,
|
|
"profile.jpg": "data:image/png;base64," + b64,
|
|
"plain.jpg": "data:image/png;base64," + b64,
|
|
}
|
|
|
|
refs, gotMarkdown := reader.processImages(mdContent, images)
|
|
|
|
if gotMarkdown != mdContent {
|
|
t.Fatalf("processImages should not rewrite markdown content")
|
|
}
|
|
if len(refs) != 3 {
|
|
t.Fatalf("expected 3 image refs, got %d", len(refs))
|
|
}
|
|
}
|