wizardchen
7aca1017db
Three follow-up fixes on top of the MinerU markdown preservation work: - Stop applying normalizeMinerUMarkdown inside ResolveAndStore. The helper is already called by MinerUReader.Read, and ResolveAndStore is shared by every parser (docreader, session attachments, ...). Running the heading/image unescape regexes globally would silently rewrite content (including inside fenced code blocks) for non-MinerU sources. - Recognize MinerU image references whose path contains spaces, e.g. "images/第 1 页.jpg". The previous regex used in extractImageRefsFromContent disallowed whitespace in the URL group, so such images were never matched and never persisted. Use a whitespace-tolerant pattern aligned with ResolveAndStore's own imgPattern. - Deduplicate uploads when the same MinerU image is referenced under multiple path forms (e.g. "images/foo.png" vs "./images/foo.png"). saveReferencedImage now caches by ref.Filename in addition to the raw ref path, so the second variant reuses the previously stored ServingURL instead of writing the same bytes to object storage again. Tests added: - TestProcessImagesMatchesPathsWithSpaces - TestResolveAndStoreDedupsSameImageRefVariants
81 lines
2.1 KiB
Go
81 lines
2.1 KiB
Go
package docparser
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
func TestResolveAndStoreRelativeHTMLImages(t *testing.T) {
|
|
png := createTestPNG(200, 150)
|
|
result := &types.ReadResult{
|
|
MarkdownContent: `<table><tr><td><img src="images/profile.jpg" alt="profile"/></td></tr></table>`,
|
|
ImageRefs: []types.ImageRef{
|
|
{
|
|
Filename: "profile.jpg",
|
|
OriginalRef: "images/profile.jpg",
|
|
MimeType: "image/png",
|
|
ImageData: png,
|
|
},
|
|
},
|
|
}
|
|
|
|
svc := &captureSaveBytes{}
|
|
r := NewImageResolver()
|
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(imgs) != 1 {
|
|
t.Fatalf("expected 1 stored image but got %d", len(imgs))
|
|
}
|
|
if len(svc.saved) != 1 {
|
|
t.Fatalf("expected SaveBytes to be called once but got %d", len(svc.saved))
|
|
}
|
|
if strings.Contains(out, `src="images/profile.jpg"`) {
|
|
t.Fatalf("expected relative html img src to be replaced, got: %s", out)
|
|
}
|
|
if !strings.Contains(out, `src="local://test/`) {
|
|
t.Fatalf("expected stored file url in html img src, got: %s", out)
|
|
}
|
|
}
|
|
|
|
// TestResolveAndStoreDedupsSameImageRefVariants ensures that when the same
|
|
// MinerU image is referenced under multiple path forms (e.g. "images/foo.png"
|
|
// and "./images/foo.png"), the bytes are uploaded to object storage only once.
|
|
func TestResolveAndStoreDedupsSameImageRefVariants(t *testing.T) {
|
|
png := createTestPNG(200, 150)
|
|
result := &types.ReadResult{
|
|
MarkdownContent: strings.Join([]string{
|
|
"",
|
|
`<img src="./images/foo.png"/>`,
|
|
}, "\n"),
|
|
ImageRefs: []types.ImageRef{
|
|
{
|
|
Filename: "foo.png",
|
|
OriginalRef: "images/foo.png",
|
|
MimeType: "image/png",
|
|
ImageData: png,
|
|
},
|
|
{
|
|
Filename: "foo.png",
|
|
OriginalRef: "./images/foo.png",
|
|
MimeType: "image/png",
|
|
ImageData: png,
|
|
},
|
|
},
|
|
}
|
|
|
|
svc := &captureSaveBytes{}
|
|
r := NewImageResolver()
|
|
_, _, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(svc.saved) != 1 {
|
|
t.Fatalf("expected SaveBytes to be called once for duplicate refs, got %d", len(svc.saved))
|
|
}
|
|
}
|