fix(docparser): address review feedback on PR #1404
Three follow-up fixes on top of the MinerU markdown preservation work: - Stop applying normalizeMinerUMarkdown inside ResolveAndStore. The helper is already called by MinerUReader.Read, and ResolveAndStore is shared by every parser (docreader, session attachments, ...). Running the heading/image unescape regexes globally would silently rewrite content (including inside fenced code blocks) for non-MinerU sources. - Recognize MinerU image references whose path contains spaces, e.g. "images/第 1 页.jpg". The previous regex used in extractImageRefsFromContent disallowed whitespace in the URL group, so such images were never matched and never persisted. Use a whitespace-tolerant pattern aligned with ResolveAndStore's own imgPattern. - Deduplicate uploads when the same MinerU image is referenced under multiple path forms (e.g. "images/foo.png" vs "./images/foo.png"). saveReferencedImage now caches by ref.Filename in addition to the raw ref path, so the second variant reuses the previously stored ServingURL instead of writing the same bytes to object storage again. Tests added: - TestProcessImagesMatchesPathsWithSpaces - TestResolveAndStoreDedupsSameImageRefVariants
This commit is contained in:
@@ -80,7 +80,7 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
fileSvc interfaces.FileService,
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown := UnwrapLinkedImages(normalizeMinerUMarkdown(result.MarkdownContent))
|
||||
markdown := UnwrapLinkedImages(result.MarkdownContent)
|
||||
md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID)
|
||||
markdown = md2
|
||||
images = append(images, imgDataURIs...)
|
||||
@@ -171,6 +171,22 @@ func (r *ImageResolver) saveReferencedImage(
|
||||
return StoredImage{}, false
|
||||
}
|
||||
|
||||
// Reuse a previously saved upload when the same source image (identified by
|
||||
// ref.Filename) has already been persisted under a different markdown ref
|
||||
// path (e.g. "images/foo.png" vs "./images/foo.png"). This avoids writing
|
||||
// the same bytes to object storage multiple times.
|
||||
if ref.Filename != "" {
|
||||
if cached, ok := savedRefs["__filename__:"+ref.Filename]; ok {
|
||||
stored := StoredImage{
|
||||
OriginalRef: refPath,
|
||||
ServingURL: cached.ServingURL,
|
||||
MimeType: cached.MimeType,
|
||||
}
|
||||
savedRefs[refPath] = stored
|
||||
return stored, true
|
||||
}
|
||||
}
|
||||
|
||||
ext := extFromMime(ref.MimeType)
|
||||
if ext == "" {
|
||||
ext = filepath.Ext(ref.Filename)
|
||||
@@ -192,6 +208,9 @@ func (r *ImageResolver) saveReferencedImage(
|
||||
MimeType: ref.MimeType,
|
||||
}
|
||||
savedRefs[refPath] = stored
|
||||
if ref.Filename != "" {
|
||||
savedRefs["__filename__:"+ref.Filename] = stored
|
||||
}
|
||||
return stored, true
|
||||
}
|
||||
|
||||
|
||||
@@ -41,3 +41,40 @@ func TestResolveAndStoreRelativeHTMLImages(t *testing.T) {
|
||||
t.Fatalf("expected stored file url in html img src, got: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveAndStoreDedupsSameImageRefVariants ensures that when the same
|
||||
// MinerU image is referenced under multiple path forms (e.g. "images/foo.png"
|
||||
// and "./images/foo.png"), the bytes are uploaded to object storage only once.
|
||||
func TestResolveAndStoreDedupsSameImageRefVariants(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: strings.Join([]string{
|
||||
"",
|
||||
`<img src="./images/foo.png"/>`,
|
||||
}, "\n"),
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "foo.png",
|
||||
OriginalRef: "images/foo.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
{
|
||||
Filename: "foo.png",
|
||||
OriginalRef: "./images/foo.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
_, _, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(svc.saved) != 1 {
|
||||
t.Fatalf("expected SaveBytes to be called once for duplicate refs, got %d", len(svc.saved))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,12 +327,19 @@ func mineruImageOriginalRefs(mdContent, imagePath string) []string {
|
||||
return matched
|
||||
}
|
||||
|
||||
// imgMarkdownPatternAllowSpaces matches markdown image syntax while allowing
|
||||
// spaces in the URL group, so that paths like "images/第 1 页.jpg" produced by
|
||||
// MinerU on Chinese documents are still detected as image references.
|
||||
var imgMarkdownPatternAllowSpaces = regexp.MustCompile(
|
||||
`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`,
|
||||
)
|
||||
|
||||
func extractImageRefsFromContent(content string) []string {
|
||||
var refs []string
|
||||
|
||||
for _, match := range imgMarkdownPattern.FindAllStringSubmatch(content, -1) {
|
||||
for _, match := range imgMarkdownPatternAllowSpaces.FindAllStringSubmatch(content, -1) {
|
||||
if len(match) >= 3 {
|
||||
refs = append(refs, match[2])
|
||||
refs = append(refs, strings.TrimSpace(match[2]))
|
||||
}
|
||||
}
|
||||
for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) {
|
||||
|
||||
@@ -64,3 +64,26 @@ func TestProcessImagesKeepsReferencedVariants(t *testing.T) {
|
||||
t.Fatalf("expected 3 image refs, got %d", len(refs))
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessImagesMatchesPathsWithSpaces guards against a regression where
|
||||
// MinerU image filenames containing spaces (common on Chinese documents,
|
||||
// e.g. "images/第 1 页.jpg") would be silently dropped because the markdown
|
||||
// regex used to extract refs disallowed whitespace inside the URL group.
|
||||
func TestProcessImagesMatchesPathsWithSpaces(t *testing.T) {
|
||||
reader := &MinerUReader{}
|
||||
mdContent := ""
|
||||
|
||||
png := createTestPNG(200, 150)
|
||||
b64 := base64.StdEncoding.EncodeToString(png)
|
||||
images := map[string]string{
|
||||
"第 1 页.jpg": "data:image/png;base64," + b64,
|
||||
}
|
||||
|
||||
refs, _ := reader.processImages(mdContent, images)
|
||||
if len(refs) != 1 {
|
||||
t.Fatalf("expected 1 image ref for path with spaces, got %d", len(refs))
|
||||
}
|
||||
if refs[0].OriginalRef != "images/第 1 页.jpg" {
|
||||
t.Fatalf("unexpected OriginalRef: %q", refs[0].OriginalRef)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user