From 7aca1017dbf842d752f1569e60e6d1ab1bbb0d82 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 21 May 2026 11:37:24 +0800 Subject: [PATCH] fix(docparser): address review feedback on PR #1404 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-up fixes on top of the MinerU markdown preservation work: - Stop applying normalizeMinerUMarkdown inside ResolveAndStore. The helper is already called by MinerUReader.Read, and ResolveAndStore is shared by every parser (docreader, session attachments, ...). Running the heading/image unescape regexes globally would silently rewrite content (including inside fenced code blocks) for non-MinerU sources. - Recognize MinerU image references whose path contains spaces, e.g. "images/第 1 页.jpg". The previous regex used in extractImageRefsFromContent disallowed whitespace in the URL group, so such images were never matched and never persisted. Use a whitespace-tolerant pattern aligned with ResolveAndStore's own imgPattern. - Deduplicate uploads when the same MinerU image is referenced under multiple path forms (e.g. "images/foo.png" vs "./images/foo.png"). saveReferencedImage now caches by ref.Filename in addition to the raw ref path, so the second variant reuses the previously stored ServingURL instead of writing the same bytes to object storage again. Tests added: - TestProcessImagesMatchesPathsWithSpaces - TestResolveAndStoreDedupsSameImageRefVariants --- .../docparser/image_resolver.go | 21 ++++++++++- .../image_resolver_relative_html_test.go | 37 +++++++++++++++++++ .../docparser/mineru_converter.go | 11 +++++- .../docparser/mineru_converter_test.go | 23 ++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go index d35bebde..16c7fb56 100644 --- a/internal/infrastructure/docparser/image_resolver.go +++ b/internal/infrastructure/docparser/image_resolver.go @@ -80,7 +80,7 @@ func (r *ImageResolver) ResolveAndStore( fileSvc interfaces.FileService, tenantID uint64, ) (updatedMarkdown string, images []StoredImage, err error) { - markdown := UnwrapLinkedImages(normalizeMinerUMarkdown(result.MarkdownContent)) + markdown := UnwrapLinkedImages(result.MarkdownContent) md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID) markdown = md2 images = append(images, imgDataURIs...) @@ -171,6 +171,22 @@ func (r *ImageResolver) saveReferencedImage( return StoredImage{}, false } + // Reuse a previously saved upload when the same source image (identified by + // ref.Filename) has already been persisted under a different markdown ref + // path (e.g. "images/foo.png" vs "./images/foo.png"). This avoids writing + // the same bytes to object storage multiple times. + if ref.Filename != "" { + if cached, ok := savedRefs["__filename__:"+ref.Filename]; ok { + stored := StoredImage{ + OriginalRef: refPath, + ServingURL: cached.ServingURL, + MimeType: cached.MimeType, + } + savedRefs[refPath] = stored + return stored, true + } + } + ext := extFromMime(ref.MimeType) if ext == "" { ext = filepath.Ext(ref.Filename) @@ -192,6 +208,9 @@ func (r *ImageResolver) saveReferencedImage( MimeType: ref.MimeType, } savedRefs[refPath] = stored + if ref.Filename != "" { + savedRefs["__filename__:"+ref.Filename] = stored + } return stored, true } diff --git a/internal/infrastructure/docparser/image_resolver_relative_html_test.go b/internal/infrastructure/docparser/image_resolver_relative_html_test.go index ccd6307f..317575c4 100644 --- a/internal/infrastructure/docparser/image_resolver_relative_html_test.go +++ b/internal/infrastructure/docparser/image_resolver_relative_html_test.go @@ -41,3 +41,40 @@ func TestResolveAndStoreRelativeHTMLImages(t *testing.T) { t.Fatalf("expected stored file url in html img src, got: %s", out) } } + +// TestResolveAndStoreDedupsSameImageRefVariants ensures that when the same +// MinerU image is referenced under multiple path forms (e.g. "images/foo.png" +// and "./images/foo.png"), the bytes are uploaded to object storage only once. +func TestResolveAndStoreDedupsSameImageRefVariants(t *testing.T) { + png := createTestPNG(200, 150) + result := &types.ReadResult{ + MarkdownContent: strings.Join([]string{ + "![](images/foo.png)", + ``, + }, "\n"), + ImageRefs: []types.ImageRef{ + { + Filename: "foo.png", + OriginalRef: "images/foo.png", + MimeType: "image/png", + ImageData: png, + }, + { + Filename: "foo.png", + OriginalRef: "./images/foo.png", + MimeType: "image/png", + ImageData: png, + }, + }, + } + + svc := &captureSaveBytes{} + r := NewImageResolver() + _, _, err := r.ResolveAndStore(context.Background(), result, svc, 1) + if err != nil { + t.Fatal(err) + } + if len(svc.saved) != 1 { + t.Fatalf("expected SaveBytes to be called once for duplicate refs, got %d", len(svc.saved)) + } +} diff --git a/internal/infrastructure/docparser/mineru_converter.go b/internal/infrastructure/docparser/mineru_converter.go index b2066072..72aa343f 100644 --- a/internal/infrastructure/docparser/mineru_converter.go +++ b/internal/infrastructure/docparser/mineru_converter.go @@ -327,12 +327,19 @@ func mineruImageOriginalRefs(mdContent, imagePath string) []string { return matched } +// imgMarkdownPatternAllowSpaces matches markdown image syntax while allowing +// spaces in the URL group, so that paths like "images/第 1 页.jpg" produced by +// MinerU on Chinese documents are still detected as image references. +var imgMarkdownPatternAllowSpaces = regexp.MustCompile( + `!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`, +) + func extractImageRefsFromContent(content string) []string { var refs []string - for _, match := range imgMarkdownPattern.FindAllStringSubmatch(content, -1) { + for _, match := range imgMarkdownPatternAllowSpaces.FindAllStringSubmatch(content, -1) { if len(match) >= 3 { - refs = append(refs, match[2]) + refs = append(refs, strings.TrimSpace(match[2])) } } for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) { diff --git a/internal/infrastructure/docparser/mineru_converter_test.go b/internal/infrastructure/docparser/mineru_converter_test.go index 36a89453..f12df39f 100644 --- a/internal/infrastructure/docparser/mineru_converter_test.go +++ b/internal/infrastructure/docparser/mineru_converter_test.go @@ -64,3 +64,26 @@ func TestProcessImagesKeepsReferencedVariants(t *testing.T) { t.Fatalf("expected 3 image refs, got %d", len(refs)) } } + +// TestProcessImagesMatchesPathsWithSpaces guards against a regression where +// MinerU image filenames containing spaces (common on Chinese documents, +// e.g. "images/第 1 页.jpg") would be silently dropped because the markdown +// regex used to extract refs disallowed whitespace inside the URL group. +func TestProcessImagesMatchesPathsWithSpaces(t *testing.T) { + reader := &MinerUReader{} + mdContent := "![](images/第 1 页.jpg)" + + png := createTestPNG(200, 150) + b64 := base64.StdEncoding.EncodeToString(png) + images := map[string]string{ + "第 1 页.jpg": "data:image/png;base64," + b64, + } + + refs, _ := reader.processImages(mdContent, images) + if len(refs) != 1 { + t.Fatalf("expected 1 image ref for path with spaces, got %d", len(refs)) + } + if refs[0].OriginalRef != "images/第 1 页.jpg" { + t.Fatalf("unexpected OriginalRef: %q", refs[0].OriginalRef) + } +}