diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go index d35bebde..16c7fb56 100644 --- a/internal/infrastructure/docparser/image_resolver.go +++ b/internal/infrastructure/docparser/image_resolver.go @@ -80,7 +80,7 @@ func (r *ImageResolver) ResolveAndStore( fileSvc interfaces.FileService, tenantID uint64, ) (updatedMarkdown string, images []StoredImage, err error) { - markdown := UnwrapLinkedImages(normalizeMinerUMarkdown(result.MarkdownContent)) + markdown := UnwrapLinkedImages(result.MarkdownContent) md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID) markdown = md2 images = append(images, imgDataURIs...) @@ -171,6 +171,22 @@ func (r *ImageResolver) saveReferencedImage( return StoredImage{}, false } + // Reuse a previously saved upload when the same source image (identified by + // ref.Filename) has already been persisted under a different markdown ref + // path (e.g. "images/foo.png" vs "./images/foo.png"). This avoids writing + // the same bytes to object storage multiple times. + if ref.Filename != "" { + if cached, ok := savedRefs["__filename__:"+ref.Filename]; ok { + stored := StoredImage{ + OriginalRef: refPath, + ServingURL: cached.ServingURL, + MimeType: cached.MimeType, + } + savedRefs[refPath] = stored + return stored, true + } + } + ext := extFromMime(ref.MimeType) if ext == "" { ext = filepath.Ext(ref.Filename) @@ -192,6 +208,9 @@ func (r *ImageResolver) saveReferencedImage( MimeType: ref.MimeType, } savedRefs[refPath] = stored + if ref.Filename != "" { + savedRefs["__filename__:"+ref.Filename] = stored + } return stored, true } diff --git a/internal/infrastructure/docparser/image_resolver_relative_html_test.go b/internal/infrastructure/docparser/image_resolver_relative_html_test.go index ccd6307f..317575c4 100644 --- a/internal/infrastructure/docparser/image_resolver_relative_html_test.go +++ b/internal/infrastructure/docparser/image_resolver_relative_html_test.go @@ -41,3 +41,40 @@ func TestResolveAndStoreRelativeHTMLImages(t *testing.T) { t.Fatalf("expected stored file url in html img src, got: %s", out) } } + +// TestResolveAndStoreDedupsSameImageRefVariants ensures that when the same +// MinerU image is referenced under multiple path forms (e.g. "images/foo.png" +// and "./images/foo.png"), the bytes are uploaded to object storage only once. +func TestResolveAndStoreDedupsSameImageRefVariants(t *testing.T) { + png := createTestPNG(200, 150) + result := &types.ReadResult{ + MarkdownContent: strings.Join([]string{ + "![](images/foo.png)", + ``, + }, "\n"), + ImageRefs: []types.ImageRef{ + { + Filename: "foo.png", + OriginalRef: "images/foo.png", + MimeType: "image/png", + ImageData: png, + }, + { + Filename: "foo.png", + OriginalRef: "./images/foo.png", + MimeType: "image/png", + ImageData: png, + }, + }, + } + + svc := &captureSaveBytes{} + r := NewImageResolver() + _, _, err := r.ResolveAndStore(context.Background(), result, svc, 1) + if err != nil { + t.Fatal(err) + } + if len(svc.saved) != 1 { + t.Fatalf("expected SaveBytes to be called once for duplicate refs, got %d", len(svc.saved)) + } +} diff --git a/internal/infrastructure/docparser/mineru_converter.go b/internal/infrastructure/docparser/mineru_converter.go index b2066072..72aa343f 100644 --- a/internal/infrastructure/docparser/mineru_converter.go +++ b/internal/infrastructure/docparser/mineru_converter.go @@ -327,12 +327,19 @@ func mineruImageOriginalRefs(mdContent, imagePath string) []string { return matched } +// imgMarkdownPatternAllowSpaces matches markdown image syntax while allowing +// spaces in the URL group, so that paths like "images/第 1 页.jpg" produced by +// MinerU on Chinese documents are still detected as image references. +var imgMarkdownPatternAllowSpaces = regexp.MustCompile( + `!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`, +) + func extractImageRefsFromContent(content string) []string { var refs []string - for _, match := range imgMarkdownPattern.FindAllStringSubmatch(content, -1) { + for _, match := range imgMarkdownPatternAllowSpaces.FindAllStringSubmatch(content, -1) { if len(match) >= 3 { - refs = append(refs, match[2]) + refs = append(refs, strings.TrimSpace(match[2])) } } for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) { diff --git a/internal/infrastructure/docparser/mineru_converter_test.go b/internal/infrastructure/docparser/mineru_converter_test.go index 36a89453..f12df39f 100644 --- a/internal/infrastructure/docparser/mineru_converter_test.go +++ b/internal/infrastructure/docparser/mineru_converter_test.go @@ -64,3 +64,26 @@ func TestProcessImagesKeepsReferencedVariants(t *testing.T) { t.Fatalf("expected 3 image refs, got %d", len(refs)) } } + +// TestProcessImagesMatchesPathsWithSpaces guards against a regression where +// MinerU image filenames containing spaces (common on Chinese documents, +// e.g. "images/第 1 页.jpg") would be silently dropped because the markdown +// regex used to extract refs disallowed whitespace inside the URL group. +func TestProcessImagesMatchesPathsWithSpaces(t *testing.T) { + reader := &MinerUReader{} + mdContent := "![](images/第 1 页.jpg)" + + png := createTestPNG(200, 150) + b64 := base64.StdEncoding.EncodeToString(png) + images := map[string]string{ + "第 1 页.jpg": "data:image/png;base64," + b64, + } + + refs, _ := reader.processImages(mdContent, images) + if len(refs) != 1 { + t.Fatalf("expected 1 image ref for path with spaces, got %d", len(refs)) + } + if refs[0].OriginalRef != "images/第 1 页.jpg" { + t.Fatalf("unexpected OriginalRef: %q", refs[0].OriginalRef) + } +}