fix(docparser): address review feedback on PR #1404

Three follow-up fixes on top of the MinerU markdown preservation work:

- Stop applying normalizeMinerUMarkdown inside ResolveAndStore. The
  helper is already called by MinerUReader.Read, and ResolveAndStore is
  shared by every parser (docreader, session attachments, ...). Running
  the heading/image unescape regexes globally would silently rewrite
  content (including inside fenced code blocks) for non-MinerU sources.

- Recognize MinerU image references whose path contains spaces, e.g.
  "images/第 1 页.jpg". The previous regex used in
  extractImageRefsFromContent disallowed whitespace in the URL group,
  so such images were never matched and never persisted. Use a
  whitespace-tolerant pattern aligned with ResolveAndStore's own
  imgPattern.

- Deduplicate uploads when the same MinerU image is referenced under
  multiple path forms (e.g. "images/foo.png" vs "./images/foo.png").
  saveReferencedImage now caches by ref.Filename in addition to the
  raw ref path, so the second variant reuses the previously stored
  ServingURL instead of writing the same bytes to object storage
  again.

Tests added:
- TestProcessImagesMatchesPathsWithSpaces
- TestResolveAndStoreDedupsSameImageRefVariants
This commit is contained in:
wizardchen
2026-05-21 11:37:24 +08:00
committed by lyingbug
parent 6210f44fb6
commit 7aca1017db
4 changed files with 89 additions and 3 deletions
@@ -80,7 +80,7 @@ func (r *ImageResolver) ResolveAndStore(
fileSvc interfaces.FileService,
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
markdown := UnwrapLinkedImages(normalizeMinerUMarkdown(result.MarkdownContent))
markdown := UnwrapLinkedImages(result.MarkdownContent)
md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID)
markdown = md2
images = append(images, imgDataURIs...)
@@ -171,6 +171,22 @@ func (r *ImageResolver) saveReferencedImage(
return StoredImage{}, false
}
// Reuse a previously saved upload when the same source image (identified by
// ref.Filename) has already been persisted under a different markdown ref
// path (e.g. "images/foo.png" vs "./images/foo.png"). This avoids writing
// the same bytes to object storage multiple times.
if ref.Filename != "" {
if cached, ok := savedRefs["__filename__:"+ref.Filename]; ok {
stored := StoredImage{
OriginalRef: refPath,
ServingURL: cached.ServingURL,
MimeType: cached.MimeType,
}
savedRefs[refPath] = stored
return stored, true
}
}
ext := extFromMime(ref.MimeType)
if ext == "" {
ext = filepath.Ext(ref.Filename)
@@ -192,6 +208,9 @@ func (r *ImageResolver) saveReferencedImage(
MimeType: ref.MimeType,
}
savedRefs[refPath] = stored
if ref.Filename != "" {
savedRefs["__filename__:"+ref.Filename] = stored
}
return stored, true
}
@@ -41,3 +41,40 @@ func TestResolveAndStoreRelativeHTMLImages(t *testing.T) {
t.Fatalf("expected stored file url in html img src, got: %s", out)
}
}
// TestResolveAndStoreDedupsSameImageRefVariants ensures that when the same
// MinerU image is referenced under multiple path forms (e.g. "images/foo.png"
// and "./images/foo.png"), the bytes are uploaded to object storage only once.
func TestResolveAndStoreDedupsSameImageRefVariants(t *testing.T) {
png := createTestPNG(200, 150)
result := &types.ReadResult{
MarkdownContent: strings.Join([]string{
"![](images/foo.png)",
`<img src="./images/foo.png"/>`,
}, "\n"),
ImageRefs: []types.ImageRef{
{
Filename: "foo.png",
OriginalRef: "images/foo.png",
MimeType: "image/png",
ImageData: png,
},
{
Filename: "foo.png",
OriginalRef: "./images/foo.png",
MimeType: "image/png",
ImageData: png,
},
},
}
svc := &captureSaveBytes{}
r := NewImageResolver()
_, _, err := r.ResolveAndStore(context.Background(), result, svc, 1)
if err != nil {
t.Fatal(err)
}
if len(svc.saved) != 1 {
t.Fatalf("expected SaveBytes to be called once for duplicate refs, got %d", len(svc.saved))
}
}
@@ -327,12 +327,19 @@ func mineruImageOriginalRefs(mdContent, imagePath string) []string {
return matched
}
// imgMarkdownPatternAllowSpaces matches markdown image syntax while allowing
// spaces in the URL group, so that paths like "images/第 1 页.jpg" produced by
// MinerU on Chinese documents are still detected as image references.
var imgMarkdownPatternAllowSpaces = regexp.MustCompile(
`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`,
)
func extractImageRefsFromContent(content string) []string {
var refs []string
for _, match := range imgMarkdownPattern.FindAllStringSubmatch(content, -1) {
for _, match := range imgMarkdownPatternAllowSpaces.FindAllStringSubmatch(content, -1) {
if len(match) >= 3 {
refs = append(refs, match[2])
refs = append(refs, strings.TrimSpace(match[2]))
}
}
for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) {
@@ -64,3 +64,26 @@ func TestProcessImagesKeepsReferencedVariants(t *testing.T) {
t.Fatalf("expected 3 image refs, got %d", len(refs))
}
}
// TestProcessImagesMatchesPathsWithSpaces guards against a regression where
// MinerU image filenames containing spaces (common on Chinese documents,
// e.g. "images/第 1 页.jpg") would be silently dropped because the markdown
// regex used to extract refs disallowed whitespace inside the URL group.
func TestProcessImagesMatchesPathsWithSpaces(t *testing.T) {
reader := &MinerUReader{}
mdContent := "![](images/第 1 页.jpg)"
png := createTestPNG(200, 150)
b64 := base64.StdEncoding.EncodeToString(png)
images := map[string]string{
"第 1 页.jpg": "data:image/png;base64," + b64,
}
refs, _ := reader.processImages(mdContent, images)
if len(refs) != 1 {
t.Fatalf("expected 1 image ref for path with spaces, got %d", len(refs))
}
if refs[0].OriginalRef != "images/第 1 页.jpg" {
t.Fatalf("unexpected OriginalRef: %q", refs[0].OriginalRef)
}
}