fix: preserve MHTML image resolution and Markdown structure (#1743)

* fix(docreader): preserve MHTML blocks and resolve titled images

* fix(docreader): harden MHTML markdown image resolution

* fix(docreader): preserve markdown hard break spaces

---------

Co-authored-by: FFFFFFpy <bushiyan@outlook.com>
This commit is contained in:
FFFFFFpy
2026-06-22 15:53:54 +08:00
committed by GitHub
parent 5f7ed2ffe9
commit a6a20175e5
8 changed files with 1112 additions and 16 deletions
+107 -6
View File
@@ -8,7 +8,7 @@ import email
import html
import logging
import os
from urllib.parse import unquote, urljoin
from urllib.parse import unquote, urljoin, urlparse
import uuid
from typing import Dict
@@ -70,8 +70,7 @@ class MHTMLParser(BaseParser):
elif content_type.startswith("image/") and self.extract_images:
image_data = part.get_payload(decode=True)
if image_data:
ext = self._image_extension(content_type)
image_path = f"images/{uuid.uuid4().hex}{ext}"
image_path = self._image_path_for_part(part, content_type, images)
images[image_path] = base64.b64encode(image_data).decode("utf-8")
self._add_image_aliases(image_aliases, part, image_path)
@@ -154,6 +153,45 @@ class MHTMLParser(BaseParser):
"image/x-icon": ".ico",
}.get(content_type, ".png")
@classmethod
def _image_path_for_part(
cls, part, content_type: str, images: Dict[str, str]
) -> str:
"""Choose a stable image path when the MHTML part exposes a filename."""
ext = cls._image_extension(content_type)
location = (part.get("Content-Location", "") or "").strip()
filename = cls._filename_from_content_location(location)
if not filename:
return f"images/{uuid.uuid4().hex}{ext}"
stem, location_ext = os.path.splitext(filename)
if not location_ext:
filename = f"{filename}{ext}"
image_path = f"images/{filename}"
if image_path not in images:
return image_path
suffix = 2
stem, location_ext = os.path.splitext(filename)
while True:
candidate = f"images/{stem}_{suffix}{location_ext}"
if candidate not in images:
return candidate
suffix += 1
@staticmethod
def _filename_from_content_location(location: str) -> str:
decoded = unquote(html.unescape(location.strip()))
if not decoded or decoded.lower().startswith("cid:"):
return ""
path = urlparse(decoded).path or decoded
filename = os.path.basename(path)
if not filename or filename in {".", ".."}:
return ""
if "/" in filename or "\\" in filename:
return ""
return filename
def _html_to_markdown(
self,
html_content: str,
@@ -171,9 +209,7 @@ class MHTMLParser(BaseParser):
self._rewrite_image_sources(soup, image_aliases, base_location)
text_fallback = soup.get_text(separator="\n", strip=True)
markdown_text = md(str(soup), heading_style="ATX")
result = "\n".join(
line.strip() for line in markdown_text.split("\n") if line.strip()
)
result = self._normalize_markdown(markdown_text)
if not result and text_fallback:
logger.warning("Markdown empty, falling back to text extraction")
return text_fallback
@@ -187,6 +223,71 @@ class MHTMLParser(BaseParser):
logger.error("HTML to Markdown conversion failed: %s", e)
return f"```html\n{html_content}\n```"
@staticmethod
def _normalize_markdown(markdown_text: str) -> str:
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
output: list[str] = []
pending_blank = False
fence_char: str | None = None
fence_len = 0
for line in text.split("\n"):
if fence_char is not None:
output.append(line)
if MHTMLParser._is_closing_fence(line, fence_char, fence_len):
fence_char = None
fence_len = 0
continue
opening = MHTMLParser._opening_fence(line)
if opening is not None:
if pending_blank and output:
output.append("")
pending_blank = False
output.append(line)
fence_char, fence_len = opening
continue
if not line.strip(" \t"):
pending_blank = True
continue
if pending_blank and output:
output.append("")
pending_blank = False
trailing_spaces = len(line) - len(line.rstrip(" "))
if trailing_spaces >= 2:
line = line.rstrip(" \t") + " "
else:
line = line.rstrip(" \t")
output.append(line)
return "\n".join(output).strip("\n")
@staticmethod
def _opening_fence(line: str) -> tuple[str, int] | None:
stripped = line.lstrip(" ")
if len(line) - len(stripped) > 3 or not stripped:
return None
fence_char = stripped[0]
if fence_char not in {"`", "~"}:
return None
fence_len = len(stripped) - len(stripped.lstrip(fence_char))
if fence_len < 3:
return None
return fence_char, fence_len
@staticmethod
def _is_closing_fence(line: str, fence_char: str, fence_len: int) -> bool:
stripped = line.lstrip(" ")
if len(line) - len(stripped) > 3:
return False
closing_len = len(stripped) - len(stripped.lstrip(fence_char))
if closing_len < fence_len:
return False
return not stripped[closing_len:].strip(" \t")
@staticmethod
def _strip_internal_links(soup: BeautifulSoup) -> None:
"""Unwrap links that don't point to an external resource."""
+147
View File
@@ -1,9 +1,14 @@
import base64
import json
import unittest
from email.message import EmailMessage
from pathlib import Path
from docreader.parser.mhtml_parser import MHTMLParser
from docreader.parser.registry import registry
REPO_ROOT = Path(__file__).resolve().parents[2]
def _minimal_mhtml_bytes() -> bytes:
root = EmailMessage()
@@ -46,6 +51,39 @@ def _minimal_mhtml_bytes() -> bytes:
return root.as_bytes()
def _mhtml_with_table_image_and_caption() -> bytes:
root = EmailMessage()
root["Subject"] = "MHTML with table image"
root.make_related()
main = EmailMessage()
main.set_content(
"<html><body>"
"<table>"
"<tr><th>体验方向</th><th>代表内容</th></tr>"
"<tr><td>赛季制建立</td><td>BP、Rank</td></tr>"
"</table>"
'<img src="cid:test-image" alt="图片" title="图片">'
"<p>高机动性身法与独特枪械反馈</p>"
"</body></html>",
subtype="html",
)
main["Content-Location"] = "https://example.com/article"
root.attach(main)
image = EmailMessage()
image.set_content(
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00"
b"\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;",
maintype="image",
subtype="gif",
)
image["Content-ID"] = "<test-image>"
root.attach(image)
return root.as_bytes()
class MHTMLParserTest(unittest.TestCase):
def test_parse_selects_main_html_and_filters_noise(self):
document = MHTMLParser(
@@ -84,6 +122,115 @@ class MHTMLParserTest(unittest.TestCase):
self.assertNotIn("cid:tiny-image", with_images.content)
self.assertEqual(without_images.images, {})
def test_table_image_and_caption_keep_markdown_block_boundaries(self):
document = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
).parse_into_text(_mhtml_with_table_image_and_caption())
self.assertEqual(len(document.images), 1)
image_ref = next(iter(document.images))
self.assertIn(f'![图片]({image_ref} "图片")', document.content)
self.assertIn(
f"| 赛季制建立 | BP、Rank |\n\n![图片]({image_ref} \"图片\")",
document.content,
)
self.assertIn(
f'![图片]({image_ref} "图片")\n\n高机动性身法与独特枪械反馈',
document.content,
)
def test_html_to_markdown_preserves_indentation_and_code_block_blanks(self):
markdown = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
)._html_to_markdown(
"<ul><li>parent<ul><li>child</li></ul></li></ul>"
"<blockquote><p>quoted</p></blockquote>"
"<pre><code>line1\n\n indented\n</code></pre>"
)
self.assertIn("* parent\n + child", markdown)
self.assertIn("\n\n> quoted\n\n", markdown)
self.assertIn("```\nline1\n\n indented\n```", markdown)
def test_html_to_markdown_preserves_nested_list_indentation(self):
markdown = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
)._html_to_markdown("<ul><li>parent<ul><li>child</li></ul></li></ul>")
self.assertIn("* parent\n + child", markdown)
def test_html_to_markdown_preserves_blockquote_boundaries(self):
markdown = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
)._html_to_markdown("<p>before</p><blockquote><p>quoted</p></blockquote><p>after</p>")
self.assertIn("before\n\n> quoted\n\nafter", markdown)
def test_html_to_markdown_preserves_fenced_code_blank_lines(self):
markdown = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
)._html_to_markdown("<pre><code>line1\n\n\nline2\n</code></pre>")
self.assertIn("```\nline1\n\n\nline2\n```", markdown)
def test_html_to_markdown_collapses_excess_blank_lines_outside_code(self):
markdown = MHTMLParser._normalize_markdown("alpha\n\n \n\t\n\nbeta")
self.assertEqual(markdown, "alpha\n\nbeta")
self.assertNotIn("\n\n\n", markdown)
def test_html_to_markdown_preserves_hard_break_spaces(self):
markdown = MHTMLParser(
file_name="article.mhtml", file_type="mhtml"
)._html_to_markdown("<p>alpha<br>beta</p>")
self.assertEqual(markdown, "alpha \nbeta")
def test_normalize_markdown_preserves_two_space_hard_break(self):
markdown = MHTMLParser._normalize_markdown("alpha \nbeta")
self.assertEqual(markdown, "alpha \nbeta")
def test_html_to_markdown_normalizes_crlf(self):
markdown = MHTMLParser._normalize_markdown("alpha\r\n\r\nbeta\rgamma")
self.assertEqual(markdown, "alpha\n\nbeta\ngamma")
def test_html_to_markdown_does_not_strip_leading_indentation_at_document_start(self):
markdown = MHTMLParser._normalize_markdown(" indented start\n")
self.assertEqual(markdown, " indented start")
def test_mhtml_shared_contract_fixture(self):
fixture = REPO_ROOT / "testdata" / "mhtml" / "titled-image.mhtml"
contract_path = REPO_ROOT / "testdata" / "mhtml" / "titled-image-contract.json"
contract = json.loads(contract_path.read_text(encoding="utf-8"))
document = MHTMLParser(
file_name="titled-image.mhtml", file_type="mhtml"
).parse_into_text(fixture.read_bytes())
self.assertEqual(document.content, contract["markdown_content"])
self.assertEqual(len(document.images), 1)
image_contract = contract["images"][0]
self.assertIn(image_contract["original_ref"], document.images)
self.assertEqual(
base64.b64decode(document.images[image_contract["original_ref"]]),
base64.b64decode(image_contract["image_data_base64"]),
)
self.assertIn(
'| 赛季制建立 | BP、Rank |\n\n![图片](images/第 1 页 (测试).gif "阶段 1) 图片")',
document.content,
)
self.assertIn(
'![图片](images/第 1 页 (测试).gif "阶段 1) 图片")\n\n高机动性身法与独特枪械反馈',
document.content,
)
self.assertNotIn("\n\n\n", document.content.split("```", 1)[0])
self.assertIn("```\nline1\n\n\nline2\n```", document.content)
self.assertIn("* parent\n + child", document.content)
self.assertIn("alpha \nbeta", document.content)
def test_registry_resolves_mhtml(self):
self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser)