fix: preserve MHTML image resolution and Markdown structure (#1743)
* fix(docreader): preserve MHTML blocks and resolve titled images * fix(docreader): harden MHTML markdown image resolution * fix(docreader): preserve markdown hard break spaces --------- Co-authored-by: FFFFFFpy <bushiyan@outlook.com>
This commit is contained in:
@@ -8,7 +8,7 @@ import email
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
from urllib.parse import unquote, urljoin
|
||||
from urllib.parse import unquote, urljoin, urlparse
|
||||
import uuid
|
||||
from typing import Dict
|
||||
|
||||
@@ -70,8 +70,7 @@ class MHTMLParser(BaseParser):
|
||||
elif content_type.startswith("image/") and self.extract_images:
|
||||
image_data = part.get_payload(decode=True)
|
||||
if image_data:
|
||||
ext = self._image_extension(content_type)
|
||||
image_path = f"images/{uuid.uuid4().hex}{ext}"
|
||||
image_path = self._image_path_for_part(part, content_type, images)
|
||||
images[image_path] = base64.b64encode(image_data).decode("utf-8")
|
||||
self._add_image_aliases(image_aliases, part, image_path)
|
||||
|
||||
@@ -154,6 +153,45 @@ class MHTMLParser(BaseParser):
|
||||
"image/x-icon": ".ico",
|
||||
}.get(content_type, ".png")
|
||||
|
||||
@classmethod
|
||||
def _image_path_for_part(
|
||||
cls, part, content_type: str, images: Dict[str, str]
|
||||
) -> str:
|
||||
"""Choose a stable image path when the MHTML part exposes a filename."""
|
||||
ext = cls._image_extension(content_type)
|
||||
location = (part.get("Content-Location", "") or "").strip()
|
||||
filename = cls._filename_from_content_location(location)
|
||||
if not filename:
|
||||
return f"images/{uuid.uuid4().hex}{ext}"
|
||||
|
||||
stem, location_ext = os.path.splitext(filename)
|
||||
if not location_ext:
|
||||
filename = f"{filename}{ext}"
|
||||
image_path = f"images/{filename}"
|
||||
if image_path not in images:
|
||||
return image_path
|
||||
|
||||
suffix = 2
|
||||
stem, location_ext = os.path.splitext(filename)
|
||||
while True:
|
||||
candidate = f"images/{stem}_{suffix}{location_ext}"
|
||||
if candidate not in images:
|
||||
return candidate
|
||||
suffix += 1
|
||||
|
||||
@staticmethod
|
||||
def _filename_from_content_location(location: str) -> str:
|
||||
decoded = unquote(html.unescape(location.strip()))
|
||||
if not decoded or decoded.lower().startswith("cid:"):
|
||||
return ""
|
||||
path = urlparse(decoded).path or decoded
|
||||
filename = os.path.basename(path)
|
||||
if not filename or filename in {".", ".."}:
|
||||
return ""
|
||||
if "/" in filename or "\\" in filename:
|
||||
return ""
|
||||
return filename
|
||||
|
||||
def _html_to_markdown(
|
||||
self,
|
||||
html_content: str,
|
||||
@@ -171,9 +209,7 @@ class MHTMLParser(BaseParser):
|
||||
self._rewrite_image_sources(soup, image_aliases, base_location)
|
||||
text_fallback = soup.get_text(separator="\n", strip=True)
|
||||
markdown_text = md(str(soup), heading_style="ATX")
|
||||
result = "\n".join(
|
||||
line.strip() for line in markdown_text.split("\n") if line.strip()
|
||||
)
|
||||
result = self._normalize_markdown(markdown_text)
|
||||
if not result and text_fallback:
|
||||
logger.warning("Markdown empty, falling back to text extraction")
|
||||
return text_fallback
|
||||
@@ -187,6 +223,71 @@ class MHTMLParser(BaseParser):
|
||||
logger.error("HTML to Markdown conversion failed: %s", e)
|
||||
return f"```html\n{html_content}\n```"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_markdown(markdown_text: str) -> str:
|
||||
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
output: list[str] = []
|
||||
pending_blank = False
|
||||
fence_char: str | None = None
|
||||
fence_len = 0
|
||||
|
||||
for line in text.split("\n"):
|
||||
if fence_char is not None:
|
||||
output.append(line)
|
||||
if MHTMLParser._is_closing_fence(line, fence_char, fence_len):
|
||||
fence_char = None
|
||||
fence_len = 0
|
||||
continue
|
||||
|
||||
opening = MHTMLParser._opening_fence(line)
|
||||
if opening is not None:
|
||||
if pending_blank and output:
|
||||
output.append("")
|
||||
pending_blank = False
|
||||
output.append(line)
|
||||
fence_char, fence_len = opening
|
||||
continue
|
||||
|
||||
if not line.strip(" \t"):
|
||||
pending_blank = True
|
||||
continue
|
||||
|
||||
if pending_blank and output:
|
||||
output.append("")
|
||||
pending_blank = False
|
||||
|
||||
trailing_spaces = len(line) - len(line.rstrip(" "))
|
||||
if trailing_spaces >= 2:
|
||||
line = line.rstrip(" \t") + " "
|
||||
else:
|
||||
line = line.rstrip(" \t")
|
||||
output.append(line)
|
||||
|
||||
return "\n".join(output).strip("\n")
|
||||
|
||||
@staticmethod
|
||||
def _opening_fence(line: str) -> tuple[str, int] | None:
|
||||
stripped = line.lstrip(" ")
|
||||
if len(line) - len(stripped) > 3 or not stripped:
|
||||
return None
|
||||
fence_char = stripped[0]
|
||||
if fence_char not in {"`", "~"}:
|
||||
return None
|
||||
fence_len = len(stripped) - len(stripped.lstrip(fence_char))
|
||||
if fence_len < 3:
|
||||
return None
|
||||
return fence_char, fence_len
|
||||
|
||||
@staticmethod
|
||||
def _is_closing_fence(line: str, fence_char: str, fence_len: int) -> bool:
|
||||
stripped = line.lstrip(" ")
|
||||
if len(line) - len(stripped) > 3:
|
||||
return False
|
||||
closing_len = len(stripped) - len(stripped.lstrip(fence_char))
|
||||
if closing_len < fence_len:
|
||||
return False
|
||||
return not stripped[closing_len:].strip(" \t")
|
||||
|
||||
@staticmethod
|
||||
def _strip_internal_links(soup: BeautifulSoup) -> None:
|
||||
"""Unwrap links that don't point to an external resource."""
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import base64
|
||||
import json
|
||||
import unittest
|
||||
from email.message import EmailMessage
|
||||
from pathlib import Path
|
||||
|
||||
from docreader.parser.mhtml_parser import MHTMLParser
|
||||
from docreader.parser.registry import registry
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _minimal_mhtml_bytes() -> bytes:
|
||||
root = EmailMessage()
|
||||
@@ -46,6 +51,39 @@ def _minimal_mhtml_bytes() -> bytes:
|
||||
return root.as_bytes()
|
||||
|
||||
|
||||
def _mhtml_with_table_image_and_caption() -> bytes:
|
||||
root = EmailMessage()
|
||||
root["Subject"] = "MHTML with table image"
|
||||
root.make_related()
|
||||
|
||||
main = EmailMessage()
|
||||
main.set_content(
|
||||
"<html><body>"
|
||||
"<table>"
|
||||
"<tr><th>体验方向</th><th>代表内容</th></tr>"
|
||||
"<tr><td>赛季制建立</td><td>BP、Rank</td></tr>"
|
||||
"</table>"
|
||||
'<img src="cid:test-image" alt="图片" title="图片">'
|
||||
"<p>高机动性身法与独特枪械反馈</p>"
|
||||
"</body></html>",
|
||||
subtype="html",
|
||||
)
|
||||
main["Content-Location"] = "https://example.com/article"
|
||||
root.attach(main)
|
||||
|
||||
image = EmailMessage()
|
||||
image.set_content(
|
||||
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00"
|
||||
b"\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;",
|
||||
maintype="image",
|
||||
subtype="gif",
|
||||
)
|
||||
image["Content-ID"] = "<test-image>"
|
||||
root.attach(image)
|
||||
|
||||
return root.as_bytes()
|
||||
|
||||
|
||||
class MHTMLParserTest(unittest.TestCase):
|
||||
def test_parse_selects_main_html_and_filters_noise(self):
|
||||
document = MHTMLParser(
|
||||
@@ -84,6 +122,115 @@ class MHTMLParserTest(unittest.TestCase):
|
||||
self.assertNotIn("cid:tiny-image", with_images.content)
|
||||
self.assertEqual(without_images.images, {})
|
||||
|
||||
def test_table_image_and_caption_keep_markdown_block_boundaries(self):
|
||||
document = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
).parse_into_text(_mhtml_with_table_image_and_caption())
|
||||
|
||||
self.assertEqual(len(document.images), 1)
|
||||
image_ref = next(iter(document.images))
|
||||
self.assertIn(f'', document.content)
|
||||
self.assertIn(
|
||||
f"| 赛季制建立 | BP、Rank |\n\n",
|
||||
document.content,
|
||||
)
|
||||
self.assertIn(
|
||||
f'\n\n高机动性身法与独特枪械反馈',
|
||||
document.content,
|
||||
)
|
||||
|
||||
def test_html_to_markdown_preserves_indentation_and_code_block_blanks(self):
|
||||
markdown = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
)._html_to_markdown(
|
||||
"<ul><li>parent<ul><li>child</li></ul></li></ul>"
|
||||
"<blockquote><p>quoted</p></blockquote>"
|
||||
"<pre><code>line1\n\n indented\n</code></pre>"
|
||||
)
|
||||
|
||||
self.assertIn("* parent\n + child", markdown)
|
||||
self.assertIn("\n\n> quoted\n\n", markdown)
|
||||
self.assertIn("```\nline1\n\n indented\n```", markdown)
|
||||
|
||||
def test_html_to_markdown_preserves_nested_list_indentation(self):
|
||||
markdown = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
)._html_to_markdown("<ul><li>parent<ul><li>child</li></ul></li></ul>")
|
||||
|
||||
self.assertIn("* parent\n + child", markdown)
|
||||
|
||||
def test_html_to_markdown_preserves_blockquote_boundaries(self):
|
||||
markdown = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
)._html_to_markdown("<p>before</p><blockquote><p>quoted</p></blockquote><p>after</p>")
|
||||
|
||||
self.assertIn("before\n\n> quoted\n\nafter", markdown)
|
||||
|
||||
def test_html_to_markdown_preserves_fenced_code_blank_lines(self):
|
||||
markdown = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
)._html_to_markdown("<pre><code>line1\n\n\nline2\n</code></pre>")
|
||||
|
||||
self.assertIn("```\nline1\n\n\nline2\n```", markdown)
|
||||
|
||||
def test_html_to_markdown_collapses_excess_blank_lines_outside_code(self):
|
||||
markdown = MHTMLParser._normalize_markdown("alpha\n\n \n\t\n\nbeta")
|
||||
|
||||
self.assertEqual(markdown, "alpha\n\nbeta")
|
||||
self.assertNotIn("\n\n\n", markdown)
|
||||
|
||||
def test_html_to_markdown_preserves_hard_break_spaces(self):
|
||||
markdown = MHTMLParser(
|
||||
file_name="article.mhtml", file_type="mhtml"
|
||||
)._html_to_markdown("<p>alpha<br>beta</p>")
|
||||
|
||||
self.assertEqual(markdown, "alpha \nbeta")
|
||||
|
||||
def test_normalize_markdown_preserves_two_space_hard_break(self):
|
||||
markdown = MHTMLParser._normalize_markdown("alpha \nbeta")
|
||||
|
||||
self.assertEqual(markdown, "alpha \nbeta")
|
||||
|
||||
def test_html_to_markdown_normalizes_crlf(self):
|
||||
markdown = MHTMLParser._normalize_markdown("alpha\r\n\r\nbeta\rgamma")
|
||||
|
||||
self.assertEqual(markdown, "alpha\n\nbeta\ngamma")
|
||||
|
||||
def test_html_to_markdown_does_not_strip_leading_indentation_at_document_start(self):
|
||||
markdown = MHTMLParser._normalize_markdown(" indented start\n")
|
||||
|
||||
self.assertEqual(markdown, " indented start")
|
||||
|
||||
def test_mhtml_shared_contract_fixture(self):
|
||||
fixture = REPO_ROOT / "testdata" / "mhtml" / "titled-image.mhtml"
|
||||
contract_path = REPO_ROOT / "testdata" / "mhtml" / "titled-image-contract.json"
|
||||
contract = json.loads(contract_path.read_text(encoding="utf-8"))
|
||||
|
||||
document = MHTMLParser(
|
||||
file_name="titled-image.mhtml", file_type="mhtml"
|
||||
).parse_into_text(fixture.read_bytes())
|
||||
|
||||
self.assertEqual(document.content, contract["markdown_content"])
|
||||
self.assertEqual(len(document.images), 1)
|
||||
image_contract = contract["images"][0]
|
||||
self.assertIn(image_contract["original_ref"], document.images)
|
||||
self.assertEqual(
|
||||
base64.b64decode(document.images[image_contract["original_ref"]]),
|
||||
base64.b64decode(image_contract["image_data_base64"]),
|
||||
)
|
||||
self.assertIn(
|
||||
'| 赛季制建立 | BP、Rank |\n\n.gif "阶段 1) 图片")',
|
||||
document.content,
|
||||
)
|
||||
self.assertIn(
|
||||
'.gif "阶段 1) 图片")\n\n高机动性身法与独特枪械反馈',
|
||||
document.content,
|
||||
)
|
||||
self.assertNotIn("\n\n\n", document.content.split("```", 1)[0])
|
||||
self.assertIn("```\nline1\n\n\nline2\n```", document.content)
|
||||
self.assertIn("* parent\n + child", document.content)
|
||||
self.assertIn("alpha \nbeta", document.content)
|
||||
|
||||
def test_registry_resolves_mhtml(self):
|
||||
self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user