feat: add EPUB and MHTML document support

This commit is contained in:
FFFFFFpy
2026-06-16 10:34:58 +09:00
committed by lyingbug
parent fe7aded648
commit e21a5da422
20 changed files with 770 additions and 18 deletions
+332
View File
@@ -0,0 +1,332 @@
"""EPUB parser.
Parses EPUB files into markdown text and optional embedded images.
"""
import base64
import logging
import os
import posixpath
import tempfile
from urllib.parse import unquote
import uuid
from typing import Dict
from bs4 import BeautifulSoup
import ebooklib
from ebooklib import epub
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
logger = logging.getLogger(__name__)
class EPUBParser(BaseParser):
"""Parser for EPUB e-book files."""
def __init__(self, *args, extract_images: bool = True, **kwargs):
super().__init__(*args, **kwargs)
self.extract_images = extract_images
def parse_into_text(self, content: bytes) -> Document:
logger.info(
"Parsing EPUB file: %s, size: %d bytes", self.file_name, len(content)
)
try:
with tempfile.NamedTemporaryFile(
suffix=".epub", delete=False, mode="wb"
) as epub_file:
epub_file.write(content)
epub_path = epub_file.name
try:
book = epub.read_epub(epub_path)
metadata = self._extract_metadata(book)
markdown_content, images = self._extract_content(book)
metadata["source_format"] = "epub"
metadata["file_size"] = len(content)
metadata["chapter_count"] = len(
[part for part in markdown_content.split("\n## ") if part.strip()]
)
metadata["image_count"] = len(images)
return Document(
content=markdown_content, images=images, metadata=metadata
)
finally:
if os.path.exists(epub_path):
os.unlink(epub_path)
except ImportError:
logger.error("ebooklib not installed")
raise
except Exception as e:
logger.warning(
"ebooklib failed to parse EPUB: %s, trying ZIP fallback", str(e)
)
return self._parse_epub_fallback(content)
def _parse_epub_fallback(self, content: bytes) -> Document:
"""Parse EPUB directly as a ZIP when ebooklib cannot read it."""
import re
import zipfile
from io import BytesIO
metadata = {"source_format": "epub", "file_size": len(content)}
images: Dict[str, str] = {}
image_aliases: Dict[str, str] = {}
markdown_parts = []
with zipfile.ZipFile(BytesIO(content), "r") as epub_zip:
html_files = [
f
for f in epub_zip.namelist()
if f.endswith((".html", ".xhtml", ".htm"))
]
def chapter_num(filename: str) -> int:
match = re.search(r"chapter(\d+)", filename, re.IGNORECASE)
return int(match.group(1)) if match else 999999
html_files.sort(key=chapter_num)
if self.extract_images:
img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg")
for img_file in epub_zip.namelist():
if img_file.lower().endswith(img_exts):
try:
img_data = epub_zip.read(img_file)
ext = os.path.splitext(img_file)[1]
img_path = f"images/{uuid.uuid4().hex}{ext}"
images[img_path] = base64.b64encode(img_data).decode(
"utf-8"
)
self._add_image_aliases(image_aliases, img_file, img_path)
except Exception as e:
logger.warning(
"Failed to extract image %s: %s", img_file, e
)
for html_file in html_files:
try:
html_content = epub_zip.read(html_file).decode(
"utf-8", errors="ignore"
)
chapter_md = self._html_to_markdown(
html_content,
image_aliases=image_aliases,
base_path=posixpath.dirname(html_file),
)
base = os.path.basename(html_file)
title = base.replace(".html", "").replace(".xhtml", "")
title = re.sub(
r"chapter[_-]?", "Chapter ", title, flags=re.IGNORECASE
)
title = title.replace("_", " ").replace("-", " ").title()
if chapter_md.strip():
markdown_parts.append(f"## {title}\n\n{chapter_md}")
except Exception as e:
logger.warning("Failed to process %s: %s", html_file, e)
metadata["chapter_count"] = len(markdown_parts)
metadata["image_count"] = len(images)
return Document(
content="\n\n".join(markdown_parts), images=images, metadata=metadata
)
def _extract_metadata(self, book) -> Dict[str, str]:
metadata: Dict[str, str] = {}
mapping = {
"title": "title",
"creator": "author",
"publisher": "publisher",
"language": "language",
"description": "description",
"date": "date",
"identifier": "isbn",
}
for dc_key, out_key in mapping.items():
try:
values = book.get_metadata("DC", dc_key)
except Exception:
values = None
if values:
if out_key == "author":
metadata[out_key] = ", ".join(value[0] for value in values)
else:
metadata[out_key] = values[0][0]
return metadata
def _extract_content(self, book) -> tuple[str, Dict[str, str]]:
markdown_parts = []
images: Dict[str, str] = {}
image_aliases: Dict[str, str] = {}
try:
toc = book.get_table_of_contents()
except Exception as e:
logger.debug("Failed to get TOC: %s, processing all HTML items", e)
toc = []
html_items = {}
for item in book.get_items():
if item.get_type() == ebooklib.ITEM_DOCUMENT:
html_items[item.get_name()] = item
if self.extract_images:
for item in book.get_items():
if item.get_type() == ebooklib.ITEM_IMAGE:
img_data = item.get_content()
ext = os.path.splitext(item.get_name())[1]
img_path = f"images/{uuid.uuid4().hex}{ext}"
images[img_path] = base64.b64encode(img_data).decode("utf-8")
self._add_image_aliases(image_aliases, item.get_name(), img_path)
if toc:
for item in toc:
entries = item if isinstance(item, tuple) else (item,)
for sub in entries:
if hasattr(sub, "get_name") and sub.get_name() in html_items:
markdown_parts.append(
self._process_chapter(
html_items[sub.get_name()],
toc_index=len(markdown_parts),
image_aliases=image_aliases,
)
)
if not markdown_parts:
for _name, item in html_items.items():
markdown_parts.append(
self._process_chapter(
item,
toc_index=len(markdown_parts),
image_aliases=image_aliases,
)
)
return "\n\n".join(part for part in markdown_parts if part.strip()), images
def _process_chapter(
self,
html_item,
toc_index: int = 0,
image_aliases: Dict[str, str] | None = None,
) -> str:
try:
html_content = html_item.get_content()
soup = BeautifulSoup(html_content, "lxml")
title_tag = soup.find(["h1", "h2"])
if title_tag:
chapter_title = title_tag.get_text().strip()
title_tag.decompose()
else:
chapter_title = html_item.get_name().replace("/", "")
chapter_title = chapter_title.replace(".xhtml", "")
chapter_title = chapter_title.replace("-", " ").title()
body_html = str(soup.body) if soup.body else str(html_content)
chapter_md = self._html_to_markdown(
body_html,
image_aliases=image_aliases,
base_path=posixpath.dirname(html_item.get_name()),
)
return f"## {chapter_title}\n\n{chapter_md}"
except Exception as e:
logger.error(
"Failed to process chapter %s: %s", html_item.get_name(), e
)
return f"## Chapter {toc_index + 1}\n\n[Error processing chapter: {e}]"
def _html_to_markdown(
self,
html_content: str,
image_aliases: Dict[str, str] | None = None,
base_path: str = "",
) -> str:
try:
from bs4 import Comment
from markdownify import markdownify as md
soup = BeautifulSoup(html_content, "lxml")
for element in soup(["script", "style"]):
element.decompose()
for comment in soup.find_all(
string=lambda text: isinstance(text, Comment)
):
comment.extract()
self._strip_internal_links(soup)
if image_aliases:
self._rewrite_image_sources(soup, image_aliases, base_path)
markdown_text = md(str(soup), heading_style="ATX")
return "\n".join(
line.strip() for line in markdown_text.split("\n") if line.strip()
)
except ImportError:
logger.warning("markdownify not available, using HTML as-is")
return f"```html\n{html_content}\n```"
except Exception as e:
logger.error("HTML to Markdown conversion failed: %s", e)
return f"```html\n{html_content}\n```"
@staticmethod
def _strip_internal_links(soup: BeautifulSoup) -> None:
"""Unwrap links that don't point to an external resource.
EPUB internal links (other chapter files, ``#fragment`` anchors, TOC
entries) become dead links after extraction. Keep only external links
and replace everything else with its text.
"""
external = ("http://", "https://", "mailto:", "tel:")
for link in soup.find_all("a"):
href = (link.get("href") or "").strip().lower()
if not href or not href.startswith(external):
link.unwrap()
@staticmethod
def _add_image_aliases(
image_aliases: Dict[str, str],
original_path: str,
image_path: str,
) -> None:
normalized = EPUBParser._normalize_epub_path(original_path)
aliases = {
original_path,
normalized,
unquote(original_path),
unquote(normalized),
posixpath.basename(normalized),
}
for alias in aliases:
if alias:
image_aliases[alias] = image_path
@staticmethod
def _rewrite_image_sources(
soup: BeautifulSoup,
image_aliases: Dict[str, str],
base_path: str = "",
) -> None:
for img in soup.find_all("img"):
src = (img.get("src") or "").strip()
if not src:
continue
normalized_src = EPUBParser._normalize_epub_path(src)
candidates = [
src,
normalized_src,
unquote(src),
unquote(normalized_src),
posixpath.basename(normalized_src),
]
if base_path:
joined = EPUBParser._normalize_epub_path(posixpath.join(base_path, src))
candidates.extend([joined, unquote(joined)])
for candidate in candidates:
if candidate in image_aliases:
img["src"] = image_aliases[candidate]
break
@staticmethod
def _normalize_epub_path(path: str) -> str:
path = unquote(path).split("#", 1)[0].split("?", 1)[0].replace("\\", "/")
normalized = posixpath.normpath(path)
return "" if normalized == "." else normalized.lstrip("/")
+222
View File
@@ -0,0 +1,222 @@
"""MHTML parser.
Parses MIME HTML web archives into markdown text and optional embedded images.
"""
import base64
import email
import html
import logging
import os
from urllib.parse import unquote, urljoin
import uuid
from typing import Dict
from bs4 import BeautifulSoup
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
logger = logging.getLogger(__name__)
_AD_DOMAINS = (
"googleads",
"doubleclick",
"googlesyndication",
"facebook.com/tr",
"analytics",
"pixel",
)
class MHTMLParser(BaseParser):
"""Parser for MHTML web archives."""
def __init__(self, *args, extract_images: bool = True, **kwargs):
super().__init__(*args, **kwargs)
self.extract_images = extract_images
def parse_into_text(self, content: bytes) -> Document:
logger.info(
"Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content)
)
msg = email.message_from_bytes(content)
html_parts = []
images: Dict[str, str] = {}
image_aliases: Dict[str, str] = {}
metadata: Dict[str, object] = {}
for part in msg.walk():
content_type = part.get_content_type()
location = part.get("Content-Location", "")
if content_type == "text/html":
payload = part.get_payload(decode=True)
if not payload:
continue
charset = part.get_content_charset() or "utf-8"
try:
html_text = payload.decode(charset, errors="ignore")
except LookupError:
html_text = payload.decode("utf-8", errors="ignore")
html_parts.append(
{
"content": html_text,
"location": location,
"size": len(html_text),
}
)
elif content_type.startswith("image/") and self.extract_images:
image_data = part.get_payload(decode=True)
if image_data:
ext = self._image_extension(content_type)
image_path = f"images/{uuid.uuid4().hex}{ext}"
images[image_path] = base64.b64encode(image_data).decode("utf-8")
self._add_image_aliases(image_aliases, part, image_path)
main_html = self._select_main_html(html_parts)
if not main_html:
logger.warning("No HTML content found in MHTML file")
return Document(
content="", images=images, metadata={"source_format": "mhtml"}
)
html_content = main_html["content"]
try:
markdown_text = self._html_to_markdown(
html_content,
image_aliases=image_aliases,
base_location=main_html.get("location", ""),
)
except Exception as e:
logger.error("Failed to convert HTML to Markdown: %s", e)
markdown_text = f"```html\n{html_content}\n```"
metadata["source_format"] = "mhtml"
metadata["file_size"] = len(content)
metadata["image_count"] = len(images)
return Document(content=markdown_text, images=images, metadata=metadata)
def _select_main_html(self, html_parts) -> dict:
"""Pick the largest non-ad HTML part as the main document body."""
if not html_parts:
return {}
def is_ad(location: str) -> bool:
if not location:
return False
loc = location.lower()
return any(ad in loc for ad in _AD_DOMAINS)
non_ad = sorted(
(part for part in html_parts if not is_ad(part.get("location", ""))),
key=lambda part: part["size"],
reverse=True,
)
if non_ad:
logger.info("Selected main HTML: %d bytes", non_ad[0]["size"])
return non_ad[0]
largest = max(html_parts, key=lambda part: part["size"])
logger.warning("Only ad content found, using largest: %d bytes", largest["size"])
return largest
@staticmethod
def _add_image_aliases(image_aliases: Dict[str, str], part, image_path: str) -> None:
"""Register the refs an MHTML document may use for an image part."""
for raw in (
part.get("Content-Location", ""),
part.get("Content-ID", ""),
part.get("X-Attachment-Id", ""),
):
raw = raw.strip()
if not raw:
continue
values = {raw, html.unescape(raw), unquote(html.unescape(raw))}
cid = raw.strip("<>")
if cid:
values.add(f"cid:{cid}")
values.add(f"cid:{unquote(cid)}")
for value in values:
if value:
image_aliases[value] = image_path
@staticmethod
def _image_extension(content_type: str) -> str:
return {
"image/png": ".png",
"image/jpeg": ".jpg",
"image/gif": ".gif",
"image/webp": ".webp",
"image/bmp": ".bmp",
"image/tiff": ".tiff",
"image/x-icon": ".ico",
}.get(content_type, ".png")
def _html_to_markdown(
self,
html_content: str,
image_aliases: Dict[str, str] | None = None,
base_location: str = "",
) -> str:
try:
from markdownify import markdownify as md
soup = BeautifulSoup(html_content, "lxml")
for tag in soup(["script", "style", "noscript", "iframe"]):
tag.decompose()
self._strip_internal_links(soup)
if image_aliases:
self._rewrite_image_sources(soup, image_aliases, base_location)
text_fallback = soup.get_text(separator="\n", strip=True)
markdown_text = md(str(soup), heading_style="ATX")
result = "\n".join(
line.strip() for line in markdown_text.split("\n") if line.strip()
)
if not result and text_fallback:
logger.warning("Markdown empty, falling back to text extraction")
return text_fallback
if not result:
return f"```html\n{html_content[:50000]}\n```"
return result
except ImportError:
logger.warning("markdownify not available, returning raw HTML")
return f"```html\n{html_content}\n```"
except Exception as e:
logger.error("HTML to Markdown conversion failed: %s", e)
return f"```html\n{html_content}\n```"
@staticmethod
def _strip_internal_links(soup: BeautifulSoup) -> None:
"""Unwrap links that don't point to an external resource."""
external = ("http://", "https://", "mailto:", "tel:")
for link in soup.find_all("a"):
href = (link.get("href") or "").strip().lower()
if not href or not href.startswith(external):
link.unwrap()
@staticmethod
def _rewrite_image_sources(
soup: BeautifulSoup,
image_aliases: Dict[str, str],
base_location: str = "",
) -> None:
for img in soup.find_all("img"):
src = (img.get("src") or "").strip()
if not src:
continue
candidates = [
src,
html.unescape(src),
unquote(html.unescape(src)),
]
if base_location:
candidates.append(urljoin(base_location, src))
base_name = os.path.basename(unquote(html.unescape(src)))
if base_name:
candidates.append(base_name)
for candidate in candidates:
if candidate in image_aliases:
img["src"] = image_aliases[candidate]
break
+4
View File
@@ -4,10 +4,12 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type
from docreader.parser.base_parser import BaseParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.epub_parser import EPUBParser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.mhtml_parser import MHTMLParser
from docreader.parser.opendataloader_parser import (
OpenDataLoaderParser,
opendataloader_available,
@@ -131,6 +133,8 @@ def _build_default_registry() -> ParserEngineRegistry:
"markdown": MarkdownParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
"epub": EPUBParser,
"mhtml": MHTMLParser,
**_image_types,
},
description="内置解析引擎",