feat: add EPUB and MHTML document support
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
"""EPUB parser.
|
||||
|
||||
Parses EPUB files into markdown text and optional embedded images.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import tempfile
|
||||
from urllib.parse import unquote
|
||||
import uuid
|
||||
from typing import Dict
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EPUBParser(BaseParser):
|
||||
"""Parser for EPUB e-book files."""
|
||||
|
||||
def __init__(self, *args, extract_images: bool = True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.extract_images = extract_images
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
logger.info(
|
||||
"Parsing EPUB file: %s, size: %d bytes", self.file_name, len(content)
|
||||
)
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".epub", delete=False, mode="wb"
|
||||
) as epub_file:
|
||||
epub_file.write(content)
|
||||
epub_path = epub_file.name
|
||||
try:
|
||||
book = epub.read_epub(epub_path)
|
||||
metadata = self._extract_metadata(book)
|
||||
markdown_content, images = self._extract_content(book)
|
||||
|
||||
metadata["source_format"] = "epub"
|
||||
metadata["file_size"] = len(content)
|
||||
metadata["chapter_count"] = len(
|
||||
[part for part in markdown_content.split("\n## ") if part.strip()]
|
||||
)
|
||||
metadata["image_count"] = len(images)
|
||||
return Document(
|
||||
content=markdown_content, images=images, metadata=metadata
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(epub_path):
|
||||
os.unlink(epub_path)
|
||||
except ImportError:
|
||||
logger.error("ebooklib not installed")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"ebooklib failed to parse EPUB: %s, trying ZIP fallback", str(e)
|
||||
)
|
||||
return self._parse_epub_fallback(content)
|
||||
|
||||
def _parse_epub_fallback(self, content: bytes) -> Document:
|
||||
"""Parse EPUB directly as a ZIP when ebooklib cannot read it."""
|
||||
import re
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
metadata = {"source_format": "epub", "file_size": len(content)}
|
||||
images: Dict[str, str] = {}
|
||||
image_aliases: Dict[str, str] = {}
|
||||
markdown_parts = []
|
||||
|
||||
with zipfile.ZipFile(BytesIO(content), "r") as epub_zip:
|
||||
html_files = [
|
||||
f
|
||||
for f in epub_zip.namelist()
|
||||
if f.endswith((".html", ".xhtml", ".htm"))
|
||||
]
|
||||
|
||||
def chapter_num(filename: str) -> int:
|
||||
match = re.search(r"chapter(\d+)", filename, re.IGNORECASE)
|
||||
return int(match.group(1)) if match else 999999
|
||||
|
||||
html_files.sort(key=chapter_num)
|
||||
|
||||
if self.extract_images:
|
||||
img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg")
|
||||
for img_file in epub_zip.namelist():
|
||||
if img_file.lower().endswith(img_exts):
|
||||
try:
|
||||
img_data = epub_zip.read(img_file)
|
||||
ext = os.path.splitext(img_file)[1]
|
||||
img_path = f"images/{uuid.uuid4().hex}{ext}"
|
||||
images[img_path] = base64.b64encode(img_data).decode(
|
||||
"utf-8"
|
||||
)
|
||||
self._add_image_aliases(image_aliases, img_file, img_path)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to extract image %s: %s", img_file, e
|
||||
)
|
||||
|
||||
for html_file in html_files:
|
||||
try:
|
||||
html_content = epub_zip.read(html_file).decode(
|
||||
"utf-8", errors="ignore"
|
||||
)
|
||||
chapter_md = self._html_to_markdown(
|
||||
html_content,
|
||||
image_aliases=image_aliases,
|
||||
base_path=posixpath.dirname(html_file),
|
||||
)
|
||||
base = os.path.basename(html_file)
|
||||
title = base.replace(".html", "").replace(".xhtml", "")
|
||||
title = re.sub(
|
||||
r"chapter[_-]?", "Chapter ", title, flags=re.IGNORECASE
|
||||
)
|
||||
title = title.replace("_", " ").replace("-", " ").title()
|
||||
if chapter_md.strip():
|
||||
markdown_parts.append(f"## {title}\n\n{chapter_md}")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to process %s: %s", html_file, e)
|
||||
|
||||
metadata["chapter_count"] = len(markdown_parts)
|
||||
metadata["image_count"] = len(images)
|
||||
return Document(
|
||||
content="\n\n".join(markdown_parts), images=images, metadata=metadata
|
||||
)
|
||||
|
||||
def _extract_metadata(self, book) -> Dict[str, str]:
|
||||
metadata: Dict[str, str] = {}
|
||||
mapping = {
|
||||
"title": "title",
|
||||
"creator": "author",
|
||||
"publisher": "publisher",
|
||||
"language": "language",
|
||||
"description": "description",
|
||||
"date": "date",
|
||||
"identifier": "isbn",
|
||||
}
|
||||
for dc_key, out_key in mapping.items():
|
||||
try:
|
||||
values = book.get_metadata("DC", dc_key)
|
||||
except Exception:
|
||||
values = None
|
||||
if values:
|
||||
if out_key == "author":
|
||||
metadata[out_key] = ", ".join(value[0] for value in values)
|
||||
else:
|
||||
metadata[out_key] = values[0][0]
|
||||
return metadata
|
||||
|
||||
def _extract_content(self, book) -> tuple[str, Dict[str, str]]:
|
||||
markdown_parts = []
|
||||
images: Dict[str, str] = {}
|
||||
image_aliases: Dict[str, str] = {}
|
||||
|
||||
try:
|
||||
toc = book.get_table_of_contents()
|
||||
except Exception as e:
|
||||
logger.debug("Failed to get TOC: %s, processing all HTML items", e)
|
||||
toc = []
|
||||
|
||||
html_items = {}
|
||||
for item in book.get_items():
|
||||
if item.get_type() == ebooklib.ITEM_DOCUMENT:
|
||||
html_items[item.get_name()] = item
|
||||
|
||||
if self.extract_images:
|
||||
for item in book.get_items():
|
||||
if item.get_type() == ebooklib.ITEM_IMAGE:
|
||||
img_data = item.get_content()
|
||||
ext = os.path.splitext(item.get_name())[1]
|
||||
img_path = f"images/{uuid.uuid4().hex}{ext}"
|
||||
images[img_path] = base64.b64encode(img_data).decode("utf-8")
|
||||
self._add_image_aliases(image_aliases, item.get_name(), img_path)
|
||||
|
||||
if toc:
|
||||
for item in toc:
|
||||
entries = item if isinstance(item, tuple) else (item,)
|
||||
for sub in entries:
|
||||
if hasattr(sub, "get_name") and sub.get_name() in html_items:
|
||||
markdown_parts.append(
|
||||
self._process_chapter(
|
||||
html_items[sub.get_name()],
|
||||
toc_index=len(markdown_parts),
|
||||
image_aliases=image_aliases,
|
||||
)
|
||||
)
|
||||
|
||||
if not markdown_parts:
|
||||
for _name, item in html_items.items():
|
||||
markdown_parts.append(
|
||||
self._process_chapter(
|
||||
item,
|
||||
toc_index=len(markdown_parts),
|
||||
image_aliases=image_aliases,
|
||||
)
|
||||
)
|
||||
|
||||
return "\n\n".join(part for part in markdown_parts if part.strip()), images
|
||||
|
||||
def _process_chapter(
|
||||
self,
|
||||
html_item,
|
||||
toc_index: int = 0,
|
||||
image_aliases: Dict[str, str] | None = None,
|
||||
) -> str:
|
||||
try:
|
||||
html_content = html_item.get_content()
|
||||
soup = BeautifulSoup(html_content, "lxml")
|
||||
title_tag = soup.find(["h1", "h2"])
|
||||
if title_tag:
|
||||
chapter_title = title_tag.get_text().strip()
|
||||
title_tag.decompose()
|
||||
else:
|
||||
chapter_title = html_item.get_name().replace("/", "")
|
||||
chapter_title = chapter_title.replace(".xhtml", "")
|
||||
chapter_title = chapter_title.replace("-", " ").title()
|
||||
body_html = str(soup.body) if soup.body else str(html_content)
|
||||
chapter_md = self._html_to_markdown(
|
||||
body_html,
|
||||
image_aliases=image_aliases,
|
||||
base_path=posixpath.dirname(html_item.get_name()),
|
||||
)
|
||||
return f"## {chapter_title}\n\n{chapter_md}"
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to process chapter %s: %s", html_item.get_name(), e
|
||||
)
|
||||
return f"## Chapter {toc_index + 1}\n\n[Error processing chapter: {e}]"
|
||||
|
||||
def _html_to_markdown(
|
||||
self,
|
||||
html_content: str,
|
||||
image_aliases: Dict[str, str] | None = None,
|
||||
base_path: str = "",
|
||||
) -> str:
|
||||
try:
|
||||
from bs4 import Comment
|
||||
from markdownify import markdownify as md
|
||||
|
||||
soup = BeautifulSoup(html_content, "lxml")
|
||||
for element in soup(["script", "style"]):
|
||||
element.decompose()
|
||||
for comment in soup.find_all(
|
||||
string=lambda text: isinstance(text, Comment)
|
||||
):
|
||||
comment.extract()
|
||||
self._strip_internal_links(soup)
|
||||
if image_aliases:
|
||||
self._rewrite_image_sources(soup, image_aliases, base_path)
|
||||
markdown_text = md(str(soup), heading_style="ATX")
|
||||
return "\n".join(
|
||||
line.strip() for line in markdown_text.split("\n") if line.strip()
|
||||
)
|
||||
except ImportError:
|
||||
logger.warning("markdownify not available, using HTML as-is")
|
||||
return f"```html\n{html_content}\n```"
|
||||
except Exception as e:
|
||||
logger.error("HTML to Markdown conversion failed: %s", e)
|
||||
return f"```html\n{html_content}\n```"
|
||||
|
||||
@staticmethod
|
||||
def _strip_internal_links(soup: BeautifulSoup) -> None:
|
||||
"""Unwrap links that don't point to an external resource.
|
||||
|
||||
EPUB internal links (other chapter files, ``#fragment`` anchors, TOC
|
||||
entries) become dead links after extraction. Keep only external links
|
||||
and replace everything else with its text.
|
||||
"""
|
||||
external = ("http://", "https://", "mailto:", "tel:")
|
||||
for link in soup.find_all("a"):
|
||||
href = (link.get("href") or "").strip().lower()
|
||||
if not href or not href.startswith(external):
|
||||
link.unwrap()
|
||||
|
||||
@staticmethod
|
||||
def _add_image_aliases(
|
||||
image_aliases: Dict[str, str],
|
||||
original_path: str,
|
||||
image_path: str,
|
||||
) -> None:
|
||||
normalized = EPUBParser._normalize_epub_path(original_path)
|
||||
aliases = {
|
||||
original_path,
|
||||
normalized,
|
||||
unquote(original_path),
|
||||
unquote(normalized),
|
||||
posixpath.basename(normalized),
|
||||
}
|
||||
for alias in aliases:
|
||||
if alias:
|
||||
image_aliases[alias] = image_path
|
||||
|
||||
@staticmethod
|
||||
def _rewrite_image_sources(
|
||||
soup: BeautifulSoup,
|
||||
image_aliases: Dict[str, str],
|
||||
base_path: str = "",
|
||||
) -> None:
|
||||
for img in soup.find_all("img"):
|
||||
src = (img.get("src") or "").strip()
|
||||
if not src:
|
||||
continue
|
||||
normalized_src = EPUBParser._normalize_epub_path(src)
|
||||
candidates = [
|
||||
src,
|
||||
normalized_src,
|
||||
unquote(src),
|
||||
unquote(normalized_src),
|
||||
posixpath.basename(normalized_src),
|
||||
]
|
||||
if base_path:
|
||||
joined = EPUBParser._normalize_epub_path(posixpath.join(base_path, src))
|
||||
candidates.extend([joined, unquote(joined)])
|
||||
for candidate in candidates:
|
||||
if candidate in image_aliases:
|
||||
img["src"] = image_aliases[candidate]
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def _normalize_epub_path(path: str) -> str:
|
||||
path = unquote(path).split("#", 1)[0].split("?", 1)[0].replace("\\", "/")
|
||||
normalized = posixpath.normpath(path)
|
||||
return "" if normalized == "." else normalized.lstrip("/")
|
||||
@@ -0,0 +1,222 @@
|
||||
"""MHTML parser.
|
||||
|
||||
Parses MIME HTML web archives into markdown text and optional embedded images.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import email
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
from urllib.parse import unquote, urljoin
|
||||
import uuid
|
||||
from typing import Dict
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_AD_DOMAINS = (
|
||||
"googleads",
|
||||
"doubleclick",
|
||||
"googlesyndication",
|
||||
"facebook.com/tr",
|
||||
"analytics",
|
||||
"pixel",
|
||||
)
|
||||
|
||||
|
||||
class MHTMLParser(BaseParser):
|
||||
"""Parser for MHTML web archives."""
|
||||
|
||||
def __init__(self, *args, extract_images: bool = True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.extract_images = extract_images
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
logger.info(
|
||||
"Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content)
|
||||
)
|
||||
msg = email.message_from_bytes(content)
|
||||
|
||||
html_parts = []
|
||||
images: Dict[str, str] = {}
|
||||
image_aliases: Dict[str, str] = {}
|
||||
metadata: Dict[str, object] = {}
|
||||
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
location = part.get("Content-Location", "")
|
||||
|
||||
if content_type == "text/html":
|
||||
payload = part.get_payload(decode=True)
|
||||
if not payload:
|
||||
continue
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
html_text = payload.decode(charset, errors="ignore")
|
||||
except LookupError:
|
||||
html_text = payload.decode("utf-8", errors="ignore")
|
||||
html_parts.append(
|
||||
{
|
||||
"content": html_text,
|
||||
"location": location,
|
||||
"size": len(html_text),
|
||||
}
|
||||
)
|
||||
elif content_type.startswith("image/") and self.extract_images:
|
||||
image_data = part.get_payload(decode=True)
|
||||
if image_data:
|
||||
ext = self._image_extension(content_type)
|
||||
image_path = f"images/{uuid.uuid4().hex}{ext}"
|
||||
images[image_path] = base64.b64encode(image_data).decode("utf-8")
|
||||
self._add_image_aliases(image_aliases, part, image_path)
|
||||
|
||||
main_html = self._select_main_html(html_parts)
|
||||
if not main_html:
|
||||
logger.warning("No HTML content found in MHTML file")
|
||||
return Document(
|
||||
content="", images=images, metadata={"source_format": "mhtml"}
|
||||
)
|
||||
html_content = main_html["content"]
|
||||
|
||||
try:
|
||||
markdown_text = self._html_to_markdown(
|
||||
html_content,
|
||||
image_aliases=image_aliases,
|
||||
base_location=main_html.get("location", ""),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to convert HTML to Markdown: %s", e)
|
||||
markdown_text = f"```html\n{html_content}\n```"
|
||||
|
||||
metadata["source_format"] = "mhtml"
|
||||
metadata["file_size"] = len(content)
|
||||
metadata["image_count"] = len(images)
|
||||
return Document(content=markdown_text, images=images, metadata=metadata)
|
||||
|
||||
def _select_main_html(self, html_parts) -> dict:
|
||||
"""Pick the largest non-ad HTML part as the main document body."""
|
||||
if not html_parts:
|
||||
return {}
|
||||
|
||||
def is_ad(location: str) -> bool:
|
||||
if not location:
|
||||
return False
|
||||
loc = location.lower()
|
||||
return any(ad in loc for ad in _AD_DOMAINS)
|
||||
|
||||
non_ad = sorted(
|
||||
(part for part in html_parts if not is_ad(part.get("location", ""))),
|
||||
key=lambda part: part["size"],
|
||||
reverse=True,
|
||||
)
|
||||
if non_ad:
|
||||
logger.info("Selected main HTML: %d bytes", non_ad[0]["size"])
|
||||
return non_ad[0]
|
||||
|
||||
largest = max(html_parts, key=lambda part: part["size"])
|
||||
logger.warning("Only ad content found, using largest: %d bytes", largest["size"])
|
||||
return largest
|
||||
|
||||
@staticmethod
|
||||
def _add_image_aliases(image_aliases: Dict[str, str], part, image_path: str) -> None:
|
||||
"""Register the refs an MHTML document may use for an image part."""
|
||||
for raw in (
|
||||
part.get("Content-Location", ""),
|
||||
part.get("Content-ID", ""),
|
||||
part.get("X-Attachment-Id", ""),
|
||||
):
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
continue
|
||||
values = {raw, html.unescape(raw), unquote(html.unescape(raw))}
|
||||
cid = raw.strip("<>")
|
||||
if cid:
|
||||
values.add(f"cid:{cid}")
|
||||
values.add(f"cid:{unquote(cid)}")
|
||||
for value in values:
|
||||
if value:
|
||||
image_aliases[value] = image_path
|
||||
|
||||
@staticmethod
|
||||
def _image_extension(content_type: str) -> str:
|
||||
return {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
"image/bmp": ".bmp",
|
||||
"image/tiff": ".tiff",
|
||||
"image/x-icon": ".ico",
|
||||
}.get(content_type, ".png")
|
||||
|
||||
def _html_to_markdown(
|
||||
self,
|
||||
html_content: str,
|
||||
image_aliases: Dict[str, str] | None = None,
|
||||
base_location: str = "",
|
||||
) -> str:
|
||||
try:
|
||||
from markdownify import markdownify as md
|
||||
|
||||
soup = BeautifulSoup(html_content, "lxml")
|
||||
for tag in soup(["script", "style", "noscript", "iframe"]):
|
||||
tag.decompose()
|
||||
self._strip_internal_links(soup)
|
||||
if image_aliases:
|
||||
self._rewrite_image_sources(soup, image_aliases, base_location)
|
||||
text_fallback = soup.get_text(separator="\n", strip=True)
|
||||
markdown_text = md(str(soup), heading_style="ATX")
|
||||
result = "\n".join(
|
||||
line.strip() for line in markdown_text.split("\n") if line.strip()
|
||||
)
|
||||
if not result and text_fallback:
|
||||
logger.warning("Markdown empty, falling back to text extraction")
|
||||
return text_fallback
|
||||
if not result:
|
||||
return f"```html\n{html_content[:50000]}\n```"
|
||||
return result
|
||||
except ImportError:
|
||||
logger.warning("markdownify not available, returning raw HTML")
|
||||
return f"```html\n{html_content}\n```"
|
||||
except Exception as e:
|
||||
logger.error("HTML to Markdown conversion failed: %s", e)
|
||||
return f"```html\n{html_content}\n```"
|
||||
|
||||
@staticmethod
|
||||
def _strip_internal_links(soup: BeautifulSoup) -> None:
|
||||
"""Unwrap links that don't point to an external resource."""
|
||||
external = ("http://", "https://", "mailto:", "tel:")
|
||||
for link in soup.find_all("a"):
|
||||
href = (link.get("href") or "").strip().lower()
|
||||
if not href or not href.startswith(external):
|
||||
link.unwrap()
|
||||
|
||||
@staticmethod
|
||||
def _rewrite_image_sources(
|
||||
soup: BeautifulSoup,
|
||||
image_aliases: Dict[str, str],
|
||||
base_location: str = "",
|
||||
) -> None:
|
||||
for img in soup.find_all("img"):
|
||||
src = (img.get("src") or "").strip()
|
||||
if not src:
|
||||
continue
|
||||
candidates = [
|
||||
src,
|
||||
html.unescape(src),
|
||||
unquote(html.unescape(src)),
|
||||
]
|
||||
if base_location:
|
||||
candidates.append(urljoin(base_location, src))
|
||||
base_name = os.path.basename(unquote(html.unescape(src)))
|
||||
if base_name:
|
||||
candidates.append(base_name)
|
||||
for candidate in candidates:
|
||||
if candidate in image_aliases:
|
||||
img["src"] = image_aliases[candidate]
|
||||
break
|
||||
@@ -4,10 +4,12 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.epub_parser import EPUBParser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.mhtml_parser import MHTMLParser
|
||||
from docreader.parser.opendataloader_parser import (
|
||||
OpenDataLoaderParser,
|
||||
opendataloader_available,
|
||||
@@ -131,6 +133,8 @@ def _build_default_registry() -> ParserEngineRegistry:
|
||||
"markdown": MarkdownParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
"epub": EPUBParser,
|
||||
"mhtml": MHTMLParser,
|
||||
**_image_types,
|
||||
},
|
||||
description="内置解析引擎",
|
||||
|
||||
Reference in New Issue
Block a user