diff --git a/docreader/parser/epub_parser.py b/docreader/parser/epub_parser.py new file mode 100644 index 00000000..7e651ab7 --- /dev/null +++ b/docreader/parser/epub_parser.py @@ -0,0 +1,332 @@ +"""EPUB parser. + +Parses EPUB files into markdown text and optional embedded images. +""" + +import base64 +import logging +import os +import posixpath +import tempfile +from urllib.parse import unquote +import uuid +from typing import Dict + +from bs4 import BeautifulSoup +import ebooklib +from ebooklib import epub + +from docreader.models.document import Document +from docreader.parser.base_parser import BaseParser + +logger = logging.getLogger(__name__) + + +class EPUBParser(BaseParser): + """Parser for EPUB e-book files.""" + + def __init__(self, *args, extract_images: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.extract_images = extract_images + + def parse_into_text(self, content: bytes) -> Document: + logger.info( + "Parsing EPUB file: %s, size: %d bytes", self.file_name, len(content) + ) + try: + with tempfile.NamedTemporaryFile( + suffix=".epub", delete=False, mode="wb" + ) as epub_file: + epub_file.write(content) + epub_path = epub_file.name + try: + book = epub.read_epub(epub_path) + metadata = self._extract_metadata(book) + markdown_content, images = self._extract_content(book) + + metadata["source_format"] = "epub" + metadata["file_size"] = len(content) + metadata["chapter_count"] = len( + [part for part in markdown_content.split("\n## ") if part.strip()] + ) + metadata["image_count"] = len(images) + return Document( + content=markdown_content, images=images, metadata=metadata + ) + finally: + if os.path.exists(epub_path): + os.unlink(epub_path) + except ImportError: + logger.error("ebooklib not installed") + raise + except Exception as e: + logger.warning( + "ebooklib failed to parse EPUB: %s, trying ZIP fallback", str(e) + ) + return self._parse_epub_fallback(content) + + def _parse_epub_fallback(self, content: bytes) -> Document: + """Parse EPUB directly as a ZIP when ebooklib cannot read it.""" + import re + import zipfile + from io import BytesIO + + metadata = {"source_format": "epub", "file_size": len(content)} + images: Dict[str, str] = {} + image_aliases: Dict[str, str] = {} + markdown_parts = [] + + with zipfile.ZipFile(BytesIO(content), "r") as epub_zip: + html_files = [ + f + for f in epub_zip.namelist() + if f.endswith((".html", ".xhtml", ".htm")) + ] + + def chapter_num(filename: str) -> int: + match = re.search(r"chapter(\d+)", filename, re.IGNORECASE) + return int(match.group(1)) if match else 999999 + + html_files.sort(key=chapter_num) + + if self.extract_images: + img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg") + for img_file in epub_zip.namelist(): + if img_file.lower().endswith(img_exts): + try: + img_data = epub_zip.read(img_file) + ext = os.path.splitext(img_file)[1] + img_path = f"images/{uuid.uuid4().hex}{ext}" + images[img_path] = base64.b64encode(img_data).decode( + "utf-8" + ) + self._add_image_aliases(image_aliases, img_file, img_path) + except Exception as e: + logger.warning( + "Failed to extract image %s: %s", img_file, e + ) + + for html_file in html_files: + try: + html_content = epub_zip.read(html_file).decode( + "utf-8", errors="ignore" + ) + chapter_md = self._html_to_markdown( + html_content, + image_aliases=image_aliases, + base_path=posixpath.dirname(html_file), + ) + base = os.path.basename(html_file) + title = base.replace(".html", "").replace(".xhtml", "") + title = re.sub( + r"chapter[_-]?", "Chapter ", title, flags=re.IGNORECASE + ) + title = title.replace("_", " ").replace("-", " ").title() + if chapter_md.strip(): + markdown_parts.append(f"## {title}\n\n{chapter_md}") + except Exception as e: + logger.warning("Failed to process %s: %s", html_file, e) + + metadata["chapter_count"] = len(markdown_parts) + metadata["image_count"] = len(images) + return Document( + content="\n\n".join(markdown_parts), images=images, metadata=metadata + ) + + def _extract_metadata(self, book) -> Dict[str, str]: + metadata: Dict[str, str] = {} + mapping = { + "title": "title", + "creator": "author", + "publisher": "publisher", + "language": "language", + "description": "description", + "date": "date", + "identifier": "isbn", + } + for dc_key, out_key in mapping.items(): + try: + values = book.get_metadata("DC", dc_key) + except Exception: + values = None + if values: + if out_key == "author": + metadata[out_key] = ", ".join(value[0] for value in values) + else: + metadata[out_key] = values[0][0] + return metadata + + def _extract_content(self, book) -> tuple[str, Dict[str, str]]: + markdown_parts = [] + images: Dict[str, str] = {} + image_aliases: Dict[str, str] = {} + + try: + toc = book.get_table_of_contents() + except Exception as e: + logger.debug("Failed to get TOC: %s, processing all HTML items", e) + toc = [] + + html_items = {} + for item in book.get_items(): + if item.get_type() == ebooklib.ITEM_DOCUMENT: + html_items[item.get_name()] = item + + if self.extract_images: + for item in book.get_items(): + if item.get_type() == ebooklib.ITEM_IMAGE: + img_data = item.get_content() + ext = os.path.splitext(item.get_name())[1] + img_path = f"images/{uuid.uuid4().hex}{ext}" + images[img_path] = base64.b64encode(img_data).decode("utf-8") + self._add_image_aliases(image_aliases, item.get_name(), img_path) + + if toc: + for item in toc: + entries = item if isinstance(item, tuple) else (item,) + for sub in entries: + if hasattr(sub, "get_name") and sub.get_name() in html_items: + markdown_parts.append( + self._process_chapter( + html_items[sub.get_name()], + toc_index=len(markdown_parts), + image_aliases=image_aliases, + ) + ) + + if not markdown_parts: + for _name, item in html_items.items(): + markdown_parts.append( + self._process_chapter( + item, + toc_index=len(markdown_parts), + image_aliases=image_aliases, + ) + ) + + return "\n\n".join(part for part in markdown_parts if part.strip()), images + + def _process_chapter( + self, + html_item, + toc_index: int = 0, + image_aliases: Dict[str, str] | None = None, + ) -> str: + try: + html_content = html_item.get_content() + soup = BeautifulSoup(html_content, "lxml") + title_tag = soup.find(["h1", "h2"]) + if title_tag: + chapter_title = title_tag.get_text().strip() + title_tag.decompose() + else: + chapter_title = html_item.get_name().replace("/", "") + chapter_title = chapter_title.replace(".xhtml", "") + chapter_title = chapter_title.replace("-", " ").title() + body_html = str(soup.body) if soup.body else str(html_content) + chapter_md = self._html_to_markdown( + body_html, + image_aliases=image_aliases, + base_path=posixpath.dirname(html_item.get_name()), + ) + return f"## {chapter_title}\n\n{chapter_md}" + except Exception as e: + logger.error( + "Failed to process chapter %s: %s", html_item.get_name(), e + ) + return f"## Chapter {toc_index + 1}\n\n[Error processing chapter: {e}]" + + def _html_to_markdown( + self, + html_content: str, + image_aliases: Dict[str, str] | None = None, + base_path: str = "", + ) -> str: + try: + from bs4 import Comment + from markdownify import markdownify as md + + soup = BeautifulSoup(html_content, "lxml") + for element in soup(["script", "style"]): + element.decompose() + for comment in soup.find_all( + string=lambda text: isinstance(text, Comment) + ): + comment.extract() + self._strip_internal_links(soup) + if image_aliases: + self._rewrite_image_sources(soup, image_aliases, base_path) + markdown_text = md(str(soup), heading_style="ATX") + return "\n".join( + line.strip() for line in markdown_text.split("\n") if line.strip() + ) + except ImportError: + logger.warning("markdownify not available, using HTML as-is") + return f"```html\n{html_content}\n```" + except Exception as e: + logger.error("HTML to Markdown conversion failed: %s", e) + return f"```html\n{html_content}\n```" + + @staticmethod + def _strip_internal_links(soup: BeautifulSoup) -> None: + """Unwrap links that don't point to an external resource. + + EPUB internal links (other chapter files, ``#fragment`` anchors, TOC + entries) become dead links after extraction. Keep only external links + and replace everything else with its text. + """ + external = ("http://", "https://", "mailto:", "tel:") + for link in soup.find_all("a"): + href = (link.get("href") or "").strip().lower() + if not href or not href.startswith(external): + link.unwrap() + + @staticmethod + def _add_image_aliases( + image_aliases: Dict[str, str], + original_path: str, + image_path: str, + ) -> None: + normalized = EPUBParser._normalize_epub_path(original_path) + aliases = { + original_path, + normalized, + unquote(original_path), + unquote(normalized), + posixpath.basename(normalized), + } + for alias in aliases: + if alias: + image_aliases[alias] = image_path + + @staticmethod + def _rewrite_image_sources( + soup: BeautifulSoup, + image_aliases: Dict[str, str], + base_path: str = "", + ) -> None: + for img in soup.find_all("img"): + src = (img.get("src") or "").strip() + if not src: + continue + normalized_src = EPUBParser._normalize_epub_path(src) + candidates = [ + src, + normalized_src, + unquote(src), + unquote(normalized_src), + posixpath.basename(normalized_src), + ] + if base_path: + joined = EPUBParser._normalize_epub_path(posixpath.join(base_path, src)) + candidates.extend([joined, unquote(joined)]) + for candidate in candidates: + if candidate in image_aliases: + img["src"] = image_aliases[candidate] + break + + @staticmethod + def _normalize_epub_path(path: str) -> str: + path = unquote(path).split("#", 1)[0].split("?", 1)[0].replace("\\", "/") + normalized = posixpath.normpath(path) + return "" if normalized == "." else normalized.lstrip("/") diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.py new file mode 100644 index 00000000..4d1fd1d1 --- /dev/null +++ b/docreader/parser/mhtml_parser.py @@ -0,0 +1,222 @@ +"""MHTML parser. + +Parses MIME HTML web archives into markdown text and optional embedded images. +""" + +import base64 +import email +import html +import logging +import os +from urllib.parse import unquote, urljoin +import uuid +from typing import Dict + +from bs4 import BeautifulSoup + +from docreader.models.document import Document +from docreader.parser.base_parser import BaseParser + +logger = logging.getLogger(__name__) + +_AD_DOMAINS = ( + "googleads", + "doubleclick", + "googlesyndication", + "facebook.com/tr", + "analytics", + "pixel", +) + + +class MHTMLParser(BaseParser): + """Parser for MHTML web archives.""" + + def __init__(self, *args, extract_images: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.extract_images = extract_images + + def parse_into_text(self, content: bytes) -> Document: + logger.info( + "Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content) + ) + msg = email.message_from_bytes(content) + + html_parts = [] + images: Dict[str, str] = {} + image_aliases: Dict[str, str] = {} + metadata: Dict[str, object] = {} + + for part in msg.walk(): + content_type = part.get_content_type() + location = part.get("Content-Location", "") + + if content_type == "text/html": + payload = part.get_payload(decode=True) + if not payload: + continue + charset = part.get_content_charset() or "utf-8" + try: + html_text = payload.decode(charset, errors="ignore") + except LookupError: + html_text = payload.decode("utf-8", errors="ignore") + html_parts.append( + { + "content": html_text, + "location": location, + "size": len(html_text), + } + ) + elif content_type.startswith("image/") and self.extract_images: + image_data = part.get_payload(decode=True) + if image_data: + ext = self._image_extension(content_type) + image_path = f"images/{uuid.uuid4().hex}{ext}" + images[image_path] = base64.b64encode(image_data).decode("utf-8") + self._add_image_aliases(image_aliases, part, image_path) + + main_html = self._select_main_html(html_parts) + if not main_html: + logger.warning("No HTML content found in MHTML file") + return Document( + content="", images=images, metadata={"source_format": "mhtml"} + ) + html_content = main_html["content"] + + try: + markdown_text = self._html_to_markdown( + html_content, + image_aliases=image_aliases, + base_location=main_html.get("location", ""), + ) + except Exception as e: + logger.error("Failed to convert HTML to Markdown: %s", e) + markdown_text = f"```html\n{html_content}\n```" + + metadata["source_format"] = "mhtml" + metadata["file_size"] = len(content) + metadata["image_count"] = len(images) + return Document(content=markdown_text, images=images, metadata=metadata) + + def _select_main_html(self, html_parts) -> dict: + """Pick the largest non-ad HTML part as the main document body.""" + if not html_parts: + return {} + + def is_ad(location: str) -> bool: + if not location: + return False + loc = location.lower() + return any(ad in loc for ad in _AD_DOMAINS) + + non_ad = sorted( + (part for part in html_parts if not is_ad(part.get("location", ""))), + key=lambda part: part["size"], + reverse=True, + ) + if non_ad: + logger.info("Selected main HTML: %d bytes", non_ad[0]["size"]) + return non_ad[0] + + largest = max(html_parts, key=lambda part: part["size"]) + logger.warning("Only ad content found, using largest: %d bytes", largest["size"]) + return largest + + @staticmethod + def _add_image_aliases(image_aliases: Dict[str, str], part, image_path: str) -> None: + """Register the refs an MHTML document may use for an image part.""" + for raw in ( + part.get("Content-Location", ""), + part.get("Content-ID", ""), + part.get("X-Attachment-Id", ""), + ): + raw = raw.strip() + if not raw: + continue + values = {raw, html.unescape(raw), unquote(html.unescape(raw))} + cid = raw.strip("<>") + if cid: + values.add(f"cid:{cid}") + values.add(f"cid:{unquote(cid)}") + for value in values: + if value: + image_aliases[value] = image_path + + @staticmethod + def _image_extension(content_type: str) -> str: + return { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/gif": ".gif", + "image/webp": ".webp", + "image/bmp": ".bmp", + "image/tiff": ".tiff", + "image/x-icon": ".ico", + }.get(content_type, ".png") + + def _html_to_markdown( + self, + html_content: str, + image_aliases: Dict[str, str] | None = None, + base_location: str = "", + ) -> str: + try: + from markdownify import markdownify as md + + soup = BeautifulSoup(html_content, "lxml") + for tag in soup(["script", "style", "noscript", "iframe"]): + tag.decompose() + self._strip_internal_links(soup) + if image_aliases: + self._rewrite_image_sources(soup, image_aliases, base_location) + text_fallback = soup.get_text(separator="\n", strip=True) + markdown_text = md(str(soup), heading_style="ATX") + result = "\n".join( + line.strip() for line in markdown_text.split("\n") if line.strip() + ) + if not result and text_fallback: + logger.warning("Markdown empty, falling back to text extraction") + return text_fallback + if not result: + return f"```html\n{html_content[:50000]}\n```" + return result + except ImportError: + logger.warning("markdownify not available, returning raw HTML") + return f"```html\n{html_content}\n```" + except Exception as e: + logger.error("HTML to Markdown conversion failed: %s", e) + return f"```html\n{html_content}\n```" + + @staticmethod + def _strip_internal_links(soup: BeautifulSoup) -> None: + """Unwrap links that don't point to an external resource.""" + external = ("http://", "https://", "mailto:", "tel:") + for link in soup.find_all("a"): + href = (link.get("href") or "").strip().lower() + if not href or not href.startswith(external): + link.unwrap() + + @staticmethod + def _rewrite_image_sources( + soup: BeautifulSoup, + image_aliases: Dict[str, str], + base_location: str = "", + ) -> None: + for img in soup.find_all("img"): + src = (img.get("src") or "").strip() + if not src: + continue + candidates = [ + src, + html.unescape(src), + unquote(html.unescape(src)), + ] + if base_location: + candidates.append(urljoin(base_location, src)) + base_name = os.path.basename(unquote(html.unescape(src))) + if base_name: + candidates.append(base_name) + for candidate in candidates: + if candidate in image_aliases: + img["src"] = image_aliases[candidate] + break diff --git a/docreader/parser/registry.py b/docreader/parser/registry.py index c3920d72..b8a2a472 100644 --- a/docreader/parser/registry.py +++ b/docreader/parser/registry.py @@ -4,10 +4,12 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type from docreader.parser.base_parser import BaseParser from docreader.parser.doc_parser import DocParser from docreader.parser.docx2_parser import Docx2Parser +from docreader.parser.epub_parser import EPUBParser from docreader.parser.excel_parser import ExcelParser from docreader.parser.image_parser import ImageParser from docreader.parser.markdown_parser import MarkdownParser from docreader.parser.markitdown_parser import MarkitdownParser +from docreader.parser.mhtml_parser import MHTMLParser from docreader.parser.opendataloader_parser import ( OpenDataLoaderParser, opendataloader_available, @@ -131,6 +133,8 @@ def _build_default_registry() -> ParserEngineRegistry: "markdown": MarkdownParser, "xlsx": ExcelParser, "xls": ExcelParser, + "epub": EPUBParser, + "mhtml": MHTMLParser, **_image_types, }, description="内置解析引擎", diff --git a/docreader/pyproject.toml b/docreader/pyproject.toml index a5acc461..1e90685e 100644 --- a/docreader/pyproject.toml +++ b/docreader/pyproject.toml @@ -6,10 +6,12 @@ readme = "README.md" requires-python = ">=3.10.18" dependencies = [ "beautifulsoup4>=4.14.2", + "ebooklib>=0.18", "grpcio>=1.78.0", "grpcio-health-checking>=1.78.0", "grpcio-tools>=1.78.0", "lxml>=6.1.0", + "markdownify>=0.13.1", "markitdown[docx,pdf,xls,xlsx]>=0.1.3", "opendataloader-pdf>=2.4.7", "openpyxl>=3.1.0", diff --git a/docreader/tests/test_epub_parser.py b/docreader/tests/test_epub_parser.py new file mode 100644 index 00000000..1b5a665a --- /dev/null +++ b/docreader/tests/test_epub_parser.py @@ -0,0 +1,92 @@ +import os +import tempfile +import unittest + +from ebooklib import epub + +from docreader.parser.epub_parser import EPUBParser +from docreader.parser.registry import registry + + +def _minimal_epub_bytes() -> bytes: + book = epub.EpubBook() + book.set_identifier("test-epub") + book.set_title("Tiny EPUB") + book.set_language("en") + book.add_author("WeKnora") + + chapter = epub.EpubHtml( + title="Chapter One", file_name="text/chapter_01.xhtml", lang="en" + ) + chapter.content = ( + "

Chapter One

" + "

Hello EPUB world.

" + '

Chapter 2 ' + 'note ' + 'the site

' + 'cover' + "" + ) + book.add_item(chapter) + book.add_item( + epub.EpubItem( + uid="pic", + file_name="images/pic.png", + media_type="image/png", + content=b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR", + ) + ) + book.toc = (epub.Link("text/chapter_01.xhtml", "Chapter One", "chapter-one"),) + book.spine = ["nav", chapter] + book.add_item(epub.EpubNcx()) + book.add_item(epub.EpubNav()) + + with tempfile.NamedTemporaryFile(suffix=".epub", delete=False) as handle: + path = handle.name + try: + epub.write_epub(path, book) + with open(path, "rb") as handle: + return handle.read() + finally: + if os.path.exists(path): + os.unlink(path) + + +class EPUBParserTest(unittest.TestCase): + def test_parse_minimal_epub(self): + document = EPUBParser( + file_name="tiny.epub", file_type="epub" + ).parse_into_text(_minimal_epub_bytes()) + + self.assertIn("Hello EPUB world", document.content) + self.assertEqual(document.metadata["source_format"], "epub") + self.assertEqual(len(document.images), 1) + image_ref = next(iter(document.images)) + self.assertTrue(image_ref.startswith("images/")) + self.assertIn(image_ref, document.content) + self.assertNotIn("../images/pic.png", document.content) + + def test_internal_links_are_unwrapped_but_external_links_remain(self): + document = EPUBParser( + file_name="tiny.epub", file_type="epub" + ).parse_into_text(_minimal_epub_bytes()) + + self.assertIn("Chapter 2", document.content) + self.assertIn("note", document.content) + self.assertNotIn("chapter_02.xhtml#sec2", document.content) + self.assertNotIn("#footnote1", document.content) + self.assertIn("[the site](https://example.com)", document.content) + + def test_parse_without_images(self): + document = EPUBParser( + file_name="tiny.epub", file_type="epub", extract_images=False + ).parse_into_text(_minimal_epub_bytes()) + + self.assertEqual(document.images, {}) + + def test_registry_resolves_epub(self): + self.assertIs(registry.get_parser_class("", "epub"), EPUBParser) + + +if __name__ == "__main__": + unittest.main() diff --git a/docreader/tests/test_mhtml_parser.py b/docreader/tests/test_mhtml_parser.py new file mode 100644 index 00000000..531ba44d --- /dev/null +++ b/docreader/tests/test_mhtml_parser.py @@ -0,0 +1,92 @@ +import unittest +from email.message import EmailMessage + +from docreader.parser.mhtml_parser import MHTMLParser +from docreader.parser.registry import registry + + +def _minimal_mhtml_bytes() -> bytes: + root = EmailMessage() + root["Subject"] = "Tiny MHTML" + root.make_related() + + main = EmailMessage() + main.set_content( + "

Main Article

" + "

Hello MHTML world.

" + '

Chapter 3 ' + 'note ' + 'the site

' + 'tiny' + "" + "", + subtype="html", + ) + main["Content-Location"] = "https://example.com/article" + root.attach(main) + + ad = EmailMessage() + ad.set_content( + "

Advertisement

" + "

Buy this unrelated thing.

", + subtype="html", + ) + ad["Content-Location"] = "https://googleads.example/frame.html" + root.attach(ad) + + image = EmailMessage() + image.set_content( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR", + maintype="image", + subtype="png", + ) + image["Content-Location"] = "cid:tiny-image" + root.attach(image) + + return root.as_bytes() + + +class MHTMLParserTest(unittest.TestCase): + def test_parse_selects_main_html_and_filters_noise(self): + document = MHTMLParser( + file_name="article.mhtml", file_type="mhtml" + ).parse_into_text(_minimal_mhtml_bytes()) + + self.assertIn("Main Article", document.content) + self.assertIn("Hello MHTML world", document.content) + self.assertNotIn("Advertisement", document.content) + self.assertNotIn("window.noise", document.content) + self.assertEqual(document.metadata["source_format"], "mhtml") + + def test_internal_links_are_unwrapped_but_external_links_remain(self): + document = MHTMLParser( + file_name="article.mhtml", file_type="mhtml" + ).parse_into_text(_minimal_mhtml_bytes()) + + self.assertIn("Chapter 3", document.content) + self.assertIn("note", document.content) + self.assertNotIn("chapter03.xhtml#sec2", document.content) + self.assertNotIn("#footnote1", document.content) + self.assertIn("[the site](https://example.com)", document.content) + + def test_image_extraction_toggle(self): + with_images = MHTMLParser( + file_name="article.mhtml", file_type="mhtml", extract_images=True + ).parse_into_text(_minimal_mhtml_bytes()) + without_images = MHTMLParser( + file_name="article.mhtml", file_type="mhtml", extract_images=False + ).parse_into_text(_minimal_mhtml_bytes()) + + self.assertEqual(len(with_images.images), 1) + image_ref = next(iter(with_images.images)) + self.assertTrue(image_ref.startswith("images/")) + self.assertIn(image_ref, with_images.content) + self.assertNotIn("cid:tiny-image", with_images.content) + self.assertEqual(without_images.images, {}) + + def test_registry_resolves_mhtml(self): + self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser) + + +if __name__ == "__main__": + unittest.main() diff --git a/docreader/uv.lock b/docreader/uv.lock index 861ef480..b010e0cd 100644 --- a/docreader/uv.lock +++ b/docreader/uv.lock @@ -460,10 +460,12 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "ebooklib" }, { name = "grpcio" }, { name = "grpcio-health-checking" }, { name = "grpcio-tools" }, { name = "lxml" }, + { name = "markdownify" }, { name = "markitdown", extra = ["docx", "pdf", "xls", "xlsx"] }, { name = "opendataloader-pdf" }, { name = "openpyxl" }, @@ -484,10 +486,12 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.14.2" }, + { name = "ebooklib", specifier = ">=0.18" }, { name = "grpcio", specifier = ">=1.78.0" }, { name = "grpcio-health-checking", specifier = ">=1.78.0" }, { name = "grpcio-tools", specifier = ">=1.78.0" }, { name = "lxml", specifier = ">=6.1.0" }, + { name = "markdownify", specifier = ">=0.13.1" }, { name = "markitdown", extras = ["docx", "pdf", "xls", "xlsx"], specifier = ">=0.1.3" }, { name = "opendataloader-pdf", specifier = ">=2.4.7" }, { name = "openpyxl", specifier = ">=3.1.0" }, diff --git a/frontend/src/components/AttachmentUpload.vue b/frontend/src/components/AttachmentUpload.vue index a5895d61..8f63f70d 100644 --- a/frontend/src/components/AttachmentUpload.vue +++ b/frontend/src/components/AttachmentUpload.vue @@ -31,7 +31,7 @@ const fileInputRef = ref(); // Supported file types (matching backend) const SUPPORTED_TYPES = [ // Documents - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.epub', '.mhtml', // Text '.txt', '.md', '.csv', '.json', '.xml', '.html', // Audio @@ -116,6 +116,7 @@ const getFileIcon = (fileName: string): string => { if (['doc', 'docx'].includes(ext || '')) return 'file-word'; if (['xls', 'xlsx'].includes(ext || '')) return 'file-excel'; if (['ppt', 'pptx'].includes(ext || '')) return 'file-powerpoint'; + if (['epub', 'mhtml'].includes(ext || '')) return 'file'; if (['txt', 'md'].includes(ext || '')) return 'file'; if (['mp3', 'wav', 'm4a', 'flac', 'ogg', 'aac'].includes(ext || '')) return 'sound'; return 'file'; diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 4bf8c821..7af5e153 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -4786,6 +4786,8 @@ export default { fileTypeText: 'Plain Text', fileTypeJson: 'JSON Files', fileTypeImage: 'Images', + fileTypeEbook: 'E-books', + fileTypeWebArchive: 'Web Archives', fileTypeAudiovisual: 'Audio', engines: { builtin: { diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index df79936b..1d708856 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -4817,6 +4817,8 @@ export default { fileTypeText: '일반 텍스트', fileTypeJson: 'JSON 파일', fileTypeImage: '이미지', + fileTypeEbook: '전자책', + fileTypeWebArchive: '웹 아카이브', fileTypeAudiovisual: '오디오', engines: { builtin: { diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index ea83c12a..61d6247f 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -4317,6 +4317,8 @@ export default { fileTypeText: 'Текстовые файлы', fileTypeJson: 'Файлы JSON', fileTypeImage: 'Изображения', + fileTypeEbook: 'Электронные книги', + fileTypeWebArchive: 'Веб-архивы', fileTypeAudiovisual: 'Аудио', engines: { builtin: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index c65f7c1f..65fd93b2 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -4780,6 +4780,8 @@ export default { fileTypeText: "纯文本", fileTypeJson: "JSON 文件", fileTypeImage: "图片", + fileTypeEbook: "电子书", + fileTypeWebArchive: "网页存档", fileTypeAudiovisual: "音频", engines: { builtin: { diff --git a/frontend/src/utils/index.ts b/frontend/src/utils/index.ts index baf0db9e..0afdaadb 100644 --- a/frontend/src/utils/index.ts +++ b/frontend/src/utils/index.ts @@ -40,7 +40,7 @@ export function formatStringDate(date: any) { year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second ); } -const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "jpg", "jpeg", "png", "csv", "xlsx", "xls", "mp3", "wav", "m4a", "flac", "ogg"]); +const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "epub", "mhtml", "jpg", "jpeg", "png", "csv", "xlsx", "xls", "mp3", "wav", "m4a", "flac", "ogg"]); /** * Returns true when the file should be **rejected**. diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index f3580c96..70f7048b 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -436,6 +436,8 @@ const fileTypeOptions = computed(() => [ { label: 'DOC', value: 'doc' }, { label: 'PPTX', value: 'pptx' }, { label: 'PPT', value: 'ppt' }, + { label: 'EPUB', value: 'epub' }, + { label: 'MHTML', value: 'mhtml' }, { label: 'TXT', value: 'txt' }, { label: 'MD', value: 'md' }, { label: 'URL', value: 'url' }, diff --git a/frontend/src/views/knowledge/settings/KBParserSettings.vue b/frontend/src/views/knowledge/settings/KBParserSettings.vue index b4a9fb3c..b338acc7 100644 --- a/frontend/src/views/knowledge/settings/KBParserSettings.vue +++ b/frontend/src/views/knowledge/settings/KBParserSettings.vue @@ -162,6 +162,8 @@ const fileTypeGroups = computed(() => { const officeExts = ['docx', 'doc'].filter(e => ft.has(e)) const pptExts = ['pptx', 'ppt'].filter(e => ft.has(e)) const excelExts = ['xlsx', 'xls'].filter(e => ft.has(e)) + const ebookExts = ['epub'].filter(e => ft.has(e)) + const webArchiveExts = ['mhtml'].filter(e => ft.has(e)) const csvExts = ['csv'].filter(e => ft.has(e)) const mdExts = ['md', 'markdown'].filter(e => ft.has(e)) const txtExts = ['txt'].filter(e => ft.has(e)) @@ -174,6 +176,8 @@ const fileTypeGroups = computed(() => { if (officeExts.length) groups.push({ key: 'office', label: t('kbSettings.parser.fileTypeWord'), icon: 'file-word', extensions: officeExts }) if (pptExts.length) groups.push({ key: 'ppt', label: t('kbSettings.parser.fileTypePpt'), icon: 'file-powerpoint', extensions: pptExts }) if (excelExts.length) groups.push({ key: 'excel', label: t('kbSettings.parser.fileTypeExcel'), icon: 'file-excel', extensions: excelExts }) + if (ebookExts.length) groups.push({ key: 'ebook', label: t('kbSettings.parser.fileTypeEbook'), icon: 'file', extensions: ebookExts }) + if (webArchiveExts.length) groups.push({ key: 'webarchive', label: t('kbSettings.parser.fileTypeWebArchive'), icon: 'file', extensions: webArchiveExts }) if (csvExts.length) groups.push({ key: 'csv', label: t('kbSettings.parser.fileTypeCsv'), icon: 'file-excel', extensions: csvExts }) if (mdExts.length) groups.push({ key: 'markdown', label: 'Markdown', icon: 'file-code', extensions: mdExts }) if (txtExts.length) groups.push({ key: 'text', label: t('kbSettings.parser.fileTypeText'), icon: 'file', extensions: txtExts }) diff --git a/frontend/src/views/platform/index.vue b/frontend/src/views/platform/index.vue index 982ee854..40929e53 100644 --- a/frontend/src/views/platform/index.vue +++ b/frontend/src/views/platform/index.vue @@ -5,7 +5,7 @@
- +
@@ -279,4 +279,4 @@ img { -o-user-drag: none; user-drag: none; } - \ No newline at end of file + diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index b6d91938..16f96184 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -112,17 +112,6 @@ export default defineConfig({ if (id.includes('highlight.js')) { return 'vendor-highlight' } - if (id.includes('tdesign-vue-next')) { - return 'vendor-tdesign' - } - if ( - id.includes('/vue/') || - id.includes('/vue-router/') || - id.includes('/pinia/') || - id.includes('/vue-i18n/') - ) { - return 'vendor-vue' - } }, }, }, diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index 10bec036..7add9178 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -21,7 +21,7 @@ import ( // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { - case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", + case "pdf", "txt", "docx", "doc", "epub", "mhtml", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", "mp3", "wav", "m4a", "flac", "ogg": return true default: diff --git a/internal/handler/session/attachment_processor.go b/internal/handler/session/attachment_processor.go index fa1c4e74..12be0cea 100644 --- a/internal/handler/session/attachment_processor.go +++ b/internal/handler/session/attachment_processor.go @@ -288,7 +288,7 @@ func isValidFileType(fileName string) bool { supportedTypes := []string{ // documents - "docx", "doc", "pdf", "ppt", "pptx", + "docx", "doc", "pdf", "ppt", "pptx", "epub", "mhtml", // spreadsheets "xlsx", "xls", // text / markup diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index c8d65a80..8fed0e3a 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -45,7 +45,7 @@ func (e *builtinEngine) Description() string { return "DocReader built-in parser engine" } func (e *builtinEngine) FileTypes(_ bool) []string { - return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} + return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "epub", "mhtml", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} } func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) { if docreaderConnected {