feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing. Optional odl-hybrid sidecar stays local-build only and is excluded from default dev-start and full profiles.
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
"""LibreOffice helpers for normalizing legacy or unusual Excel uploads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_XLS_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
|
||||
_ZIP_MAGIC = b"PK\x03\x04"
|
||||
|
||||
|
||||
def detect_excel_format(content: bytes) -> str | None:
|
||||
"""Return pandas/excel format id: xlsx, xls, xlsb, ods, or None."""
|
||||
if not content:
|
||||
return None
|
||||
|
||||
from pandas.io.excel._base import inspect_excel_format
|
||||
|
||||
ext = inspect_excel_format(content_or_path=content)
|
||||
if ext in ("xlsx", "xls", "xlsb", "ods"):
|
||||
return ext
|
||||
if ext == "zip":
|
||||
return "xlsx"
|
||||
|
||||
if content.startswith(_ZIP_MAGIC):
|
||||
return "xlsx"
|
||||
if len(content) >= len(_XLS_MAGIC) and content.startswith(_XLS_MAGIC):
|
||||
return "xls"
|
||||
return None
|
||||
|
||||
|
||||
def engine_for_format(ext: str | None) -> str:
|
||||
if ext == "xls":
|
||||
return "xlrd"
|
||||
if ext in ("xlsx", "xlsb"):
|
||||
return "openpyxl"
|
||||
if ext == "ods":
|
||||
return "odf"
|
||||
return "openpyxl"
|
||||
|
||||
|
||||
def convert_excel_to_xlsx_bytes(content: bytes, suffix: str = ".xlsx") -> bytes | None:
|
||||
"""Convert arbitrary spreadsheet bytes to XLSX using LibreOffice, if available."""
|
||||
soffice = find_soffice()
|
||||
if not soffice:
|
||||
return None
|
||||
|
||||
max_attempts = 3
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
with tempfile.TemporaryDirectory() as temp_dir, tempfile.TemporaryDirectory() as profile_dir:
|
||||
src = os.path.join(temp_dir, f"input{suffix}")
|
||||
with open(src, "wb") as handle:
|
||||
handle.write(content)
|
||||
|
||||
user_installation = Path(profile_dir).as_uri()
|
||||
cmd = [
|
||||
soffice,
|
||||
"--headless",
|
||||
f"-env:UserInstallation={user_installation}",
|
||||
"--convert-to",
|
||||
"xlsx",
|
||||
"--outdir",
|
||||
temp_dir,
|
||||
src,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, timeout=120)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
logger.warning("LibreOffice convert failed to start: %s", exc)
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.decode("utf-8", errors="ignore")
|
||||
logger.warning(
|
||||
"LibreOffice convert failed (attempt %s/%s): %s",
|
||||
attempt,
|
||||
max_attempts,
|
||||
stderr,
|
||||
)
|
||||
if attempt < max_attempts:
|
||||
time.sleep(0.5 * attempt)
|
||||
continue
|
||||
return None
|
||||
|
||||
for name in os.listdir(temp_dir):
|
||||
if name.endswith(".xlsx"):
|
||||
with open(os.path.join(temp_dir, name), "rb") as handle:
|
||||
converted = handle.read()
|
||||
logger.info(
|
||||
"Converted spreadsheet via LibreOffice (%s -> xlsx, %d bytes)",
|
||||
suffix,
|
||||
len(converted),
|
||||
)
|
||||
return converted
|
||||
|
||||
if attempt < max_attempts:
|
||||
time.sleep(0.5 * attempt)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_excel_bytes(content: bytes, file_type: str | None = None) -> bytes:
|
||||
"""Return bytes readable by pandas, converting via LibreOffice when needed."""
|
||||
ext = detect_excel_format(content)
|
||||
if ext is not None:
|
||||
return content
|
||||
|
||||
suffixes = []
|
||||
if file_type:
|
||||
suffixes.append(f".{file_type.lstrip('.')}")
|
||||
suffixes.extend([".xlsx", ".xls", ".et", ".csv"])
|
||||
seen: set[str] = set()
|
||||
for suffix in suffixes:
|
||||
if suffix in seen:
|
||||
continue
|
||||
seen.add(suffix)
|
||||
converted = convert_excel_to_xlsx_bytes(content, suffix=suffix)
|
||||
if converted and detect_excel_format(converted) is not None:
|
||||
return converted
|
||||
|
||||
raise ValueError(
|
||||
"Unrecognized Excel file format; the file may be corrupt, encrypted, "
|
||||
"or not a spreadsheet"
|
||||
)
|
||||
|
||||
|
||||
def find_soffice() -> Optional[str]:
|
||||
possible_paths = [
|
||||
"/usr/bin/soffice",
|
||||
"/usr/lib/libreoffice/program/soffice",
|
||||
"/opt/libreoffice25.2/program/soffice",
|
||||
"/Applications/LibreOffice.app/Contents/MacOS/soffice",
|
||||
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
|
||||
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
|
||||
]
|
||||
for path in possible_paths:
|
||||
if path and os.path.exists(path):
|
||||
return path
|
||||
|
||||
result = subprocess.run(["which", "soffice"], capture_output=True, text=True)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip()
|
||||
return None
|
||||
@@ -13,6 +13,14 @@ import pandas as pd
|
||||
|
||||
from docreader.models.document import Chunk, Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.excel_convert import (
|
||||
convert_excel_to_xlsx_bytes,
|
||||
detect_excel_format,
|
||||
engine_for_format,
|
||||
normalize_excel_bytes,
|
||||
)
|
||||
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
|
||||
from docreader.parser.xlsx_repair import repair_xlsx_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -60,13 +68,11 @@ class ExcelParser(BaseParser):
|
||||
text: List[str] = []
|
||||
start, end = 0, 0
|
||||
|
||||
# Load Excel file from bytes into pandas ExcelFile object
|
||||
excel_file = pd.ExcelFile(BytesIO(content))
|
||||
excel_file = _open_excel_file(content, file_type=self.file_type)
|
||||
|
||||
# Process each sheet in the Excel file
|
||||
for excel_sheet_name in excel_file.sheet_names:
|
||||
# Parse the sheet into a DataFrame
|
||||
df = excel_file.parse(sheet_name=excel_sheet_name)
|
||||
df = _read_sheet_dataframe(excel_file, excel_sheet_name)
|
||||
# Remove rows where all values are NaN (completely empty rows)
|
||||
df.dropna(how="all", inplace=True)
|
||||
|
||||
@@ -97,6 +103,90 @@ class ExcelParser(BaseParser):
|
||||
return Document(content="".join(text), chunks=chunks)
|
||||
|
||||
|
||||
def _read_sheet_dataframe(excel_file: pd.ExcelFile, sheet_name: str) -> pd.DataFrame:
|
||||
"""Read a worksheet into a DataFrame with stable column labels."""
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
# XLSX is preprocessed (merge fill); use A/B/C column letters and keep row 1 as data.
|
||||
if excel_file.engine == "openpyxl":
|
||||
df = excel_file.parse(sheet_name=sheet_name, header=None)
|
||||
df.columns = [get_column_letter(idx + 1) for idx in range(len(df.columns))]
|
||||
return df
|
||||
|
||||
df = excel_file.parse(sheet_name=sheet_name, header=0)
|
||||
if df.empty:
|
||||
df = excel_file.parse(sheet_name=sheet_name, header=None)
|
||||
df.columns = [get_column_letter(idx + 1) for idx in range(len(df.columns))]
|
||||
elif any(str(col).startswith("Unnamed:") for col in df.columns):
|
||||
df = excel_file.parse(sheet_name=sheet_name, header=None)
|
||||
df.columns = [get_column_letter(idx + 1) for idx in range(len(df.columns))]
|
||||
return df
|
||||
|
||||
|
||||
def _prepare_xlsx_bytes(data: bytes) -> bytes:
|
||||
repaired = repair_xlsx_bytes(data)
|
||||
if repaired is not None:
|
||||
data = repaired
|
||||
return fill_merged_cells_xlsx(data)
|
||||
|
||||
|
||||
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
|
||||
"""Open an Excel workbook with explicit engine selection and fallbacks."""
|
||||
data = content
|
||||
converted_via_soffice = False
|
||||
|
||||
while True:
|
||||
ext = detect_excel_format(data)
|
||||
if ext is None:
|
||||
if converted_via_soffice:
|
||||
raise ValueError(
|
||||
"Excel file format cannot be determined, you must specify an engine manually."
|
||||
)
|
||||
try:
|
||||
data = normalize_excel_bytes(data, file_type=file_type)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Excel file format cannot be determined, you must specify an engine manually."
|
||||
) from exc
|
||||
converted_via_soffice = True
|
||||
continue
|
||||
|
||||
if ext == "ods":
|
||||
converted = convert_excel_to_xlsx_bytes(data, suffix=".ods")
|
||||
if converted:
|
||||
data = converted
|
||||
continue
|
||||
|
||||
engine = engine_for_format(ext)
|
||||
if ext == "xlsx":
|
||||
data = _prepare_xlsx_bytes(data)
|
||||
engine = "openpyxl"
|
||||
try:
|
||||
return pd.ExcelFile(BytesIO(data), engine=engine)
|
||||
except ImportError as exc:
|
||||
raise ValueError(
|
||||
f"Excel engine {engine!r} is not available for .{ext} files"
|
||||
) from exc
|
||||
except KeyError as exc:
|
||||
if "sharedStrings.xml" not in str(exc) or engine != "openpyxl":
|
||||
raise
|
||||
repaired = repair_xlsx_bytes(data)
|
||||
if repaired is None:
|
||||
raise
|
||||
logger.info("Repaired XLSX sharedStrings packaging before parse")
|
||||
data = _prepare_xlsx_bytes(repaired)
|
||||
continue
|
||||
except ValueError as exc:
|
||||
if converted_via_soffice or "cannot be determined" not in str(exc):
|
||||
raise
|
||||
try:
|
||||
data = normalize_excel_bytes(content, file_type=file_type)
|
||||
except ValueError:
|
||||
raise
|
||||
converted_via_soffice = True
|
||||
continue
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage: Parse an Excel file and display results
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
"""Optional PDF engine backed by LiteParse (LlamaIndex, MIT).
|
||||
|
||||
LiteParse is a fast Rust/PDFium text extractor that performs spatial reading-order
|
||||
reconstruction natively (multi-column aware) and is considerably faster than the
|
||||
Python text path. It is exposed as a *selectable* engine (``liteparse``) rather
|
||||
than replacing the builtin engine, so users can opt in per knowledge base.
|
||||
|
||||
Scope/limitations (documented intentionally):
|
||||
* Text-first engine: it returns reading-order plain text, not figures. Scanned
|
||||
pages carry no text layer, so for image-dominated PDFs we fall back to the
|
||||
builtin scanned renderer (page -> JPEG, OCR'd by the Go App) to stay robust.
|
||||
* docreader never runs OCR itself; OCR/VLM remain Go-side responsibilities.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# If the extracted text averages fewer characters per page than this, the PDF is
|
||||
# treated as scanned/image-dominated and routed to the builtin image renderer.
|
||||
_MIN_CHARS_PER_PAGE = 20
|
||||
# If at least this fraction of sampled pages are image-dominated, the PDF is
|
||||
# scanned (even when it carries a garbled OCR text layer) and is routed to the
|
||||
# builtin image renderer rather than trusting the low-quality text.
|
||||
_SCANNED_PAGE_FRACTION = 0.5
|
||||
|
||||
|
||||
def liteparse_available(_overrides=None):
|
||||
"""Engine availability probe used by the registry/UI."""
|
||||
try:
|
||||
import liteparse # noqa: F401
|
||||
except Exception as e: # pragma: no cover - depends on install
|
||||
return False, f"liteparse 未安装: {e}"
|
||||
return True, ""
|
||||
|
||||
|
||||
class LiteParseParser(BaseParser):
|
||||
"""Parse a PDF with LiteParse, falling back to scanned rendering when empty."""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
import liteparse
|
||||
|
||||
from docreader.parser.pdf_parser import (
|
||||
PDFScannedParser,
|
||||
estimate_scanned_fraction,
|
||||
)
|
||||
|
||||
# Image-dominated PDFs (incl. ones with a garbled OCR text layer) carry
|
||||
# no trustworthy text; render them as images for Go-side OCR instead.
|
||||
try:
|
||||
scanned_frac = estimate_scanned_fraction(content)
|
||||
except Exception:
|
||||
scanned_frac = 0.0
|
||||
if scanned_frac >= _SCANNED_PAGE_FRACTION:
|
||||
logger.info(
|
||||
"LiteParseParser: %s is image-dominated (%.0f%% scanned pages); "
|
||||
"using builtin scanned renderer",
|
||||
self.file_name,
|
||||
scanned_frac * 100,
|
||||
)
|
||||
return PDFScannedParser(
|
||||
file_name=self.file_name, file_type=self.file_type
|
||||
).parse_into_text(content)
|
||||
|
||||
engine = liteparse.LiteParse(ocr_enabled=False, quiet=True)
|
||||
result = engine.parse(content)
|
||||
page_count = int(result.num_pages)
|
||||
|
||||
page_texts = []
|
||||
for i in range(page_count):
|
||||
page = result.get_page(i)
|
||||
page_texts.append((getattr(page, "text", "") or "").strip())
|
||||
|
||||
doc_text = (getattr(result, "text", "") or "").strip()
|
||||
if not doc_text:
|
||||
doc_text = "\n\n".join(t for t in page_texts if t)
|
||||
|
||||
# Image-dominated / scanned PDFs yield little to no text: defer to the
|
||||
# builtin scanned renderer so the Go App can OCR the page images.
|
||||
if page_count and len(doc_text) < _MIN_CHARS_PER_PAGE * page_count:
|
||||
logger.info(
|
||||
"LiteParseParser: %s looks scanned (%d chars / %d pages); "
|
||||
"falling back to builtin scanned renderer",
|
||||
self.file_name,
|
||||
len(doc_text),
|
||||
page_count,
|
||||
)
|
||||
return PDFScannedParser(
|
||||
file_name=self.file_name, file_type=self.file_type
|
||||
).parse_into_text(content)
|
||||
|
||||
logger.info(
|
||||
"LiteParseParser: %s -> %d pages, content_len=%d",
|
||||
self.file_name,
|
||||
page_count,
|
||||
len(doc_text),
|
||||
)
|
||||
return Document(
|
||||
content=doc_text,
|
||||
images={},
|
||||
metadata={
|
||||
"page_count": page_count,
|
||||
"image_source_type": "pdf_text_layer",
|
||||
"parser_engine": "liteparse",
|
||||
},
|
||||
)
|
||||
@@ -18,6 +18,8 @@ import re
|
||||
import uuid
|
||||
from typing import Dict, List, Match, Optional, Tuple
|
||||
|
||||
_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
@@ -58,6 +60,71 @@ class MarkdownTableUtil:
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _split_row_cells(row_line: str) -> List[str]:
|
||||
"""Split a markdown table row into cells, preserving empty cells."""
|
||||
inner = row_line.strip()
|
||||
if not inner.startswith("|"):
|
||||
return []
|
||||
parts = inner.split("|")
|
||||
if parts and parts[0].strip() == "":
|
||||
parts = parts[1:]
|
||||
if parts and parts[-1].strip() == "":
|
||||
parts = parts[:-1]
|
||||
return [part.strip() for part in parts]
|
||||
|
||||
@staticmethod
|
||||
def _is_table_row(line: str) -> bool:
|
||||
stripped = line.strip()
|
||||
return stripped.startswith("|") and "|" in stripped[1:]
|
||||
|
||||
@classmethod
|
||||
def _is_separator_row(cls, line: str) -> bool:
|
||||
cells = cls._split_row_cells(line)
|
||||
return bool(cells) and all(_SEPARATOR_CELL.match(cell) for cell in cells)
|
||||
|
||||
@classmethod
|
||||
def _is_empty_row(cls, line: str) -> bool:
|
||||
cells = cls._split_row_cells(line)
|
||||
return bool(cells) and all(cell == "" for cell in cells)
|
||||
|
||||
@classmethod
|
||||
def _separator_row_for(cls, header_line: str) -> str:
|
||||
cells = cls._split_row_cells(header_line)
|
||||
return "| " + " | ".join("---" for _ in cells) + " |"
|
||||
|
||||
@classmethod
|
||||
def _normalize_table_block(cls, block: List[str]) -> List[str]:
|
||||
"""Fix MarkItDown-style tables: drop bogus prefix rows, ensure GFM delimiter."""
|
||||
while block and cls._is_empty_row(block[0]):
|
||||
block.pop(0)
|
||||
if block and cls._is_separator_row(block[0]):
|
||||
block.pop(0)
|
||||
# GFM/marked need "| --- |" after the first row. Headerless Word tables
|
||||
# only have data rows after we strip the fake empty+separator prefix.
|
||||
if len(block) >= 2 and not cls._is_separator_row(block[1]):
|
||||
sep = cls._separator_row_for(block[0])
|
||||
block = [block[0], sep] + block[1:]
|
||||
return block
|
||||
|
||||
def normalize_spurious_table_prefixes(self, content: str) -> str:
|
||||
"""Remove bogus empty/separator prefix rows from MarkItDown table output."""
|
||||
lines = content.split("\n")
|
||||
out: List[str] = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
if not self._is_table_row(line):
|
||||
out.append(line)
|
||||
i += 1
|
||||
continue
|
||||
block: List[str] = []
|
||||
while i < len(lines) and self._is_table_row(lines[i]):
|
||||
block.append(lines[i])
|
||||
i += 1
|
||||
out.extend(self._normalize_table_block(block))
|
||||
return "\n".join(out)
|
||||
|
||||
def format_table(self, content: str) -> str:
|
||||
"""Format all Markdown tables in the content.
|
||||
|
||||
@@ -70,8 +137,7 @@ class MarkdownTableUtil:
|
||||
|
||||
def process_align(match: Match[str]) -> str:
|
||||
"""Process alignment row to standardize format."""
|
||||
# Split by | and remove empty strings
|
||||
columns = [col.strip() for col in match.group(0).split("|") if col.strip()]
|
||||
columns = self._split_row_cells(match.group(0))
|
||||
|
||||
processed = []
|
||||
for col in columns:
|
||||
@@ -87,8 +153,7 @@ class MarkdownTableUtil:
|
||||
|
||||
def process_line(match: Match[str]) -> str:
|
||||
"""Process regular table row to standardize format."""
|
||||
# Split by | and remove empty strings
|
||||
columns = [col.strip() for col in match.group(0).split("|") if col.strip()]
|
||||
columns = self._split_row_cells(match.group(0))
|
||||
|
||||
# Preserve original indentation
|
||||
prefix = match.group(1)
|
||||
@@ -99,8 +164,7 @@ class MarkdownTableUtil:
|
||||
formatted_content = self.line_pattern.sub(process_line, formatted_content)
|
||||
# Then format alignment rows (must be done after to avoid conflicts)
|
||||
formatted_content = self.align_pattern.sub(process_align, formatted_content)
|
||||
|
||||
return formatted_content
|
||||
return self.normalize_spurious_table_prefixes(formatted_content)
|
||||
|
||||
@staticmethod
|
||||
def _self_test():
|
||||
|
||||
@@ -9,6 +9,11 @@ from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.ppt_convert import normalize_ppt_bytes
|
||||
from docreader.parser.pptx_media import (
|
||||
attach_pptx_media_to_markdown,
|
||||
markdown_needs_pptx_media_attach,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,16 +37,41 @@ class StdMarkitdownParser(BaseParser):
|
||||
Uses self.file_type (inherited from BaseParser) to hint the stream format.
|
||||
"""
|
||||
ext = self.file_type
|
||||
if ext and not ext.startswith('.'):
|
||||
ext = '.' + ext
|
||||
ft = (ext or "").lstrip(".").lower()
|
||||
pptx_bytes: bytes | None = None
|
||||
if ft in ("ppt", "pptx"):
|
||||
content, ext = normalize_ppt_bytes(content, ft)
|
||||
pptx_bytes = content
|
||||
ft = "pptx"
|
||||
elif ext and not ext.startswith("."):
|
||||
ext = "." + ext
|
||||
|
||||
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
|
||||
result = self.markitdown.convert(
|
||||
result = self._convert_markitdown(content, ext, keep_data_uris=True)
|
||||
if result is None:
|
||||
logger.warning(
|
||||
"MarkItDown failed with embedded images for %s; retrying without data URIs",
|
||||
ft or ext,
|
||||
)
|
||||
result = self._convert_markitdown(content, ext, keep_data_uris=False)
|
||||
|
||||
text = result.text_content
|
||||
images: dict[str, str] = {}
|
||||
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
|
||||
text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
|
||||
return Document(content=text, images=images)
|
||||
|
||||
def _convert_markitdown(self, content: bytes, ext: str | None, *, keep_data_uris: bool):
|
||||
try:
|
||||
return self.markitdown.convert(
|
||||
io.BytesIO(content),
|
||||
file_extension=ext,
|
||||
keep_data_uris=True
|
||||
keep_data_uris=keep_data_uris,
|
||||
)
|
||||
return Document(content=result.text_content)
|
||||
except Exception:
|
||||
if keep_data_uris:
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
class MarkitdownParser(PipelineParser):
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
"""PDF parser backed by OpenDataLoader PDF (Apache-2.0).
|
||||
|
||||
Requires Java 11+ on PATH and the ``opendataloader-pdf`` Python package.
|
||||
Each ``convert()`` spawns a JVM; concurrency is limited via
|
||||
``DOCREADER_ODL_MAX_WORKERS``.
|
||||
|
||||
Hybrid mode (``docling-fast``, etc.) needs a running
|
||||
``opendataloader-pdf-hybrid`` server — configure ``DOCREADER_ODL_HYBRID_URL``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, Mapping, Optional, Tuple
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MIN_CHARS_PER_PAGE = 20
|
||||
_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")
|
||||
_MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
_IMAGE_FILE_NUM_RE = re.compile(r"^imageFile(\d+)\.", re.I)
|
||||
|
||||
|
||||
def _override_str(overrides: Optional[Mapping[str, Any]], key: str, default: str = "") -> str:
|
||||
if overrides:
|
||||
v = overrides.get(key)
|
||||
if v is not None and str(v).strip() != "":
|
||||
return str(v).strip()
|
||||
return default
|
||||
|
||||
|
||||
def _override_bool(overrides: Optional[Mapping[str, Any]], key: str, default: bool) -> bool:
|
||||
if overrides:
|
||||
v = overrides.get(key)
|
||||
if v is not None and str(v).strip() != "":
|
||||
return str(v).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
return default
|
||||
|
||||
|
||||
def _java_available() -> Tuple[bool, str]:
|
||||
if not shutil.which("java"):
|
||||
return False, "需要 Java 11+(JRE),请安装并在 PATH 中配置 java"
|
||||
return True, ""
|
||||
|
||||
|
||||
def _package_available() -> Tuple[bool, str]:
|
||||
try:
|
||||
import opendataloader_pdf # noqa: F401
|
||||
except ImportError as e:
|
||||
return False, f"opendataloader-pdf 未安装: {e}"
|
||||
return True, ""
|
||||
|
||||
|
||||
def _ping_hybrid(
|
||||
url: str,
|
||||
*,
|
||||
timeout_sec: float = 5.0,
|
||||
retries: int = 3,
|
||||
retry_delay_sec: float = 2.0,
|
||||
) -> Tuple[bool, str]:
|
||||
import time
|
||||
|
||||
base = url.rstrip("/")
|
||||
health_url = f"{base}/health"
|
||||
last_err = ""
|
||||
for attempt in range(max(1, retries)):
|
||||
try:
|
||||
req = urllib.request.Request(health_url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=timeout_sec) as resp:
|
||||
if 200 <= resp.status < 300:
|
||||
return True, ""
|
||||
last_err = f"hybrid 健康检查 HTTP {resp.status}: {health_url}"
|
||||
except urllib.error.URLError as e:
|
||||
last_err = f"无法连接 OpenDataLoader hybrid 服务 ({health_url}): {e}"
|
||||
except Exception as e:
|
||||
last_err = f"hybrid 健康检查失败: {e}"
|
||||
if attempt + 1 < retries:
|
||||
time.sleep(retry_delay_sec)
|
||||
hint = (
|
||||
";若刚执行 make dev-start --odl-hybrid,请等待镜像构建/服务就绪"
|
||||
"(docker logs WeKnora-odl-hybrid)"
|
||||
)
|
||||
return False, last_err + hint
|
||||
|
||||
|
||||
def opendataloader_available(
|
||||
overrides: Optional[Mapping[str, Any]] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Registry / ListEngines availability probe."""
|
||||
ok, msg = _java_available()
|
||||
if not ok:
|
||||
return False, msg
|
||||
ok, msg = _package_available()
|
||||
if not ok:
|
||||
return False, msg
|
||||
|
||||
hybrid = _resolve_hybrid(overrides)
|
||||
if hybrid and hybrid.lower() not in ("off", ""):
|
||||
url = _resolve_hybrid_url(overrides)
|
||||
if url:
|
||||
return _ping_hybrid(url, retries=6, retry_delay_sec=5.0, timeout_sec=5.0)
|
||||
return True, ""
|
||||
|
||||
|
||||
def _resolve_hybrid(overrides: Optional[Mapping[str, Any]] = None) -> str:
|
||||
return _override_str(overrides, "odl_hybrid", CONFIG.odl_hybrid)
|
||||
|
||||
|
||||
def _resolve_hybrid_url(overrides: Optional[Mapping[str, Any]] = None) -> str:
|
||||
return _override_str(overrides, "odl_hybrid_url", CONFIG.odl_hybrid_url)
|
||||
|
||||
|
||||
def _find_markdown_file(output_dir: str, pdf_stem: str) -> str:
|
||||
candidates = []
|
||||
for root, _, files in os.walk(output_dir):
|
||||
for name in files:
|
||||
if name.lower().endswith(".md"):
|
||||
path = os.path.join(root, name)
|
||||
candidates.append(path)
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"OpenDataLoader 未在 {output_dir} 生成 markdown 文件")
|
||||
for path in candidates:
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
if base == pdf_stem or base.startswith(pdf_stem):
|
||||
return path
|
||||
candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _normalize_odl_image_url(raw: str) -> str:
|
||||
"""OpenDataLoader wraps paths as ``<images/foo.png>``; storage may HTML-escape them."""
|
||||
s = html.unescape((raw or "").strip())
|
||||
s = s.replace("<", "<").replace(">", ">").replace("&", "&")
|
||||
s = s.strip().strip("<>").strip().strip('"').strip("'")
|
||||
if s.startswith("./"):
|
||||
s = s[2:]
|
||||
return s.replace("\\", "/")
|
||||
|
||||
|
||||
def _canonical_image_ref(abs_path: str, output_dir: str) -> str:
|
||||
"""Use ``images/<file>`` keys to match OpenDataLoader markdown conventions."""
|
||||
rel = os.path.relpath(abs_path, output_dir).replace("\\", "/")
|
||||
name = os.path.basename(abs_path)
|
||||
if rel.startswith("images/"):
|
||||
return rel
|
||||
return f"images/{name}"
|
||||
|
||||
|
||||
def _collect_images_under_output(output_dir: str) -> Dict[str, str]:
|
||||
"""Collect every extracted image under the convert output tree."""
|
||||
images: Dict[str, str] = {}
|
||||
for root, _, files in os.walk(output_dir):
|
||||
for name in files:
|
||||
if not name.lower().endswith(_IMAGE_SUFFIXES):
|
||||
continue
|
||||
abs_path = os.path.join(root, name)
|
||||
ref = _canonical_image_ref(abs_path, output_dir)
|
||||
if ref in images:
|
||||
continue
|
||||
with open(abs_path, "rb") as f:
|
||||
images[ref] = base64.b64encode(f.read()).decode("utf-8")
|
||||
return images
|
||||
|
||||
|
||||
def _register_image_alias(aliases: Dict[str, str], alias: str, canonical: str) -> None:
|
||||
key = _normalize_odl_image_url(alias)
|
||||
if key:
|
||||
aliases[key] = canonical
|
||||
|
||||
|
||||
def _build_path_alias_map(images: Dict[str, str]) -> Dict[str, str]:
|
||||
"""Map ODL markdown spellings (angle brackets, entities, basenames) to dict keys."""
|
||||
aliases: Dict[str, str] = {}
|
||||
for ref in images:
|
||||
base = os.path.basename(ref)
|
||||
variants = [
|
||||
ref,
|
||||
base,
|
||||
f"images/{base}",
|
||||
f"<{ref}>",
|
||||
f"<images/{base}>",
|
||||
f"<{ref}>",
|
||||
f"<images/{base}>",
|
||||
]
|
||||
for variant in variants:
|
||||
_register_image_alias(aliases, variant, ref)
|
||||
return aliases
|
||||
|
||||
|
||||
def _resolve_image_ref(url: str, aliases: Dict[str, str]) -> Optional[str]:
|
||||
key = _normalize_odl_image_url(url)
|
||||
if not key or key.startswith("data:"):
|
||||
return None
|
||||
if key in aliases:
|
||||
return aliases[key]
|
||||
base = os.path.basename(key)
|
||||
for candidate in (base, f"images/{base}"):
|
||||
if candidate in aliases:
|
||||
return aliases[candidate]
|
||||
m = _IMAGE_FILE_NUM_RE.match(base)
|
||||
if m:
|
||||
num = int(m.group(1))
|
||||
numbered = []
|
||||
for ref in {aliases[k] for k in aliases}:
|
||||
bm = _IMAGE_FILE_NUM_RE.match(os.path.basename(ref))
|
||||
if bm:
|
||||
numbered.append((int(bm.group(1)), ref))
|
||||
numbered.sort(key=lambda x: x[0])
|
||||
for n, ref in numbered:
|
||||
if n == num:
|
||||
return ref
|
||||
if numbered and 1 <= num <= len(numbered):
|
||||
return numbered[num - 1][1]
|
||||
return None
|
||||
|
||||
|
||||
def _rewrite_markdown_image_refs(
|
||||
markdown: str, images: Dict[str, str]
|
||||
) -> str:
|
||||
if not images:
|
||||
return markdown
|
||||
aliases = _build_path_alias_map(images)
|
||||
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
alt, raw_url = match.group(1), match.group(2)
|
||||
url = raw_url.strip().split()[0] if raw_url else ""
|
||||
canonical = _resolve_image_ref(url, aliases)
|
||||
if canonical is None:
|
||||
return match.group(0)
|
||||
return f""
|
||||
|
||||
return _MD_IMAGE_RE.sub(repl, markdown)
|
||||
|
||||
|
||||
def _run_convert(
|
||||
pdf_path: str,
|
||||
output_dir: str,
|
||||
image_dir: str,
|
||||
overrides: Optional[Mapping[str, Any]] = None,
|
||||
) -> None:
|
||||
import opendataloader_pdf
|
||||
|
||||
kwargs: Dict[str, Any] = {
|
||||
"input_path": pdf_path,
|
||||
"output_dir": output_dir,
|
||||
"format": "markdown",
|
||||
"image_output": "external",
|
||||
"image_dir": image_dir,
|
||||
"quiet": True,
|
||||
"markdown_with_html": _override_bool(
|
||||
overrides, "odl_markdown_with_html", CONFIG.odl_markdown_with_html
|
||||
),
|
||||
}
|
||||
hybrid = _resolve_hybrid(overrides)
|
||||
if hybrid and hybrid.lower() not in ("off", ""):
|
||||
kwargs["hybrid"] = hybrid
|
||||
hybrid_url = _resolve_hybrid_url(overrides)
|
||||
if hybrid_url:
|
||||
kwargs["hybrid_url"] = hybrid_url
|
||||
hybrid_mode = _override_str(overrides, "odl_hybrid_mode", CONFIG.odl_hybrid_mode)
|
||||
if hybrid_mode:
|
||||
kwargs["hybrid_mode"] = hybrid_mode
|
||||
if _override_bool(overrides, "odl_hybrid_fallback", CONFIG.odl_hybrid_fallback):
|
||||
kwargs["hybrid_fallback"] = True
|
||||
|
||||
opendataloader_pdf.convert(**kwargs)
|
||||
|
||||
|
||||
class OpenDataLoaderParser(BaseParser):
|
||||
"""Parse PDFs with OpenDataLoader (layout-aware markdown + external images)."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
self._engine_overrides: Dict[str, Any] = {
|
||||
k: v
|
||||
for k, v in kwargs.items()
|
||||
if k.startswith("odl_") or k in ("mineru_endpoint", "mineru_api_key")
|
||||
}
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
ok, msg = opendataloader_available(self._engine_overrides)
|
||||
if not ok:
|
||||
raise RuntimeError(msg)
|
||||
|
||||
safe_name = os.path.basename(self.file_name) or "document.pdf"
|
||||
if not safe_name.lower().endswith(".pdf"):
|
||||
safe_name = f"{os.path.splitext(safe_name)[0] or 'document'}.pdf"
|
||||
pdf_stem = os.path.splitext(safe_name)[0]
|
||||
|
||||
max_workers = CONFIG.odl_max_workers
|
||||
with parser_worker_limit("opendataloader", max_workers):
|
||||
with tempfile.TemporaryDirectory(prefix="weknora-odl-") as tmp_dir:
|
||||
pdf_path = os.path.join(tmp_dir, safe_name)
|
||||
with open(pdf_path, "wb") as f:
|
||||
f.write(content)
|
||||
image_dir = os.path.join(tmp_dir, "images")
|
||||
os.makedirs(image_dir, exist_ok=True)
|
||||
|
||||
_run_convert(
|
||||
pdf_path,
|
||||
tmp_dir,
|
||||
image_dir,
|
||||
overrides=self._engine_overrides,
|
||||
)
|
||||
|
||||
md_path = _find_markdown_file(tmp_dir, pdf_stem)
|
||||
with open(md_path, encoding="utf-8", errors="replace") as f:
|
||||
text = f.read()
|
||||
|
||||
images = _collect_images_under_output(tmp_dir)
|
||||
text = _rewrite_markdown_image_refs(text, images)
|
||||
|
||||
if len(text.strip()) < _MIN_CHARS_PER_PAGE:
|
||||
logger.info(
|
||||
"OpenDataLoaderParser: %s yielded little text; "
|
||||
"falling back to builtin scanned renderer",
|
||||
self.file_name,
|
||||
)
|
||||
from docreader.parser.pdf_parser import PDFScannedParser
|
||||
|
||||
return PDFScannedParser(
|
||||
file_name=self.file_name, file_type=self.file_type
|
||||
).parse_into_text(content)
|
||||
|
||||
logger.info(
|
||||
"OpenDataLoaderParser: %s -> content_len=%d images=%d",
|
||||
self.file_name,
|
||||
len(text),
|
||||
len(images),
|
||||
)
|
||||
return Document(
|
||||
content=text,
|
||||
images=images,
|
||||
metadata={
|
||||
"parser_engine": "opendataloader",
|
||||
"odl_hybrid": _resolve_hybrid(self._engine_overrides) or "off",
|
||||
},
|
||||
)
|
||||
+668
-41
@@ -20,6 +20,7 @@ import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
|
||||
from docreader.config import CONFIG
|
||||
@@ -87,12 +88,51 @@ EMBED_MAX_IMAGES = _env_int("DOCREADER_PDF_EMBED_MAX_IMAGES", 50)
|
||||
# Reconstruct reading order with a geometric XY-cut so multi-column pages are
|
||||
# linearised column-by-column instead of line-interleaved.
|
||||
LAYOUT_ORDERING = _env_bool("DOCREADER_PDF_LAYOUT_ORDERING", True)
|
||||
# When glyphs are positioned without explicit space characters (common in OCR /
|
||||
# search text layers), insert a space if the horizontal gap exceeds this
|
||||
# multiple of the line's median glyph width.
|
||||
WORD_GAP_WIDTH_RATIO = _env_float("DOCREADER_PDF_WORD_GAP_WIDTH_RATIO", 0.4)
|
||||
# Promote visually larger lines to markdown headings (font-size proxy = rect
|
||||
# height relative to the page's median line height).
|
||||
DETECT_HEADINGS = _env_bool("DOCREADER_PDF_DETECT_HEADINGS", True)
|
||||
# Drop invisible (render-mode 3), off-page and degenerate text — a cheap guard
|
||||
# against hidden-text prompt injection and OCR artefacts.
|
||||
FILTER_HIDDEN_TEXT = _env_bool("DOCREADER_PDF_FILTER_HIDDEN_TEXT", True)
|
||||
# Narrow side strips (arXiv watermarks, page labels) narrower than this share of
|
||||
# page width are dropped when they look like vertical / single-glyph noise.
|
||||
MARGIN_COL_WIDTH_RATIO = _env_float("DOCREADER_PDF_MARGIN_COL_WIDTH_RATIO", 0.12)
|
||||
# Minimum characters on a line before font-size heuristics may promote it to a
|
||||
# markdown heading (avoids ``### C`` from margin glyphs).
|
||||
MIN_HEADING_LINE_CHARS = _env_int("DOCREADER_PDF_MIN_HEADING_LINE_CHARS", 8)
|
||||
# Strip pdfium placeholder glyphs (U+FFFE) and soft hyphens; remove axis/legend text
|
||||
# from vector figures when a Figure caption is present on the page.
|
||||
SANITIZE_PDF_TEXT = _env_bool("DOCREADER_PDF_SANITIZE_TEXT", True)
|
||||
STRIP_CHART_TEXT_DEBRIS = _env_bool("DOCREADER_PDF_STRIP_CHART_DEBRIS", True)
|
||||
# Render detected vector chart regions (no embedded bitmap) as JPEG for VLM/OCR.
|
||||
RENDER_VECTOR_FIGURES = _env_bool("DOCREADER_PDF_RENDER_VECTOR_FIGURES", True)
|
||||
MIN_CHART_REGION_CHARS = _env_int("DOCREADER_PDF_MIN_CHART_REGION_CHARS", 18)
|
||||
MIN_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MIN_CHART_REGION_AREA", 0.015)
|
||||
MAX_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MAX_CHART_REGION_AREA", 0.42)
|
||||
MAX_FIGURE_HEIGHT_RATIO = _env_float("DOCREADER_PDF_MAX_FIGURE_HEIGHT_RATIO", 0.38)
|
||||
|
||||
# pdfium / Adobe text layers often emit U+FFFE for missing hyphenation or ligatures.
|
||||
_PDF_ARTIFACT_RE = re.compile(r"[\u00ad\u200b-\u200f\ufeff\ufffe\uffff]")
|
||||
_PDF_ARTIFACT_JOIN_RE = re.compile(r"(\w)[\u00ad\ufffe](\w)")
|
||||
_CHART_DEBRIS_LINE_RE = re.compile(
|
||||
r"^(?:"
|
||||
r"[\d\s.]+|"
|
||||
r"\d{1,2}|"
|
||||
r"\d+-layer|"
|
||||
r"iter\.\s*\(1e4\)|"
|
||||
r"(?:training|test)\s+error\s*\(%\)"
|
||||
r")$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CHART_LAYER_RE = re.compile(r"^\d+-layer$", re.IGNORECASE)
|
||||
_FIGURE_CAPTION_RE = re.compile(r"^Figure\s+\d+\b", re.IGNORECASE)
|
||||
_FIGURE_CAPTION_SEARCH_RE = re.compile(r"\bFigure\s+(\d+)\b", re.IGNORECASE)
|
||||
_ARXIV_LINE_RE = re.compile(r"^arXiv:\s*\S+", re.IGNORECASE)
|
||||
_PAGE_NUM_LINE_RE = re.compile(r"^\d{1,3}$")
|
||||
|
||||
|
||||
def _close_pdfium_resource(resource) -> None:
|
||||
@@ -150,6 +190,394 @@ def _extract_page_text(page) -> str:
|
||||
_close_pdfium_resource(textpage)
|
||||
|
||||
|
||||
def _sanitize_pdf_text(text: str) -> str:
|
||||
"""Remove PDF text-layer placeholders and repair broken hyphenations."""
|
||||
if not text:
|
||||
return text
|
||||
text = _PDF_ARTIFACT_RE.sub("", text)
|
||||
text = _PDF_ARTIFACT_JOIN_RE.sub(r"\1\2", text)
|
||||
return text
|
||||
|
||||
|
||||
def _is_chart_debris_line(line: str) -> bool:
|
||||
t = line.strip()
|
||||
if not t:
|
||||
return False
|
||||
if _CHART_DEBRIS_LINE_RE.match(t):
|
||||
return True
|
||||
if _CHART_LAYER_RE.match(t):
|
||||
return True
|
||||
# Tick labels like "0 1 2 3 4 5 6 0"
|
||||
if re.fullmatch(r"[\d\s.()-]+", t) and len(t) <= 24 and sum(c.isdigit() for c in t) >= 3:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _strip_chart_text_debris(text: str) -> str:
|
||||
"""Drop runs of axis/legend lines leaked from vector figures into the text layer."""
|
||||
if not text:
|
||||
return text
|
||||
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
||||
out: list = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
if _is_chart_debris_line(lines[i]):
|
||||
j = i
|
||||
while j < len(lines) and (
|
||||
_is_chart_debris_line(lines[j]) or not lines[j].strip()
|
||||
):
|
||||
j += 1
|
||||
if j - i >= 3:
|
||||
i = j
|
||||
continue
|
||||
out.append(lines[i])
|
||||
i += 1
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _strip_arxiv_and_page_num_lines(text: str) -> str:
|
||||
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
||||
kept: list = []
|
||||
for ln in lines:
|
||||
t = ln.strip()
|
||||
if _ARXIV_LINE_RE.match(t):
|
||||
continue
|
||||
if _PAGE_NUM_LINE_RE.match(t):
|
||||
continue
|
||||
if "arXiv:" in ln:
|
||||
ln = re.sub(r"\s*arXiv:\s*\S+\s*(?:\[[^\]]+\])?\s*[^\n]*", "", ln).strip()
|
||||
if not ln:
|
||||
continue
|
||||
kept.append(ln)
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def _strip_lines_above_figure_captions(text: str) -> str:
|
||||
"""Remove diagram/chart label lines that sit immediately above a Figure caption."""
|
||||
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
||||
out: list = []
|
||||
for ln in lines:
|
||||
if _line_has_figure_caption(ln):
|
||||
while out and _is_figure_interior_line(out[-1]):
|
||||
out.pop()
|
||||
out.append(ln)
|
||||
else:
|
||||
out.append(ln)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _is_body_paragraph_line(text: str) -> bool:
|
||||
t = text.strip()
|
||||
if len(t) < 48:
|
||||
return False
|
||||
return len(t.split()) >= 8
|
||||
|
||||
|
||||
def _is_figure_interior_line(text: str) -> bool:
|
||||
"""Short, non-body line directly above a Figure caption (diagram labels, ticks)."""
|
||||
t = text.strip()
|
||||
if not t or _FIGURE_CAPTION_RE.match(t):
|
||||
return False
|
||||
if _ARXIV_LINE_RE.match(t) or _PAGE_NUM_LINE_RE.match(t):
|
||||
return True
|
||||
if _is_body_paragraph_line(t):
|
||||
return False
|
||||
if _is_chart_debris_line(t):
|
||||
return True
|
||||
# Prose sentence above a figure (wrapped paragraph tail) — keep in text.
|
||||
if t.endswith((".", "。", "!", "?", "!")) and len(t) >= 15:
|
||||
return False
|
||||
if len(t.split()) >= 7:
|
||||
return False
|
||||
if len(t) <= 40:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _postprocess_pdf_text(text: str) -> str:
|
||||
if SANITIZE_PDF_TEXT:
|
||||
text = _sanitize_pdf_text(text)
|
||||
text = _strip_arxiv_and_page_num_lines(text)
|
||||
text = _strip_lines_above_figure_captions(text)
|
||||
if STRIP_CHART_TEXT_DEBRIS:
|
||||
text = _strip_chart_text_debris(text)
|
||||
return text
|
||||
|
||||
|
||||
def _char_looks_chart_axis_tick(ch: str) -> bool:
|
||||
"""Axis tick / numeric chart labels only (not words like ``layer`` in diagrams)."""
|
||||
t = ch.strip()
|
||||
if not t:
|
||||
return False
|
||||
if len(t) == 1 and t in "0123456789.%()-":
|
||||
return True
|
||||
if _CHART_LAYER_RE.match(t):
|
||||
return True
|
||||
if re.fullmatch(r"iter\.\s*\(1e4\)", t, re.I):
|
||||
return True
|
||||
if re.fullmatch(r"(?:training|test)\s+error\s*\(%\)", t, re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _chars_bbox(char_list: list) -> tuple:
|
||||
return (
|
||||
min(c["x0"] for c in char_list),
|
||||
min(c["y0"] for c in char_list),
|
||||
max(c["x1"] for c in char_list),
|
||||
max(c["y1"] for c in char_list),
|
||||
)
|
||||
|
||||
|
||||
def _bbox_area_ratio(bbox, page_w: float, page_h: float) -> float:
|
||||
page_area = float(page_w) * float(page_h)
|
||||
if page_area <= 0:
|
||||
return 0.0
|
||||
x0, y0, x1, y1 = bbox
|
||||
return max(0.0, (x1 - x0) * (y1 - y0) / page_area)
|
||||
|
||||
|
||||
def _chart_region_bbox(chars: list, page_w: float, page_h: float):
|
||||
"""Bounding box of numeric chart axis labels (fallback when caption walk fails)."""
|
||||
chart = [c for c in chars if _char_looks_chart_axis_tick(c["ch"])]
|
||||
if len(chart) < MIN_CHART_REGION_CHARS:
|
||||
return None
|
||||
bbox = _chars_bbox(chart)
|
||||
ratio = _bbox_area_ratio(bbox, page_w, page_h)
|
||||
if ratio < MIN_CHART_REGION_AREA_RATIO or ratio > MAX_CHART_REGION_AREA_RATIO:
|
||||
return None
|
||||
x0, y0, x1, y1 = bbox
|
||||
pad_x = max(8.0, (x1 - x0) * 0.08)
|
||||
pad_y = max(8.0, (y1 - y0) * 0.08)
|
||||
return (
|
||||
max(0.0, x0 - pad_x),
|
||||
max(0.0, y0 - pad_y),
|
||||
min(page_w, x1 + pad_x),
|
||||
min(page_h, y1 + pad_y),
|
||||
)
|
||||
|
||||
|
||||
def _expand_chart_bbox(bbox, page_w: float, page_h: float, margin_frac: float = 0.18):
|
||||
x0, y0, x1, y1 = bbox
|
||||
dx = (x1 - x0) * margin_frac
|
||||
dy = (y1 - y0) * margin_frac
|
||||
return (
|
||||
max(0.0, x0 - dx),
|
||||
max(0.0, y0 - dy),
|
||||
min(page_w, x1 + dx),
|
||||
min(page_h, y1 + dy),
|
||||
)
|
||||
|
||||
|
||||
def _render_page_clip_jpeg(page, bbox, scale: float, quality: int, max_edge: int) -> bytes:
|
||||
"""Render a PDF page region to JPEG (bbox in PDF points, bottom-left origin)."""
|
||||
left, bottom, right, top = bbox
|
||||
scale_eff = _effective_scale(page, scale, max_edge)
|
||||
bitmap = None
|
||||
try:
|
||||
bitmap = page.render(scale=scale_eff)
|
||||
pil = bitmap.to_pil().convert("RGB")
|
||||
finally:
|
||||
_close_pdfium_resource(bitmap)
|
||||
page_w, page_h = page.get_size()
|
||||
x0 = int(left * scale_eff)
|
||||
x1 = int(right * scale_eff)
|
||||
y0 = int((page_h - top) * scale_eff)
|
||||
y1 = int((page_h - bottom) * scale_eff)
|
||||
if x1 <= x0 or y1 <= y0:
|
||||
raise ValueError("degenerate clip bbox")
|
||||
return _pil_to_jpeg_bytes(pil.crop((x0, y0, x1, y1)), quality)
|
||||
|
||||
|
||||
def _pil_to_jpeg_bytes(pil, quality: int) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
if pil.mode not in ("RGB", "L"):
|
||||
pil = pil.convert("RGB")
|
||||
pil.save(buf, format="JPEG", quality=quality, optimize=True)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _group_lines_with_chars(chars: list) -> list:
|
||||
"""Group glyphs into lines; each line includes its char list and bbox."""
|
||||
if not chars:
|
||||
return []
|
||||
heights = [c["y1"] - c["y0"] for c in chars if c["y1"] > c["y0"]]
|
||||
med_h = statistics.median(heights) if heights else 1.0
|
||||
ordered = sorted(chars, key=lambda c: -(c["y0"] + c["y1"]) / 2)
|
||||
groups: list = []
|
||||
cur: list = []
|
||||
ref = None
|
||||
for c in ordered:
|
||||
yc = (c["y0"] + c["y1"]) / 2
|
||||
if ref is None or abs(yc - ref) <= 0.5 * med_h:
|
||||
cur.append(c)
|
||||
ref = yc if ref is None else ref
|
||||
else:
|
||||
groups.append(cur)
|
||||
cur = [c]
|
||||
ref = yc
|
||||
if cur:
|
||||
groups.append(cur)
|
||||
|
||||
lines: list = []
|
||||
for grp in groups:
|
||||
grp_sorted = sorted(grp, key=lambda c: c["x0"])
|
||||
text = _join_line_glyphs(grp_sorted)
|
||||
if not text:
|
||||
continue
|
||||
hs = [c["y1"] - c["y0"] for c in grp_sorted if c["y1"] > c["y0"]]
|
||||
lines.append(
|
||||
{
|
||||
"text": text,
|
||||
"h": statistics.median(hs) if hs else med_h,
|
||||
"chars": grp_sorted,
|
||||
"bbox": _chars_bbox(grp_sorted),
|
||||
}
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def _line_has_figure_caption(text: str) -> bool:
|
||||
return bool(_FIGURE_CAPTION_SEARCH_RE.search((text or "").strip()))
|
||||
|
||||
|
||||
def _bbox_above_caption(lines: list, cap_i: int, page_w: float, page_h: float):
|
||||
"""Region above a Figure caption line (PDF coords, bottom-left origin)."""
|
||||
cap_bbox = lines[cap_i]["bbox"]
|
||||
cap_top = cap_bbox[3]
|
||||
x0, x1 = cap_bbox[0], cap_bbox[2]
|
||||
fig_h = page_h * min(MAX_FIGURE_HEIGHT_RATIO, 0.35)
|
||||
y_bottom = cap_top
|
||||
y_top = min(page_h, cap_top + fig_h)
|
||||
|
||||
for j in range(cap_i - 1, -1, -1):
|
||||
t = lines[j]["text"]
|
||||
b = lines[j]["bbox"]
|
||||
if b[3] < y_bottom - 4:
|
||||
continue
|
||||
if b[1] > y_top + 4:
|
||||
break
|
||||
if _is_body_paragraph_line(t) and not _is_figure_interior_line(t):
|
||||
break
|
||||
if _is_figure_interior_line(t) or _is_chart_debris_line(t) or not t.strip():
|
||||
x0 = min(x0, b[0])
|
||||
x1 = max(x1, b[2])
|
||||
y_top = max(y_top, min(page_h, b[3] + fig_h * 0.15))
|
||||
|
||||
min_h = page_h * 0.08
|
||||
if y_top - y_bottom < min_h:
|
||||
y_top = min(page_h, y_bottom + min_h)
|
||||
margin_x = max(8.0, (x1 - x0) * 0.05)
|
||||
return (
|
||||
max(0.0, x0 - margin_x),
|
||||
y_bottom,
|
||||
min(page_w, x1 + margin_x),
|
||||
y_top,
|
||||
)
|
||||
|
||||
|
||||
def _cap_bbox_height(bbox, page_h: float, cap_y_top: float) -> tuple:
|
||||
"""Limit figure bbox height (PDF coords, bottom-left origin)."""
|
||||
x0, y0, x1, y1 = bbox
|
||||
max_top = min(y1, cap_y_top + page_h * MAX_FIGURE_HEIGHT_RATIO)
|
||||
if max_top <= y0:
|
||||
return bbox
|
||||
return (x0, y0, x1, max_top)
|
||||
|
||||
|
||||
def _inject_figure_markdown_before_captions(text: str, clips: list) -> str:
|
||||
"""Place ``![...]()`` immediately before each Figure caption line in page text."""
|
||||
if not clips:
|
||||
return text
|
||||
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
||||
clip_idx = 0
|
||||
for i, ln in enumerate(lines):
|
||||
if clip_idx >= len(clips):
|
||||
break
|
||||
if not _line_has_figure_caption(ln):
|
||||
continue
|
||||
if i > 0 and lines[i - 1].lstrip().startswith("!["):
|
||||
continue
|
||||
ref_path = clips[clip_idx][0]
|
||||
fname = os.path.basename(ref_path)
|
||||
img_md = f""
|
||||
lines[i] = f"{img_md}\n\n{ln}"
|
||||
clip_idx += 1
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _extract_vector_figure_clips(
|
||||
page,
|
||||
page_index: int,
|
||||
plain_text: str,
|
||||
raw,
|
||||
base_name: str,
|
||||
scale: float,
|
||||
quality: int,
|
||||
max_edge: int,
|
||||
) -> list:
|
||||
"""Render vector figure regions anchored at each ``Figure N.`` caption on the page.
|
||||
|
||||
Returns ``[(ref_path, b64, y_sort, caption_line), ...]`` for markdown injection.
|
||||
"""
|
||||
if not RENDER_VECTOR_FIGURES or not re.search(r"\bFigure\s+\d+", plain_text, re.I):
|
||||
return []
|
||||
textpage = None
|
||||
try:
|
||||
textpage = page.get_textpage()
|
||||
chars, page_w = _page_chars(textpage, page, raw)
|
||||
if not chars:
|
||||
return []
|
||||
page_h = page.get_size()[1]
|
||||
lines = _merge_orphan_punctuation_lines(_group_lines_with_chars(chars))
|
||||
caption_indices = [
|
||||
i for i, ln in enumerate(lines) if _line_has_figure_caption(ln["text"])
|
||||
]
|
||||
if not caption_indices:
|
||||
return []
|
||||
|
||||
results: list = []
|
||||
for fig_idx, cap_i in enumerate(caption_indices):
|
||||
cap_line = lines[cap_i]["text"].strip()
|
||||
m = _FIGURE_CAPTION_SEARCH_RE.search(cap_line)
|
||||
if m:
|
||||
cap_line = cap_line[m.start() :].split("\n", 1)[0].strip()
|
||||
|
||||
bbox = _bbox_above_caption(lines, cap_i, page_w, page_h)
|
||||
if bbox is None:
|
||||
bbox = _chart_region_bbox(chars, page_w, page_h)
|
||||
if bbox is None:
|
||||
continue
|
||||
|
||||
ratio = _bbox_area_ratio(bbox, page_w, page_h)
|
||||
if ratio > MAX_CHART_REGION_AREA_RATIO:
|
||||
bbox = _cap_bbox_height(bbox, page_h, lines[cap_i]["bbox"][3])
|
||||
ratio = _bbox_area_ratio(bbox, page_w, page_h)
|
||||
if ratio > MAX_CHART_REGION_AREA_RATIO:
|
||||
continue
|
||||
if ratio < MIN_CHART_REGION_AREA_RATIO:
|
||||
continue
|
||||
|
||||
bbox = _expand_chart_bbox(bbox, page_w, page_h, margin_frac=0.06)
|
||||
jpeg = _render_page_clip_jpeg(page, bbox, scale, quality, max_edge)
|
||||
fname = f"{base_name}_p{page_index + 1}_fig{fig_idx + 1}.jpg"
|
||||
ref_path = f"images/{fname}"
|
||||
results.append(
|
||||
(
|
||||
ref_path,
|
||||
base64.b64encode(jpeg).decode("utf-8"),
|
||||
bbox[3],
|
||||
cap_line,
|
||||
)
|
||||
)
|
||||
return results
|
||||
except Exception:
|
||||
logger.debug("vector figure clip failed on page %d", page_index, exc_info=True)
|
||||
return []
|
||||
finally:
|
||||
_close_pdfium_resource(textpage)
|
||||
|
||||
|
||||
def _collect_invisible_boxes(page, raw) -> list:
|
||||
"""Bounding boxes of invisible (render-mode 3) text objects on the page."""
|
||||
boxes: list = []
|
||||
@@ -251,6 +679,109 @@ def _split_columns(chars: list, scale: float, width: float, depth: int = 0) -> l
|
||||
)
|
||||
|
||||
|
||||
def _column_x_span(chars: list) -> float:
|
||||
if not chars:
|
||||
return 0.0
|
||||
return max(c["x1"] for c in chars) - min(c["x0"] for c in chars)
|
||||
|
||||
|
||||
def _column_single_line_fraction(lines: list) -> float:
|
||||
if not lines:
|
||||
return 0.0
|
||||
single = sum(1 for ln in lines if len(ln["text"]) <= 2)
|
||||
return single / len(lines)
|
||||
|
||||
|
||||
def _is_artifact_column(chars: list, width: float) -> bool:
|
||||
"""Detect margin strips and vertical watermarks (e.g. arXiv sidebar).
|
||||
|
||||
Docling / MinerU solve this with learned layout regions; here we use
|
||||
geometry only: a narrow column whose lines are mostly one glyph tall is not
|
||||
part of the reading order.
|
||||
"""
|
||||
if not chars or width <= 0:
|
||||
return True
|
||||
span = _column_x_span(chars)
|
||||
if span <= 0:
|
||||
return True
|
||||
lines = _group_lines(chars)
|
||||
single_frac = _column_single_line_fraction(lines)
|
||||
narrow = span / width < MARGIN_COL_WIDTH_RATIO
|
||||
if narrow and single_frac >= 0.45:
|
||||
return True
|
||||
ys = [(c["y0"] + c["y1"]) / 2 for c in chars]
|
||||
y_span = max(ys) - min(ys)
|
||||
# Vertical text: tall stack, narrow horizontal extent, mostly one char/line.
|
||||
if y_span > span * 3.5 and len(chars) >= 8 and single_frac >= 0.35:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _filter_reading_columns(chars: list, scale: float, width: float) -> list:
|
||||
"""Split into columns and drop margin / watermark strips."""
|
||||
cols = _split_columns(chars, scale, width)
|
||||
kept = [c for c in cols if not _is_artifact_column(c, width)]
|
||||
if kept:
|
||||
return kept
|
||||
# All columns looked like noise — keep the widest glyph set (main body).
|
||||
if len(cols) > 1:
|
||||
return [max(cols, key=_column_x_span)]
|
||||
return cols
|
||||
|
||||
|
||||
def _merge_orphan_punctuation_lines(lines: list) -> list:
|
||||
"""Attach lines that are only punctuation to the previous visual line.
|
||||
|
||||
Many PDFs place ``.`` in figure labels or footnotes on a slightly different
|
||||
baseline; grouping by y then leaves ``Figure 1`` and ``2:`` on separate lines.
|
||||
"""
|
||||
if not lines:
|
||||
return []
|
||||
merged: list = []
|
||||
for ln in lines:
|
||||
t = ln["text"].strip()
|
||||
if (
|
||||
merged
|
||||
and t
|
||||
and len(t) <= 4
|
||||
and all(c in ".,;:!?…·" or c.isspace() for c in t)
|
||||
):
|
||||
suffix = "".join(t.split())
|
||||
prev = merged[-1]["text"]
|
||||
if suffix and prev and not prev.endswith((" ", "-")):
|
||||
merged[-1]["text"] = prev + suffix
|
||||
else:
|
||||
merged[-1]["text"] = (prev + " " + t).strip()
|
||||
continue
|
||||
merged.append(dict(ln))
|
||||
return merged
|
||||
|
||||
|
||||
def _join_line_glyphs(ln_sorted: list) -> str:
|
||||
"""Join a visual line's glyphs, inferring word spaces from horizontal gaps."""
|
||||
if not ln_sorted:
|
||||
return ""
|
||||
widths = [c["x1"] - c["x0"] for c in ln_sorted if c["x1"] > c["x0"]]
|
||||
med_w = statistics.median(widths) if widths else 1.0
|
||||
gap_threshold = med_w * WORD_GAP_WIDTH_RATIO
|
||||
|
||||
parts: list[str] = []
|
||||
for i, cur in enumerate(ln_sorted):
|
||||
ch = cur["ch"]
|
||||
if i == 0:
|
||||
parts.append(ch)
|
||||
continue
|
||||
prev = ln_sorted[i - 1]
|
||||
if ch.isspace() or prev["ch"].isspace():
|
||||
if not ch.isspace() or (parts and not parts[-1].endswith(" ")):
|
||||
parts.append(ch)
|
||||
continue
|
||||
if cur["x0"] - prev["x1"] > gap_threshold:
|
||||
parts.append(" ")
|
||||
parts.append(ch)
|
||||
return "".join(parts).strip()
|
||||
|
||||
|
||||
def _group_lines(chars: list) -> list:
|
||||
"""Group a column's glyphs into lines (top-to-bottom, glyphs sorted by x)."""
|
||||
if not chars:
|
||||
@@ -277,7 +808,7 @@ def _group_lines(chars: list) -> list:
|
||||
out: list = []
|
||||
for ln in lines:
|
||||
ln_sorted = sorted(ln, key=lambda c: c["x0"])
|
||||
text = "".join(c["ch"] for c in ln_sorted).strip()
|
||||
text = _join_line_glyphs(ln_sorted)
|
||||
if not text:
|
||||
continue
|
||||
hs = [c["y1"] - c["y0"] for c in ln_sorted if c["y1"] - c["y0"] > 0]
|
||||
@@ -293,7 +824,12 @@ def _segments_to_markdown(lines: list) -> str:
|
||||
|
||||
def level(ln) -> int:
|
||||
txt = ln["text"]
|
||||
if not DETECT_HEADINGS or body <= 0 or len(txt) > 80:
|
||||
if (
|
||||
not DETECT_HEADINGS
|
||||
or body <= 0
|
||||
or len(txt) > 80
|
||||
or len(txt) < MIN_HEADING_LINE_CHARS
|
||||
):
|
||||
return 0
|
||||
if txt[-1:] in ".。!!??,,;;::":
|
||||
return 0
|
||||
@@ -317,6 +853,100 @@ def _segments_to_markdown(lines: list) -> str:
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _chars_to_layout_markdown(chars: list, scale: float, width: float) -> str:
|
||||
blocks: list = []
|
||||
for col in _filter_reading_columns(chars, scale, width):
|
||||
lines = _merge_orphan_punctuation_lines(_group_lines(col))
|
||||
md = _segments_to_markdown(lines)
|
||||
if md:
|
||||
blocks.append(md)
|
||||
return "\n".join(blocks)
|
||||
|
||||
|
||||
def _layout_line_stats(text: str) -> tuple:
|
||||
"""Return (line_count, single_char_line_count, punct_only_line_count)."""
|
||||
lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
return 0, 0, 0
|
||||
single = sum(1 for ln in lines if len(ln) <= 2)
|
||||
punct_only = sum(
|
||||
1
|
||||
for ln in lines
|
||||
if len(ln) <= 4 and re.fullmatch(r"[\s.,;:!?…·\-–—]+", ln)
|
||||
)
|
||||
return len(lines), single, punct_only
|
||||
|
||||
|
||||
def _layout_garbled_line_fraction(text: str) -> float:
|
||||
"""Share of lines that look like broken OCR (many 1–2 letter tokens)."""
|
||||
lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
return 0.0
|
||||
garbled = 0
|
||||
for ln in lines:
|
||||
words = ln.split()
|
||||
if len(words) >= 6 and sum(1 for w in words if len(w) <= 2) / len(words) > 0.45:
|
||||
garbled += 1
|
||||
return garbled / len(lines)
|
||||
|
||||
|
||||
def _plain_is_well_formed(plain: str) -> bool:
|
||||
"""True when pdfium plain text already has usable words and punctuation.
|
||||
|
||||
Academic PDFs (arXiv) and TOCs already expose a good text layer; running
|
||||
geometric layout on them often destroys citations and words. Scanned books
|
||||
with a poor text layer (no commas in refs, short glued tokens) still need
|
||||
layout gap inference.
|
||||
"""
|
||||
plain = (plain or "").strip()
|
||||
if not plain:
|
||||
return False
|
||||
if re.search(r"\[\w+,\s", plain):
|
||||
return True
|
||||
if plain.count(" . . ") >= 2:
|
||||
return True
|
||||
words = re.findall(r"\S+", plain)
|
||||
if len(words) < 30:
|
||||
return False
|
||||
avg_len = sum(len(w) for w in words) / len(words)
|
||||
return avg_len >= 5.0
|
||||
|
||||
|
||||
def _should_prefer_plain(plain: str, layout: str) -> bool:
|
||||
"""Fall back to pdfium plain text when layout reconstruction looks broken."""
|
||||
layout = (layout or "").strip()
|
||||
plain = (plain or "").strip()
|
||||
if not layout:
|
||||
return True
|
||||
if not plain:
|
||||
return False
|
||||
n, single, punct_only = _layout_line_stats(layout)
|
||||
if n == 0:
|
||||
return True
|
||||
if single / n >= 0.18 or punct_only / n >= 0.12:
|
||||
return True
|
||||
garbled = _layout_garbled_line_fraction(layout)
|
||||
if garbled >= 0.20 and _layout_garbled_line_fraction(plain) < 0.08:
|
||||
return True
|
||||
if re.search(r"\[\w+,\s", plain) and re.search(
|
||||
r"\[\w+\s+\w+\s+\d", layout
|
||||
):
|
||||
return True
|
||||
# Title / lead sentence from plain should survive in layout.
|
||||
for ln in plain.splitlines():
|
||||
probe = ln.strip()
|
||||
if len(probe) < 24:
|
||||
continue
|
||||
alnum = "".join(c for c in probe if c.isalnum())[:16]
|
||||
if len(alnum) < 12:
|
||||
continue
|
||||
layout_alnum = "".join(c for c in layout if c.isalnum())
|
||||
if alnum not in layout_alnum:
|
||||
return True
|
||||
break
|
||||
return False
|
||||
|
||||
|
||||
def _extract_layout_text(page, raw) -> str:
|
||||
"""Layout-aware extraction: reading order + headings + hidden-text filter.
|
||||
|
||||
@@ -331,12 +961,7 @@ def _extract_layout_text(page, raw) -> str:
|
||||
return ""
|
||||
heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]
|
||||
scale = (statistics.median(heights) if heights else 1.0) or 1.0
|
||||
blocks = []
|
||||
for col in _split_columns(chars, scale, width):
|
||||
md = _segments_to_markdown(_group_lines(col))
|
||||
if md:
|
||||
blocks.append(md)
|
||||
return "\n".join(blocks)
|
||||
return _chars_to_layout_markdown(chars, scale, width)
|
||||
except Exception:
|
||||
logger.debug("layout extraction failed; using plain text", exc_info=True)
|
||||
return _extract_page_text(page)
|
||||
@@ -623,37 +1248,6 @@ def _extract_embedded_images(pdf, classes, raw, base_name: str, quality: int) ->
|
||||
return result
|
||||
|
||||
|
||||
def estimate_scanned_fraction(content: bytes, sample: int = 12) -> float:
|
||||
"""Return the fraction of (sampled) pages that look image-dominated.
|
||||
|
||||
Used by alternative engines (e.g. liteparse) that lack image-object access
|
||||
to decide whether a PDF is scanned, applying the same image-area signal the
|
||||
builtin router uses. Samples up to ``sample`` pages for speed on big PDFs.
|
||||
"""
|
||||
import pypdfium2 as pdfium
|
||||
import pypdfium2.raw as pdfium_r
|
||||
|
||||
pdf = pdfium.PdfDocument(content)
|
||||
try:
|
||||
page_count = len(pdf)
|
||||
if page_count <= 0:
|
||||
return 0.0
|
||||
step = max(1, page_count // sample)
|
||||
indices = list(range(0, page_count, step))
|
||||
scanned = 0
|
||||
for i in indices:
|
||||
page = pdf[i]
|
||||
try:
|
||||
ratio = _page_image_area_ratio(page, pdfium_r)
|
||||
finally:
|
||||
_close_pdfium_resource(page)
|
||||
if ratio >= SCAN_IMAGE_AREA_RATIO:
|
||||
scanned += 1
|
||||
return scanned / len(indices) if indices else 0.0
|
||||
finally:
|
||||
_close_pdfium_resource(pdf)
|
||||
|
||||
|
||||
def _strip_repeating_lines(texts: list, classes: list) -> list:
|
||||
"""Remove running headers/footers that repeat across most text pages.
|
||||
|
||||
@@ -791,6 +1385,7 @@ class PDFParser(BaseParser):
|
||||
# Pass 1: cheap text extraction + image-area classification.
|
||||
texts: list = []
|
||||
classes: list = []
|
||||
vector_clips: dict = {}
|
||||
for i in range(page_count):
|
||||
page = pdf[i]
|
||||
try:
|
||||
@@ -800,9 +1395,36 @@ class PDFParser(BaseParser):
|
||||
# Layout reconstruction only pays off (and is only spent) on
|
||||
# native text pages; scanned pages are rendered, not read.
|
||||
if cls == "text" and LAYOUT_ORDERING:
|
||||
text = _extract_layout_text(page, pdfium_r) or plain
|
||||
if _plain_is_well_formed(plain):
|
||||
text = plain
|
||||
else:
|
||||
layout = _extract_layout_text(page, pdfium_r)
|
||||
if layout and not _should_prefer_plain(plain, layout):
|
||||
text = layout
|
||||
else:
|
||||
text = plain
|
||||
else:
|
||||
text = plain
|
||||
if cls == "text":
|
||||
clips = _extract_vector_figure_clips(
|
||||
page,
|
||||
i,
|
||||
plain,
|
||||
pdfium_r,
|
||||
base_name,
|
||||
scale,
|
||||
quality,
|
||||
CONFIG.pdf_render_max_edge,
|
||||
)
|
||||
if clips:
|
||||
vector_clips[i] = clips
|
||||
for ref_path, b64, _y, _cap in clips:
|
||||
images[ref_path] = b64
|
||||
text = _postprocess_pdf_text(text)
|
||||
if cls == "text" and vector_clips.get(i):
|
||||
text = _inject_figure_markdown_before_captions(
|
||||
text, vector_clips[i]
|
||||
)
|
||||
finally:
|
||||
_close_pdfium_resource(page)
|
||||
texts.append(text)
|
||||
@@ -841,6 +1463,7 @@ class PDFParser(BaseParser):
|
||||
|
||||
# Assemble markdown in reading order.
|
||||
embedded_count = 0
|
||||
vector_figure_count = 0
|
||||
blocks = []
|
||||
for i in range(page_count):
|
||||
if classes[i] == "scanned":
|
||||
@@ -850,7 +1473,10 @@ class PDFParser(BaseParser):
|
||||
stripped = texts[i].strip()
|
||||
if stripped:
|
||||
blocks.append(stripped)
|
||||
for ref_path, _b64, _y in embedded.get(i, []):
|
||||
vector_figure_count += len(vector_clips.get(i, []))
|
||||
page_images = list(embedded.get(i, []))
|
||||
page_images.sort(key=lambda item: item[2], reverse=True)
|
||||
for ref_path, _b64, _y in page_images:
|
||||
fname = os.path.basename(ref_path)
|
||||
blocks.append(f"")
|
||||
embedded_count += 1
|
||||
@@ -862,6 +1488,7 @@ class PDFParser(BaseParser):
|
||||
"scanned_page_count": len(scanned_indices),
|
||||
"text_page_count": page_count - len(scanned_indices),
|
||||
"embedded_image_count": embedded_count,
|
||||
"vector_figure_count": vector_figure_count,
|
||||
"image_source_type": "scanned_pdf" if scanned_indices else "pdf_text_layer",
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""LibreOffice helpers for legacy binary PowerPoint (.ppt) uploads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from docreader.parser.excel_convert import find_soffice
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
|
||||
_ZIP_MAGIC = b"PK\x03\x04"
|
||||
|
||||
|
||||
def is_ole_compound(content: bytes) -> bool:
|
||||
return len(content) >= len(_OLE_MAGIC) and content.startswith(_OLE_MAGIC)
|
||||
|
||||
|
||||
def is_zip_openxml(content: bytes) -> bool:
|
||||
return len(content) >= len(_ZIP_MAGIC) and content.startswith(_ZIP_MAGIC)
|
||||
|
||||
|
||||
def needs_ppt_to_pptx_conversion(content: bytes, file_type: str | None) -> bool:
|
||||
"""True when content is legacy .ppt (OLE), not modern .pptx (ZIP)."""
|
||||
ext = (file_type or "").lstrip(".").lower()
|
||||
if ext == "pptx" or is_zip_openxml(content):
|
||||
return False
|
||||
if ext == "ppt" or is_ole_compound(content):
|
||||
return is_ole_compound(content) or ext == "ppt"
|
||||
return False
|
||||
|
||||
|
||||
def convert_ppt_to_pptx_bytes(content: bytes, suffix: str = ".ppt") -> bytes | None:
|
||||
"""Convert legacy PowerPoint bytes to PPTX using LibreOffice, if available."""
|
||||
soffice = find_soffice()
|
||||
if not soffice:
|
||||
return None
|
||||
|
||||
max_attempts = 3
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
with tempfile.TemporaryDirectory() as temp_dir, tempfile.TemporaryDirectory() as profile_dir:
|
||||
src = os.path.join(temp_dir, f"input{suffix}")
|
||||
with open(src, "wb") as handle:
|
||||
handle.write(content)
|
||||
|
||||
user_installation = Path(profile_dir).as_uri()
|
||||
cmd = [
|
||||
soffice,
|
||||
"--headless",
|
||||
f"-env:UserInstallation={user_installation}",
|
||||
"--convert-to",
|
||||
"pptx",
|
||||
"--outdir",
|
||||
temp_dir,
|
||||
src,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, timeout=120)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
logger.warning("LibreOffice PPT convert failed to start: %s", exc)
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.decode("utf-8", errors="ignore")
|
||||
logger.warning(
|
||||
"LibreOffice PPT convert failed (attempt %s/%s): %s",
|
||||
attempt,
|
||||
max_attempts,
|
||||
stderr,
|
||||
)
|
||||
if attempt < max_attempts:
|
||||
time.sleep(0.5 * attempt)
|
||||
continue
|
||||
return None
|
||||
|
||||
for name in os.listdir(temp_dir):
|
||||
if name.endswith(".pptx"):
|
||||
with open(os.path.join(temp_dir, name), "rb") as handle:
|
||||
converted = handle.read()
|
||||
logger.info(
|
||||
"Converted presentation via LibreOffice (%s -> pptx, %d bytes)",
|
||||
suffix,
|
||||
len(converted),
|
||||
)
|
||||
return converted
|
||||
|
||||
if attempt < max_attempts:
|
||||
time.sleep(0.5 * attempt)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_ppt_bytes(content: bytes, file_type: str | None) -> tuple[bytes, str]:
|
||||
"""Return (bytes, extension) suitable for MarkItDown (pptx when converted)."""
|
||||
ext = (file_type or "").lstrip(".").lower()
|
||||
|
||||
if is_zip_openxml(content):
|
||||
return content, ".pptx"
|
||||
|
||||
if not needs_ppt_to_pptx_conversion(content, ext):
|
||||
dotted = f".{ext}" if ext else ".pptx"
|
||||
return content, dotted
|
||||
|
||||
suffix = ".ppt" if ext in ("", "ppt") else f".{ext}"
|
||||
converted = convert_ppt_to_pptx_bytes(content, suffix=suffix)
|
||||
if converted:
|
||||
return converted, ".pptx"
|
||||
|
||||
raise ValueError(
|
||||
"Legacy PowerPoint (.ppt) is not supported by MarkItDown directly; "
|
||||
"LibreOffice is required to convert it to .pptx. Install LibreOffice "
|
||||
"(soffice) in the docreader environment or upload .pptx instead."
|
||||
)
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Extract and rasterize images embedded in PPTX (e.g. WMF) when MarkItDown cannot inline them."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import uuid
|
||||
import zipfile
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MARKDOWN_IMAGE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
_RASTER_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
|
||||
_VECTOR_EXT = {".wmf", ".emf", ".svg"}
|
||||
|
||||
|
||||
def _find_convert() -> str | None:
|
||||
for path in ("/usr/bin/convert", "/usr/local/bin/convert"):
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "convert"], capture_output=True, text=True, check=False
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip()
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _rasterize_with_imagemagick(data: bytes, suffix: str) -> bytes | None:
|
||||
convert = _find_convert()
|
||||
if not convert:
|
||||
return None
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
src = os.path.join(temp_dir, f"input{suffix}")
|
||||
dst = os.path.join(temp_dir, "output.png")
|
||||
with open(src, "wb") as handle:
|
||||
handle.write(data)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[convert, src, dst],
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
logger.warning("ImageMagick convert failed: %s", exc)
|
||||
return None
|
||||
if result.returncode != 0 or not os.path.isfile(dst):
|
||||
stderr = (result.stderr or b"").decode("utf-8", errors="ignore")
|
||||
logger.warning("ImageMagick convert exit %s: %s", result.returncode, stderr)
|
||||
return None
|
||||
with open(dst, "rb") as handle:
|
||||
return handle.read()
|
||||
|
||||
|
||||
def _rasterize_with_pillow(data: bytes) -> bytes | None:
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
if img.mode not in ("RGB", "L"):
|
||||
img = img.convert("RGB")
|
||||
out = io.BytesIO()
|
||||
img.save(out, format="PNG")
|
||||
return out.getvalue()
|
||||
except Exception as exc:
|
||||
logger.debug("Pillow could not open media bytes: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def rasterize_media_bytes(name: str, data: bytes) -> bytes | None:
|
||||
ext = os.path.splitext(name)[1].lower()
|
||||
if ext in _RASTER_EXT:
|
||||
png = _rasterize_with_pillow(data)
|
||||
if png:
|
||||
return png
|
||||
if ext in _VECTOR_EXT or ext in _RASTER_EXT:
|
||||
return _rasterize_with_imagemagick(data, ext or ".bin")
|
||||
return _rasterize_with_imagemagick(data, ext or ".bin")
|
||||
|
||||
|
||||
def list_pptx_media(pptx_bytes: bytes) -> List[Tuple[str, bytes]]:
|
||||
"""Return (zip path, raw bytes) for each file under ppt/media/, in archive order."""
|
||||
items: List[Tuple[str, bytes]] = []
|
||||
with zipfile.ZipFile(io.BytesIO(pptx_bytes)) as archive:
|
||||
for name in archive.namelist():
|
||||
if not name.startswith("ppt/media/"):
|
||||
continue
|
||||
base = os.path.basename(name)
|
||||
if not base or base.startswith("."):
|
||||
continue
|
||||
items.append((name, archive.read(name)))
|
||||
return items
|
||||
|
||||
|
||||
def extract_pptx_media_rasterized(pptx_bytes: bytes) -> List[bytes]:
|
||||
"""Rasterize all ppt/media assets to PNG bytes, skipping failures."""
|
||||
rasterized: List[bytes] = []
|
||||
for path, raw in list_pptx_media(pptx_bytes):
|
||||
png = rasterize_media_bytes(os.path.basename(path), raw)
|
||||
if png:
|
||||
rasterized.append(png)
|
||||
logger.info("Rasterized pptx media %s (%d -> %d bytes)", path, len(raw), len(png))
|
||||
else:
|
||||
logger.warning("Failed to rasterize pptx media %s", path)
|
||||
return rasterized
|
||||
|
||||
|
||||
def _is_unresolved_image_ref(url: str) -> bool:
|
||||
if not url or url.startswith("data:") or url.startswith("images/"):
|
||||
return False
|
||||
if url.startswith(("http://", "https://")):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def attach_pptx_media_to_markdown(
|
||||
markdown: str, pptx_bytes: bytes
|
||||
) -> Tuple[str, Dict[str, str]]:
|
||||
"""Replace unresolved  refs with images/ paths and inline image payloads."""
|
||||
media = extract_pptx_media_rasterized(pptx_bytes)
|
||||
if not media:
|
||||
return markdown, {}
|
||||
|
||||
images: Dict[str, str] = {}
|
||||
media_iter = iter(media)
|
||||
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
alt, url = match.group(1), match.group(2)
|
||||
if not _is_unresolved_image_ref(url):
|
||||
return match.group(0)
|
||||
try:
|
||||
png = next(media_iter)
|
||||
except StopIteration:
|
||||
return match.group(0)
|
||||
ref = f"images/{uuid.uuid4()}.png"
|
||||
images[ref] = base64.b64encode(png).decode()
|
||||
return f""
|
||||
|
||||
return _MARKDOWN_IMAGE.sub(repl, markdown), images
|
||||
|
||||
|
||||
def markdown_needs_pptx_media_attach(markdown: str) -> bool:
|
||||
return any(_is_unresolved_image_ref(m.group(2)) for m in _MARKDOWN_IMAGE.finditer(markdown))
|
||||
@@ -7,8 +7,11 @@ from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.liteparse_parser import LiteParseParser, liteparse_available
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.opendataloader_parser import (
|
||||
OpenDataLoaderParser,
|
||||
opendataloader_available,
|
||||
)
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -151,11 +154,11 @@ def _build_default_registry() -> ParserEngineRegistry:
|
||||
)
|
||||
|
||||
reg.register(
|
||||
"liteparse",
|
||||
{"pdf": LiteParseParser},
|
||||
description="LiteParse 解析引擎(快速空间阅读顺序,适合数字版 PDF)",
|
||||
check_available=liteparse_available,
|
||||
unavailable_hint="liteparse 未安装",
|
||||
"opendataloader",
|
||||
{"pdf": OpenDataLoaderParser},
|
||||
description="OpenDataLoader PDF(版面分析,需 Java 11+)",
|
||||
check_available=opendataloader_available,
|
||||
unavailable_hint="请安装 opendataloader-pdf 与 Java 11+",
|
||||
)
|
||||
|
||||
# NOTE: Engine listing is managed by Go-side engine registry
|
||||
|
||||
+137
-29
@@ -1,9 +1,11 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from lxml.etree import XPath
|
||||
from playwright.async_api import async_playwright
|
||||
from playwright.async_api import Page, async_playwright
|
||||
from trafilatura import extract, utils, xpaths
|
||||
|
||||
from docreader.config import CONFIG
|
||||
@@ -15,6 +17,14 @@ from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GOTO_TIMEOUT_MS = 30_000
|
||||
_NETWORK_IDLE_TIMEOUT_MS = 10_000
|
||||
_SPA_WAIT_TIMEOUT_MS = 15_000
|
||||
# Minimum visible characters before treating an SPA shell as "rendered".
|
||||
_SPA_MIN_TEXT_LEN = 80
|
||||
# Minimum visible characters for Playwright text fallback when trafilatura fails.
|
||||
_MIN_FALLBACK_TEXT_LEN = 50
|
||||
|
||||
# Monkey-patch trafilatura internals to better support WeChat Official Account
|
||||
# articles, whose images live on `mmbiz.qpic.cn` without a standard file
|
||||
# extension and whose main content sits inside `#js_content` /
|
||||
@@ -40,6 +50,78 @@ except (AttributeError, ImportError) as e:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ScrapeResult:
|
||||
html: str
|
||||
visible_text: str
|
||||
page_title: str
|
||||
|
||||
|
||||
def extract_markdown_from_html(html: str) -> Optional[str]:
|
||||
"""Run trafilatura on HTML; return markdown or None if nothing extracted."""
|
||||
if not html or not html.strip():
|
||||
return None
|
||||
md_text = extract(
|
||||
html,
|
||||
output_format="markdown",
|
||||
with_metadata=True,
|
||||
include_images=True,
|
||||
include_tables=True,
|
||||
include_links=True,
|
||||
)
|
||||
if not md_text or not md_text.strip():
|
||||
return None
|
||||
return md_text
|
||||
|
||||
|
||||
def build_visible_text_fallback(visible_text: str, page_title: str = "") -> Optional[str]:
|
||||
"""Build markdown from Playwright-visible text when trafilatura finds no article body."""
|
||||
text = (visible_text or "").strip()
|
||||
if len(text) < _MIN_FALLBACK_TEXT_LEN:
|
||||
return None
|
||||
title = (page_title or "").strip()
|
||||
if title and not text.startswith(title):
|
||||
return f"# {title}\n\n{text}"
|
||||
return text
|
||||
|
||||
|
||||
async def wait_for_rendered_content(page: Page) -> None:
|
||||
"""Wait for SPA/JS pages beyond the initial HTML shell."""
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=_NETWORK_IDLE_TIMEOUT_MS)
|
||||
logger.info("Network idle after navigation")
|
||||
except Exception:
|
||||
logger.info("Network idle wait timed out, continuing")
|
||||
|
||||
try:
|
||||
await page.wait_for_function(
|
||||
"""(minLen) => {
|
||||
const root = document.querySelector('#app')
|
||||
|| document.querySelector('main')
|
||||
|| document.body;
|
||||
return ((root?.innerText || '').trim().length >= minLen);
|
||||
}""",
|
||||
arg=_SPA_MIN_TEXT_LEN,
|
||||
timeout=_SPA_WAIT_TIMEOUT_MS,
|
||||
)
|
||||
logger.info("SPA/root visible text reached minimum length")
|
||||
except Exception:
|
||||
logger.info("SPA text wait timed out, using current DOM")
|
||||
|
||||
|
||||
async def read_visible_text(page: Page) -> str:
|
||||
"""Prefer #app/main innerText, then fall back to body."""
|
||||
return await page.evaluate(
|
||||
"""() => {
|
||||
const root = document.querySelector('#app')
|
||||
|| document.querySelector('main')
|
||||
|| document.querySelector('[role="main"]')
|
||||
|| document.body;
|
||||
return (root?.innerText || '').trim();
|
||||
}"""
|
||||
)
|
||||
|
||||
|
||||
class StdWebParser(BaseParser):
|
||||
"""Standard web page parser using Playwright and Trafilatura.
|
||||
|
||||
@@ -61,16 +143,17 @@ class StdWebParser(BaseParser):
|
||||
super().__init__(file_name=title, **kwargs)
|
||||
logger.info(f"Initialized WebParser with title: {title}")
|
||||
|
||||
async def scrape(self, url: str) -> str:
|
||||
async def scrape(self, url: str) -> _ScrapeResult:
|
||||
"""Scrape web page content using Playwright.
|
||||
|
||||
Args:
|
||||
url: The URL of the web page to scrape
|
||||
|
||||
Returns:
|
||||
HTML content of the web page as string, empty string on error
|
||||
HTML, visible text, and document title; empty fields on hard failure
|
||||
"""
|
||||
logger.info(f"Starting web page scraping for URL: {url}")
|
||||
empty = _ScrapeResult(html="", visible_text="", page_title="")
|
||||
try:
|
||||
async with async_playwright() as p:
|
||||
kwargs = {}
|
||||
@@ -83,30 +166,42 @@ class StdWebParser(BaseParser):
|
||||
|
||||
logger.info(f"Navigating to URL: {url}")
|
||||
try:
|
||||
# Navigate to URL with 30 second timeout
|
||||
await page.goto(url, timeout=30000)
|
||||
await page.goto(
|
||||
url,
|
||||
timeout=_GOTO_TIMEOUT_MS,
|
||||
wait_until="domcontentloaded",
|
||||
)
|
||||
logger.info("Initial page load complete")
|
||||
except Exception as e:
|
||||
logger.error(f"Error navigating to URL: {str(e)}")
|
||||
await browser.close()
|
||||
return ""
|
||||
return empty
|
||||
|
||||
logger.info("Retrieving page HTML content")
|
||||
# Get the full HTML content of the page
|
||||
await wait_for_rendered_content(page)
|
||||
|
||||
page_title = await page.title()
|
||||
visible_text = await read_visible_text(page)
|
||||
content = await page.content()
|
||||
logger.info(f"Retrieved {len(content)} bytes of HTML content")
|
||||
logger.info(
|
||||
"Retrieved %d bytes HTML, %d chars visible text, title=%r",
|
||||
len(content),
|
||||
len(visible_text),
|
||||
page_title[:80] if page_title else "",
|
||||
)
|
||||
|
||||
await browser.close()
|
||||
logger.info("Browser closed")
|
||||
|
||||
# Return raw HTML content for further processing
|
||||
logger.info("Successfully retrieved HTML content")
|
||||
return content
|
||||
return _ScrapeResult(
|
||||
html=content,
|
||||
visible_text=visible_text,
|
||||
page_title=page_title or "",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to scrape web page: {str(e)}")
|
||||
# Return empty string on error
|
||||
return ""
|
||||
return empty
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse web page content into a Document object.
|
||||
@@ -117,36 +212,49 @@ class StdWebParser(BaseParser):
|
||||
Returns:
|
||||
Document object containing the parsed markdown content
|
||||
"""
|
||||
# Decode bytes to get the URL string
|
||||
url = endecode.decode_bytes(content)
|
||||
|
||||
logger.info(f"Scraping web page: {url}")
|
||||
# Run async scraping in sync context
|
||||
chtml = asyncio.run(self.scrape(url))
|
||||
# Extract clean content from HTML using Trafilatura
|
||||
# Convert to markdown format with metadata, images, tables, and links
|
||||
md_text = extract(
|
||||
chtml,
|
||||
output_format="markdown",
|
||||
with_metadata=True,
|
||||
include_images=True,
|
||||
include_tables=True,
|
||||
include_links=True,
|
||||
)
|
||||
scrape_result = asyncio.run(self.scrape(url))
|
||||
if not scrape_result.html and not scrape_result.visible_text:
|
||||
logger.error("Failed to scrape web page (no HTML or visible text)")
|
||||
return Document(content=f"Error parsing web page: {url}")
|
||||
|
||||
md_text = extract_markdown_from_html(scrape_result.html)
|
||||
if not md_text:
|
||||
md_text = build_visible_text_fallback(
|
||||
scrape_result.visible_text,
|
||||
scrape_result.page_title,
|
||||
)
|
||||
if md_text:
|
||||
logger.info(
|
||||
"Trafilatura empty; using Playwright visible-text fallback (%d chars)",
|
||||
len(md_text),
|
||||
)
|
||||
|
||||
if not md_text:
|
||||
logger.error("Failed to parse web page")
|
||||
return Document(content=f"Error parsing web page: {url}")
|
||||
|
||||
# Extract title from trafilatura metadata output (e.g. "title: xxx" line)
|
||||
metadata = {}
|
||||
title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
|
||||
if title_match:
|
||||
extracted_title = title_match.group(1).strip()
|
||||
if extracted_title:
|
||||
metadata["title"] = extracted_title
|
||||
logger.info(f"Extracted article title from trafilatura: {extracted_title}")
|
||||
logger.info(
|
||||
f"Extracted article title from trafilatura: {extracted_title}"
|
||||
)
|
||||
elif scrape_result.page_title:
|
||||
metadata["title"] = scrape_result.page_title.strip()
|
||||
logger.info(
|
||||
"Using page title from Playwright: %s", metadata["title"]
|
||||
)
|
||||
else:
|
||||
logger.info(f"No title found in trafilatura output, first 200 chars: {md_text[:200]!r}")
|
||||
logger.info(
|
||||
"No title found in trafilatura output, first 200 chars: %r",
|
||||
md_text[:200],
|
||||
)
|
||||
return Document(content=md_text, metadata=metadata)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Fill merged cell values before pandas reads an XLSX workbook."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fill_merged_cells_xlsx(content: bytes) -> bytes:
|
||||
"""Unmerge ranges and copy the master cell value into every covered cell.
|
||||
|
||||
openpyxl only stores values on the top-left cell of a merge; pandas then
|
||||
sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
|
||||
"""
|
||||
if not zipfile.is_zipfile(BytesIO(content)):
|
||||
return content
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
wb = load_workbook(BytesIO(content), data_only=True)
|
||||
changed = False
|
||||
for ws in wb.worksheets:
|
||||
if not ws.merged_cells.ranges:
|
||||
continue
|
||||
for merge_range in list(ws.merged_cells.ranges):
|
||||
master_value = ws.cell(merge_range.min_row, merge_range.min_col).value
|
||||
ws.unmerge_cells(str(merge_range))
|
||||
for row in range(merge_range.min_row, merge_range.max_row + 1):
|
||||
for col in range(merge_range.min_col, merge_range.max_col + 1):
|
||||
ws.cell(row, col).value = master_value
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
return content
|
||||
|
||||
out = BytesIO()
|
||||
wb.save(out)
|
||||
logger.info("Filled merged cells in XLSX before parse")
|
||||
return out.getvalue()
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Repair common XLSX packaging issues before openpyxl/pandas read."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from typing import Callable, Dict, Iterable, Set
|
||||
|
||||
SST_PART = "xl/sharedStrings.xml"
|
||||
_SST_OVERRIDE_RE = re.compile(
|
||||
r'<Override[^>]*PartName="[^"]*sharedStrings\.xml"[^>]*/>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SST_REL_RE = re.compile(
|
||||
r'<Relationship[^>]*Type="[^"]*sharedStrings"[^>]*/>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def repair_xlsx_bytes(content: bytes) -> bytes | None:
|
||||
"""Return repaired XLSX bytes, or None if no repair was applied.
|
||||
|
||||
Handles workbooks that reference ``xl/sharedStrings.xml`` in package
|
||||
metadata but omit the part (common with some exporters). When worksheets
|
||||
only use inline strings, manifest references are stripped so openpyxl can
|
||||
read the file.
|
||||
"""
|
||||
if not zipfile.is_zipfile(io.BytesIO(content)):
|
||||
return None
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(content), "r") as zin:
|
||||
names = _normalized_names(zin.namelist())
|
||||
sst_path = _find_shared_strings_path(names)
|
||||
if sst_path:
|
||||
if sst_path == SST_PART:
|
||||
return None
|
||||
return _rewrite_zip(
|
||||
zin, lambda files: _rename_shared_strings_part(files, sst_path)
|
||||
)
|
||||
if not _package_references_shared_strings(zin, names):
|
||||
return None
|
||||
if _worksheets_use_shared_string_cells(zin, names):
|
||||
return None
|
||||
return _rewrite_zip(zin, _strip_shared_strings_manifest)
|
||||
|
||||
|
||||
def _normalized_names(namelist: Iterable[str]) -> Set[str]:
|
||||
return {name.replace("\\", "/") for name in namelist}
|
||||
|
||||
|
||||
def _find_shared_strings_path(names: Set[str]) -> str | None:
|
||||
for name in names:
|
||||
if name.lower().endswith("sharedstrings.xml"):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _package_references_shared_strings(
|
||||
zin: zipfile.ZipFile, names: Set[str]
|
||||
) -> bool:
|
||||
content_types = "[Content_Types].xml"
|
||||
if content_types in names:
|
||||
ct = zin.read(content_types).decode("utf-8", errors="replace")
|
||||
if "sharedstrings.xml" in ct.lower():
|
||||
return True
|
||||
|
||||
rels_path = "xl/_rels/workbook.xml.rels"
|
||||
if rels_path in names:
|
||||
rels = zin.read(rels_path).decode("utf-8", errors="replace")
|
||||
if "sharedstrings" in rels.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _worksheets_use_shared_string_cells(
|
||||
zin: zipfile.ZipFile, names: Set[str]
|
||||
) -> bool:
|
||||
for name in names:
|
||||
if not name.startswith("xl/worksheets/") or not name.endswith(".xml"):
|
||||
continue
|
||||
sheet = zin.read(name).decode("utf-8", errors="replace")
|
||||
if re.search(r'\bt="s"', sheet):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _rename_shared_strings_part(
|
||||
files: Dict[str, bytes], source_path: str
|
||||
) -> Dict[str, bytes]:
|
||||
updated = dict(files)
|
||||
updated[SST_PART] = updated.pop(source_path)
|
||||
return updated
|
||||
|
||||
|
||||
def _strip_shared_strings_manifest(files: Dict[str, bytes]) -> Dict[str, bytes]:
|
||||
updated = dict(files)
|
||||
ct_path = "[Content_Types].xml"
|
||||
if ct_path in updated:
|
||||
ct = updated[ct_path].decode("utf-8")
|
||||
ct = _SST_OVERRIDE_RE.sub("", ct)
|
||||
updated[ct_path] = ct.encode("utf-8")
|
||||
|
||||
rels_path = "xl/_rels/workbook.xml.rels"
|
||||
if rels_path in updated:
|
||||
rels = updated[rels_path].decode("utf-8")
|
||||
rels = _SST_REL_RE.sub("", rels)
|
||||
updated[rels_path] = rels.encode("utf-8")
|
||||
return updated
|
||||
|
||||
|
||||
def _rewrite_zip(
|
||||
zin: zipfile.ZipFile,
|
||||
transform: Callable[[Dict[str, bytes]], Dict[str, bytes]],
|
||||
) -> bytes:
|
||||
files: Dict[str, bytes] = {}
|
||||
for info in zin.infolist():
|
||||
name = info.filename.replace("\\", "/")
|
||||
files[name] = zin.read(info.filename)
|
||||
files = transform(files)
|
||||
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
|
||||
for name, data in files.items():
|
||||
zout.writestr(name, data)
|
||||
return out.getvalue()
|
||||
Reference in New Issue
Block a user