feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+210
View File
@@ -0,0 +1,210 @@
import io
import os
import shutil
import subprocess
import tempfile
import unittest
import zipfile
import openpyxl
import pandas as pd
from docreader.parser.excel_convert import detect_excel_format, engine_for_format
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
from docreader.parser.xlsx_repair import repair_xlsx_bytes
def _xlsx_with_phantom_shared_strings() -> bytes:
"""Workbook with inline strings but a dangling sharedStrings manifest entry."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "hello"
ws["B1"] = 42
bio = io.BytesIO()
wb.save(bio)
with tempfile.TemporaryDirectory() as tmpdir:
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
zin.extractall(tmpdir)
ct_path = f"{tmpdir}/[Content_Types].xml"
with open(ct_path, encoding="utf-8") as f:
ct = f.read()
override = (
'<Override PartName="/xl/sharedStrings.xml" '
'ContentType="application/vnd.openxmlformats-officedocument.'
'spreadsheetml.sharedStrings+xml"/>'
)
with open(ct_path, "w", encoding="utf-8") as f:
f.write(ct.replace("</Types>", override + "</Types>"))
out = io.BytesIO()
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
for root, _, files in os.walk(tmpdir):
for name in files:
path = os.path.join(root, name)
arc = os.path.relpath(path, tmpdir)
zout.write(path, arc)
return out.getvalue()
class ExcelFormatDetectionTest(unittest.TestCase):
def test_detect_xlsx_and_engine(self):
wb = openpyxl.Workbook()
bio = io.BytesIO()
wb.save(bio)
content = bio.getvalue()
self.assertEqual(detect_excel_format(content), "xlsx")
self.assertEqual(engine_for_format("xlsx"), "openpyxl")
def test_detect_xls_magic(self):
content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
self.assertEqual(detect_excel_format(content), "xls")
self.assertEqual(engine_for_format("xls"), "xlrd")
def test_open_legacy_xls_bytes_with_xlsx_extension(self):
if not shutil.which("soffice"):
self.skipTest("LibreOffice not available")
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "legacy"
xlsx_bio = io.BytesIO()
wb.save(xlsx_bio)
with tempfile.TemporaryDirectory() as tmpdir:
src = os.path.join(tmpdir, "sheet.xlsx")
with open(src, "wb") as handle:
handle.write(xlsx_bio.getvalue())
subprocess.run(
[
"soffice",
"--headless",
"--convert-to",
"xls",
"--outdir",
tmpdir,
src,
],
check=True,
capture_output=True,
)
xls_path = os.path.join(tmpdir, "sheet.xls")
with open(xls_path, "rb") as handle:
xls_bytes = handle.read()
document = ExcelParser(file_name="fake.xlsx", file_type="xlsx").parse_into_text(
xls_bytes
)
self.assertIn("legacy", document.content)
class XlsxRepairTest(unittest.TestCase):
def test_repair_removes_phantom_shared_strings_reference(self):
broken = _xlsx_with_phantom_shared_strings()
with self.assertRaises(KeyError):
pd.read_excel(io.BytesIO(broken))
repaired = repair_xlsx_bytes(broken)
self.assertIsNotNone(repaired)
df = pd.read_excel(io.BytesIO(repaired), header=None)
self.assertEqual(df.values.tolist(), [["hello", 42]])
def test_repair_skips_when_shared_string_cells_need_table(self):
import xlsxwriter
bio = io.BytesIO()
wb = xlsxwriter.Workbook(bio, {"in_memory": True})
ws = wb.add_worksheet()
ws.write(0, 0, "hello")
wb.close()
with tempfile.TemporaryDirectory() as tmpdir:
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
zin.extractall(tmpdir)
os.remove(f"{tmpdir}/xl/sharedStrings.xml")
out = io.BytesIO()
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
for root, _, files in os.walk(tmpdir):
for name in files:
path = os.path.join(root, name)
arc = os.path.relpath(path, tmpdir)
zout.write(path, arc)
broken = out.getvalue()
self.assertIsNone(repair_xlsx_bytes(broken))
class XlsxMergeFillTest(unittest.TestCase):
def test_fill_merged_cells_propagates_master_value(self):
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "title"
ws.merge_cells("A1:B1")
ws["A2"] = "left"
ws["B2"] = "right"
ws.merge_cells("A2:A3")
ws["B3"] = "only-b"
bio = io.BytesIO()
wb.save(bio)
filled = fill_merged_cells_xlsx(bio.getvalue())
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
out_ws = out_wb.active
self.assertEqual(out_ws["B1"].value, "title")
self.assertEqual(out_ws["A3"].value, "left")
self.assertEqual(out_ws["B3"].value, "only-b")
def test_parse_en_mergecell_workbook(self):
path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"xlsx",
"en_mergecell.xlsx",
)
if not os.path.isfile(path):
self.skipTest("en_mergecell.xlsx fixture not available")
with open(path, "rb") as handle:
document = ExcelParser().parse_into_text(handle.read())
chunks = [chunk.content.strip() for chunk in document.chunks]
self.assertEqual(len(chunks), 12)
self.assertIn("A: A1", chunks[0])
self.assertIn("A: A2", chunks[1])
self.assertIn("B: B3", chunks[2])
self.assertNotIn("Unnamed:", document.content)
self.assertIn("A: A7", chunks[6])
self.assertIn("A: A7", chunks[7])
self.assertIn("D: D10", chunks[9])
class ExcelParserTest(unittest.TestCase):
def test_parse_phantom_shared_strings_workbook(self):
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
self.assertIn("hello", document.content)
self.assertIn("42", document.content)
self.assertGreater(len(document.chunks), 0)
def test_parse_en_calcchain_shared_strings_case(self):
path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"xlsx",
"en_calcchain.xlsx",
)
if not os.path.isfile(path):
self.skipTest("en_calcchain.xlsx fixture not available")
with open(path, "rb") as f:
document = ExcelParser().parse_into_text(f.read())
self.assertGreater(len(document.content), 0)
self.assertGreater(len(document.chunks), 0)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,57 @@
import io
import unittest
from pathlib import Path
from markitdown import MarkItDown
from docreader.parser.markdown_parser import MarkdownTableUtil
class TestMarkdownTableUtil(unittest.TestCase):
def test_preserves_empty_cells(self):
"""Interior empty cells must not be dropped during formatting."""
raw = "| a | | c |\n| --- | --- | --- |\n| 1 | 2 | 3 |"
formatted = MarkdownTableUtil().format_table(raw)
self.assertIn("| a | | c |", formatted)
self.assertEqual(formatted.count("|"), raw.count("|"))
def test_format_nonempty_table(self):
raw = "|Name|Age|\n|---|---|\n|John|30|"
formatted = MarkdownTableUtil().format_table(raw)
self.assertIn("| Name | Age |", formatted)
self.assertIn("| --- | --- |", formatted)
self.assertIn("| John | 30 |", formatted)
def test_normalize_markitdown_en_tables(self):
docx = (
Path(__file__).resolve().parents[2]
/ "testdata"
/ "rag_test"
/ "docx"
/ "en_tables.docx"
)
if not docx.is_file():
docx = Path(__file__).resolve().parents[2].parent / "testdata/rag_test/docx/en_tables.docx"
raw = MarkItDown().convert(io.BytesIO(docx.read_bytes()), file_extension=".docx").text_content
normalized = MarkdownTableUtil().format_table(raw)
self.assertNotIn("| | | | |", normalized)
self.assertIn("| Name | Game | Fame | Blame |", normalized)
idx_name = normalized.index("| Name | Game | Fame | Blame |")
idx_sep = normalized.index("| --- | --- | --- | --- |", idx_name)
self.assertLess(idx_name, idx_sep)
self.assertIn("| Lebron James | Basketball |", normalized)
# Headerless 2-row tables: delimiter inserted so GFM renderers show a table
self.assertIn(
"| Sinple | Table |\n| --- | --- |\n| Without | Header |", normalized
)
self.assertIn(
"| Simple Multiparagraph | Table Full |\n| --- | --- |\n"
"| Of Paragraphs | In each Cell. |",
normalized,
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,103 @@
"""Unit tests for OpenDataLoader parser helpers (no JVM required)."""
import os
import tempfile
import unittest
from unittest import mock
from docreader.parser.opendataloader_parser import (
OpenDataLoaderParser,
_collect_images_under_output,
_find_markdown_file,
_normalize_odl_image_url,
_rewrite_markdown_image_refs,
opendataloader_available,
)
class OpenDataLoaderHelpersTest(unittest.TestCase):
def test_find_markdown_prefers_stem_match(self):
with tempfile.TemporaryDirectory() as d:
other = os.path.join(d, "other.md")
target = os.path.join(d, "paper.md")
with open(other, "w") as f:
f.write("x")
with open(target, "w") as f:
f.write("# Title")
self.assertEqual(_find_markdown_file(d, "paper"), target)
def test_collect_and_rewrite_images(self):
with tempfile.TemporaryDirectory() as d:
img_dir = os.path.join(d, "images")
os.makedirs(img_dir)
png = os.path.join(img_dir, "fig1.png")
with open(png, "wb") as f:
f.write(b"\x89PNG\r\n\x1a\n")
images = _collect_images_under_output(d)
self.assertIn("images/fig1.png", images)
md = "See ![fig](images/fig1.png) and ![alt](./fig1.png)."
out = _rewrite_markdown_image_refs(md, images)
self.assertIn("![fig](images/fig1.png)", out)
self.assertIn("![alt](images/fig1.png)", out)
def test_rewrite_odl_angle_bracket_and_entity_urls(self):
images = {"images/imageFile1.png": "e30="}
for md_in in (
"![image 1](<images/imageFile1.png>)",
"![image 1](&lt;images/imageFile1.png&gt;)",
):
out = _rewrite_markdown_image_refs(md_in, images)
self.assertEqual("![image 1](images/imageFile1.png)", out)
def test_normalize_odl_image_url(self):
self.assertEqual(
_normalize_odl_image_url("&lt;images/imageFile2.png&gt;"),
"images/imageFile2.png",
)
self.assertEqual(
_normalize_odl_image_url("<images/imageFile2.png>"),
"images/imageFile2.png",
)
def test_rewrite_skips_data_uris(self):
md = "![x](data:image/png;base64,abc)"
self.assertEqual(_rewrite_markdown_image_refs(md, {"images/a.png": "e30="}), md)
class OpenDataLoaderParserTest(unittest.TestCase):
@mock.patch("docreader.parser.opendataloader_parser.opendataloader_available")
@mock.patch("docreader.parser.opendataloader_parser._run_convert")
def test_parse_reads_markdown_and_images(self, mock_convert, mock_avail):
mock_avail.return_value = (True, "")
def fake_convert(pdf_path, output_dir, image_dir, overrides=None):
stem = os.path.splitext(os.path.basename(pdf_path))[0]
md_path = os.path.join(output_dir, f"{stem}.md")
with open(md_path, "w") as f:
f.write("# Hello\n\n![pic](images/pic.png)\n")
os.makedirs(image_dir, exist_ok=True)
with open(os.path.join(image_dir, "pic.png"), "wb") as f:
f.write(b"png")
mock_convert.side_effect = fake_convert
parser = OpenDataLoaderParser(file_name="doc.pdf", file_type="pdf")
doc = parser.parse_into_text(b"%PDF-1.4 fake")
self.assertIn("# Hello", doc.content)
self.assertIn("images/pic.png", doc.content)
self.assertIn("images/pic.png", doc.images)
self.assertEqual(doc.metadata.get("parser_engine"), "opendataloader")
@mock.patch("docreader.parser.opendataloader_parser.shutil.which", return_value=None)
def test_availability_requires_java(self, _which):
with mock.patch(
"docreader.parser.opendataloader_parser._package_available",
return_value=(True, ""),
):
ok, msg = opendataloader_available()
self.assertFalse(ok)
self.assertIn("Java", msg)
if __name__ == "__main__":
unittest.main()
+248 -2
View File
@@ -6,10 +6,15 @@ from PIL import Image
from docreader.parser.pdf_parser import (
PDFParser,
_classify_page,
_filter_reading_columns,
_group_lines,
_is_artifact_column,
_join_line_glyphs,
_merge_orphan_punctuation_lines,
_point_in_boxes,
_segments_to_markdown,
_select_embedded_images,
_should_prefer_plain,
_split_columns,
_strip_repeating_lines,
)
@@ -122,13 +127,25 @@ class ReadingOrderTest(unittest.TestCase):
def test_group_lines_orders_by_y_then_x(self):
# Two visual lines; within a line glyphs given out of x-order.
chars = [
_char("B", 120, 130, 700, 712),
_char("A", 100, 110, 700, 712), # same line, left of B
_char("B", 110, 120, 700, 712), # adjacent to A (no word-sized gap)
_char("A", 100, 110, 700, 712),
_char("C", 100, 110, 680, 692), # next line down
]
lines = _group_lines(chars)
self.assertEqual([ln["text"] for ln in lines], ["AB", "C"])
def test_join_line_glyphs_inserts_word_spaces(self):
# Wide gap between "copy" and "of" mimics positioned OCR / text layers.
chars = [
_char("c", 0, 4, 0, 10),
_char("f", 10, 14, 0, 10),
]
self.assertEqual(_join_line_glyphs(chars), "c f")
def test_join_line_glyphs_keeps_adjacent_letters(self):
chars = [_char("A", 100, 110, 700, 712), _char("B", 110, 120, 700, 712)]
self.assertEqual(_join_line_glyphs(chars), "AB")
class HeadingDetectionTest(unittest.TestCase):
def test_promotes_large_line_to_heading(self):
@@ -156,6 +173,235 @@ class HiddenTextFilterTest(unittest.TestCase):
self.assertFalse(_point_in_boxes(20.0, 5.0, boxes))
class MarginColumnFilterTest(unittest.TestCase):
def test_drops_narrow_vertical_margin_column(self):
# Mimics arXiv sidebar: narrow x span, one glyph per line.
margin = [
_char(c, 20, 28, 500 - i * 14, 512 - i * 14)
for i, c in enumerate("0202luJ22")
]
body = [
_char("L", 160, 170, 700, 712),
_char("a", 170, 180, 700, 712),
_char("n", 180, 190, 700, 712),
]
cols = _filter_reading_columns(margin + body, scale=10.0, width=612.0)
self.assertEqual(len(cols), 1)
self.assertEqual(cols[0][0]["ch"], "L")
def test_keeps_real_two_column_layout(self):
left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)]
right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)]
cols = _filter_reading_columns(left + right, scale=12.0, width=600.0)
self.assertEqual(len(cols), 2)
class PunctuationMergeTest(unittest.TestCase):
def test_merges_orphan_periods(self):
lines = [
_line("Figure 1 2", 10.0),
_line(". .", 10.0),
_line("Next", 10.0),
]
merged = _merge_orphan_punctuation_lines(lines)
self.assertEqual([ln["text"] for ln in merged], ["Figure 1 2..", "Next"])
class PdfTextSanitizeTest(unittest.TestCase):
def test_removes_fffe_placeholder(self):
from docreader.parser.pdf_parser import _postprocess_pdf_text
raw = "multi\ufffelayer and non\ufffetrivial"
out = _postprocess_pdf_text(raw)
self.assertEqual(out, "multilayer and nontrivial")
def test_strips_chart_axis_run(self):
from docreader.parser.pdf_parser import _postprocess_pdf_text
raw = (
"Deep convolutional neural networks have led to breakthroughs.\n"
"0 1 2 3 4 5 6 0\n"
"10\n"
"20\n"
"iter. (1e4)\n"
"training error (%)\n"
"56-layer\n"
"20-layer\n"
"Figure 1. Training error on CIFAR-10.\n"
)
out = _postprocess_pdf_text(raw)
self.assertIn("breakthroughs", out)
self.assertNotIn("56-layer", out)
self.assertIn("Figure 1.", out)
def test_strips_diagram_labels_above_caption(self):
from docreader.parser.pdf_parser import _postprocess_pdf_text
raw = (
"Paragraph before.\n"
"identity\n"
"weight layer\n"
"relu\n"
"Figure 2. Residual learning block.\n"
"Paragraph after.\n"
)
out = _postprocess_pdf_text(raw)
self.assertIn("Paragraph before.", out)
self.assertIn("Figure 2.", out)
self.assertIn("Paragraph after.", out)
self.assertNotIn("identity", out)
self.assertNotIn("weight layer", out)
def test_strips_arxiv_header_line(self):
from docreader.parser.pdf_parser import _postprocess_pdf_text
raw = "Body text.\n1\narXiv:1512.03385v1 [cs.CV] 10 Dec 2015\nMore body."
out = _postprocess_pdf_text(raw)
self.assertNotIn("arXiv:", out)
self.assertIn("Body text.", out)
class PlainWellFormedTest(unittest.TestCase):
def test_academic_plain_skips_layout(self):
from docreader.parser.pdf_parser import _plain_is_well_formed
plain = (
"Recent work [DL15, MBXS17] shows progress on NLP tasks "
"with pre-trained models."
)
self.assertTrue(_plain_is_well_formed(plain))
def test_glued_scan_plain_needs_layout(self):
from docreader.parser.pdf_parser import _plain_is_well_formed
self.assertFalse(_plain_is_well_formed("Thisisadigitalcopyofabook"))
class LayoutQualityFallbackTest(unittest.TestCase):
def test_prefers_plain_when_many_single_char_lines(self):
plain = "Language Models are Few-Shot Learners\nTom Brown"
layout = "0\n2\n0\n2\nl\nu\nJ\nLan ua e Models"
self.assertTrue(_should_prefer_plain(plain, layout))
def test_keeps_good_layout(self):
plain = "Hello world"
layout = "Hello world"
self.assertFalse(_should_prefer_plain(plain, layout))
class ResNetPaperFigureTest(unittest.TestCase):
"""Regression: ResNet PDF (arXiv:1512.03385) vector figures and captions."""
def test_resnet_figures_and_captions(self):
import os
from docreader.parser.pdf_parser import PDFParser
for path in (
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"pdf_en",
"resnet.pdf",
),
"/tmp/resnet.pdf",
):
if os.path.isfile(path):
break
else:
self.skipTest("resnet.pdf not available")
with open(path, "rb") as f:
doc = PDFParser(file_name="resnet.pdf", file_type="pdf").parse_into_text(
f.read()
)
self.assertGreater(doc.metadata.get("vector_figure_count", 0), 0)
self.assertIn("![", doc.content)
self.assertIn("Figure 2. Residual learning", doc.content)
self.assertNotIn("arXiv:", doc.content)
fig2 = doc.content.find("Figure 2. Residual learning")
before = doc.content[max(0, fig2 - 120) : fig2]
self.assertIn("![", before)
self.assertNotIn("identity", before)
class Gpt3PaperLayoutTest(unittest.TestCase):
"""Regression: arXiv GPT-3 paper title page must not be one-glyph-per-line."""
def test_gpt3_page0_title_and_authors(self):
import os
import pypdfium2 as pdfium
import pypdfium2.raw as pdfium_r
from docreader.parser.pdf_parser import PDFParser, _extract_layout_text
pdf_path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"pdf_en",
"gpt3.pdf",
)
if not os.path.isfile(pdf_path):
self.skipTest("gpt3.pdf not in testdata")
with open(pdf_path, "rb") as f:
content = f.read()
with pdfium.PdfDocument(content) as pdf:
page = pdf[0]
try:
layout = _extract_layout_text(page, pdfium_r)
finally:
page.close()
# Margin sidebar must not appear as one-glyph-per-line prefix.
self.assertNotRegex(layout[:300], r"^0\n2\n0\n2")
self.assertIn("Few-Shot Learners", layout)
doc = PDFParser(file_name="gpt3.pdf", file_type="pdf").parse_into_text(content)
self.assertIn("Language Models are Few-Shot Learners", doc.content)
self.assertIn("Tom B. Brown", doc.content[:1200])
self.assertIn("[DL15, MBXS17, PNZtY18]", doc.content)
self.assertIn("task-specific architectures), and more recently", doc.content)
self.assertNotIn("k ifi hi d l", doc.content)
class ScanEnglishDictLayoutTest(unittest.TestCase):
"""Regression: Google Books-style PDFs lose spaces without gap inference."""
def test_scan_en_dict_page0_has_word_spaces(self):
import os
import pypdfium2 as pdfium
import pypdfium2.raw as pdfium_r
from docreader.parser.pdf_parser import _extract_layout_text
pdf_path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"pdf_scan",
"scan_en_dict.pdf",
)
if not os.path.isfile(pdf_path):
self.skipTest("scan_en_dict.pdf not in testdata")
with open(pdf_path, "rb") as f:
pdf = pdfium.PdfDocument(f.read())
try:
text = _extract_layout_text(pdf[0], pdfium_r)
finally:
pdf.close()
self.assertIn("This is a digital copy of a book", text)
self.assertNotIn("Thisisadigitalcopyofabook", text)
class PDFRouterIntegrationTest(unittest.TestCase):
def test_image_only_pdf_routes_to_scanned(self):
pdf_bytes = _make_image_only_pdf(2)
+86
View File
@@ -0,0 +1,86 @@
import shutil
import unittest
from pathlib import Path
from docreader.parser.ppt_convert import (
convert_ppt_to_pptx_bytes,
is_ole_compound,
is_zip_openxml,
needs_ppt_to_pptx_conversion,
normalize_ppt_bytes,
)
TESTDATA = Path(__file__).resolve().parents[2] / "testdata" / "rag_test"
LEGACY_PPT = TESTDATA / "ppt_old" / "en_38256.ppt"
WMF_IMAGE_PPT = LEGACY_PPT
IMAGE_HEAVY_PPT = TESTDATA / "ppt_old" / "en_41384.ppt"
PPTX_SAMPLE = TESTDATA / "pptx" / "en_marker.pptx"
class TestPptConvert(unittest.TestCase):
def test_legacy_ppt_magic(self):
content = LEGACY_PPT.read_bytes()
self.assertTrue(is_ole_compound(content))
self.assertFalse(is_zip_openxml(content))
self.assertTrue(needs_ppt_to_pptx_conversion(content, "ppt"))
def test_pptx_does_not_need_conversion(self):
content = PPTX_SAMPLE.read_bytes()
self.assertTrue(is_zip_openxml(content))
self.assertFalse(needs_ppt_to_pptx_conversion(content, "pptx"))
def test_normalize_pptx_passthrough(self):
content = PPTX_SAMPLE.read_bytes()
out, ext = normalize_ppt_bytes(content, "pptx")
self.assertEqual(out, content)
self.assertEqual(ext, ".pptx")
def test_legacy_ppt_requires_soffice(self):
if not shutil.which("soffice"):
with self.assertRaises(ValueError) as ctx:
normalize_ppt_bytes(LEGACY_PPT.read_bytes(), "ppt")
self.assertIn("LibreOffice", str(ctx.exception))
self.skipTest("LibreOffice not available")
converted = convert_ppt_to_pptx_bytes(LEGACY_PPT.read_bytes(), suffix=".ppt")
self.assertIsNotNone(converted)
self.assertTrue(is_zip_openxml(converted))
out, ext = normalize_ppt_bytes(LEGACY_PPT.read_bytes(), "ppt")
self.assertEqual(ext, ".pptx")
self.assertTrue(is_zip_openxml(out))
def test_wmf_legacy_ppt_extracts_rasterized_image(self):
if not shutil.which("soffice"):
self.skipTest("LibreOffice not available")
if not shutil.which("convert"):
self.skipTest("ImageMagick convert not available")
if not WMF_IMAGE_PPT.is_file():
self.skipTest("testdata missing")
from docreader.parser.markitdown_parser import MarkitdownParser
doc = MarkitdownParser(file_type="ppt").parse_into_text(
WMF_IMAGE_PPT.read_bytes()
)
self.assertEqual(len(doc.images), 1)
self.assertNotIn("bd10496_.jpg", doc.content)
self.assertIn("images/", doc.content)
def test_image_heavy_legacy_ppt_extracts_images(self):
if not shutil.which("soffice"):
self.skipTest("LibreOffice not available")
if not IMAGE_HEAVY_PPT.is_file():
self.skipTest("testdata missing")
from docreader.parser.markitdown_parser import MarkitdownParser
doc = MarkitdownParser(file_type="ppt").parse_into_text(
IMAGE_HEAVY_PPT.read_bytes()
)
self.assertGreaterEqual(len(doc.images), 2)
self.assertNotIn("![](.jpg)", doc.content)
for ref in doc.images:
self.assertTrue(ref.startswith("images/"))
if __name__ == "__main__":
unittest.main()
+42
View File
@@ -0,0 +1,42 @@
import unittest
from docreader.parser.web_parser import (
build_visible_text_fallback,
extract_markdown_from_html,
)
class TestWebParserHelpers(unittest.TestCase):
def test_extract_markdown_empty_html(self):
self.assertIsNone(extract_markdown_from_html(""))
self.assertIsNone(extract_markdown_from_html(" "))
def test_extract_markdown_article_html(self):
html = """
<html><head><title>Demo</title></head><body>
<article><h1>Hello</h1><p>World paragraph with enough text for extraction.</p></article>
</body></html>
"""
md = extract_markdown_from_html(html)
self.assertIsNotNone(md)
self.assertIn("Hello", md)
def test_build_fallback_too_short(self):
self.assertIsNone(build_visible_text_fallback("short"))
self.assertIsNone(build_visible_text_fallback(""))
def test_build_fallback_with_title(self):
text = "A" * 60
md = build_visible_text_fallback(text, page_title="WeKnora")
self.assertIsNotNone(md)
self.assertTrue(md.startswith("# WeKnora"))
self.assertIn(text, md)
def test_build_fallback_without_title(self):
text = "B" * 60
md = build_visible_text_fallback(text, page_title="")
self.assertEqual(md, text)
if __name__ == "__main__":
unittest.main()