feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+70 -6
View File
@@ -18,6 +18,8 @@ import re
import uuid
from typing import Dict, List, Match, Optional, Tuple
_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
@@ -58,6 +60,71 @@ class MarkdownTableUtil:
re.MULTILINE,
)
@staticmethod
def _split_row_cells(row_line: str) -> List[str]:
"""Split a markdown table row into cells, preserving empty cells."""
inner = row_line.strip()
if not inner.startswith("|"):
return []
parts = inner.split("|")
if parts and parts[0].strip() == "":
parts = parts[1:]
if parts and parts[-1].strip() == "":
parts = parts[:-1]
return [part.strip() for part in parts]
@staticmethod
def _is_table_row(line: str) -> bool:
stripped = line.strip()
return stripped.startswith("|") and "|" in stripped[1:]
@classmethod
def _is_separator_row(cls, line: str) -> bool:
cells = cls._split_row_cells(line)
return bool(cells) and all(_SEPARATOR_CELL.match(cell) for cell in cells)
@classmethod
def _is_empty_row(cls, line: str) -> bool:
cells = cls._split_row_cells(line)
return bool(cells) and all(cell == "" for cell in cells)
@classmethod
def _separator_row_for(cls, header_line: str) -> str:
cells = cls._split_row_cells(header_line)
return "| " + " | ".join("---" for _ in cells) + " |"
@classmethod
def _normalize_table_block(cls, block: List[str]) -> List[str]:
"""Fix MarkItDown-style tables: drop bogus prefix rows, ensure GFM delimiter."""
while block and cls._is_empty_row(block[0]):
block.pop(0)
if block and cls._is_separator_row(block[0]):
block.pop(0)
# GFM/marked need "| --- |" after the first row. Headerless Word tables
# only have data rows after we strip the fake empty+separator prefix.
if len(block) >= 2 and not cls._is_separator_row(block[1]):
sep = cls._separator_row_for(block[0])
block = [block[0], sep] + block[1:]
return block
def normalize_spurious_table_prefixes(self, content: str) -> str:
"""Remove bogus empty/separator prefix rows from MarkItDown table output."""
lines = content.split("\n")
out: List[str] = []
i = 0
while i < len(lines):
line = lines[i]
if not self._is_table_row(line):
out.append(line)
i += 1
continue
block: List[str] = []
while i < len(lines) and self._is_table_row(lines[i]):
block.append(lines[i])
i += 1
out.extend(self._normalize_table_block(block))
return "\n".join(out)
def format_table(self, content: str) -> str:
"""Format all Markdown tables in the content.
@@ -70,8 +137,7 @@ class MarkdownTableUtil:
def process_align(match: Match[str]) -> str:
"""Process alignment row to standardize format."""
# Split by | and remove empty strings
columns = [col.strip() for col in match.group(0).split("|") if col.strip()]
columns = self._split_row_cells(match.group(0))
processed = []
for col in columns:
@@ -87,8 +153,7 @@ class MarkdownTableUtil:
def process_line(match: Match[str]) -> str:
"""Process regular table row to standardize format."""
# Split by | and remove empty strings
columns = [col.strip() for col in match.group(0).split("|") if col.strip()]
columns = self._split_row_cells(match.group(0))
# Preserve original indentation
prefix = match.group(1)
@@ -99,8 +164,7 @@ class MarkdownTableUtil:
formatted_content = self.line_pattern.sub(process_line, formatted_content)
# Then format alignment rows (must be done after to avoid conflicts)
formatted_content = self.align_pattern.sub(process_align, formatted_content)
return formatted_content
return self.normalize_spurious_table_prefixes(formatted_content)
@staticmethod
def _self_test():