feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing. Optional odl-hybrid sidecar stays local-build only and is excluded from default dev-start and full profiles.
This commit is contained in:
@@ -62,6 +62,67 @@ DEFAULT_CONFIGS = [
|
||||
]
|
||||
DEFAULT_CONFIGS.sort(key=lambda x: -x.priority)
|
||||
|
||||
_TABLE_ROW_PATTERN = re.compile(r"^\s*(?:\|[^|\n]*)+\|\s*$", re.MULTILINE)
|
||||
_MARKDOWN_TABLE_PRIORITY = 15
|
||||
|
||||
|
||||
def _is_empty_table_header_row(header: str) -> bool:
|
||||
"""True when the column-name line is only pipes/whitespace (MarkItDown quirk)."""
|
||||
newline = header.find("\n")
|
||||
if newline < 0:
|
||||
return False
|
||||
row = header[:newline].strip()
|
||||
return bool(row) and all(ch in "| \t" for ch in row)
|
||||
|
||||
|
||||
def _extract_separator_line(header: str) -> str:
|
||||
for line in header.split("\n"):
|
||||
if "---" in line:
|
||||
return line + "\n"
|
||||
return ""
|
||||
|
||||
|
||||
def _table_row_column_count(line: str) -> int:
|
||||
line = line.strip()
|
||||
if not line.startswith("|"):
|
||||
return 0
|
||||
parts = line.split("|")
|
||||
if parts and parts[0].strip() == "":
|
||||
parts = parts[1:]
|
||||
if parts and parts[-1].strip() == "":
|
||||
parts = parts[:-1]
|
||||
return len(parts)
|
||||
|
||||
|
||||
def _first_table_row_column_count(text: str) -> int:
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if line and _TABLE_ROW_PATTERN.match(line):
|
||||
return _table_row_column_count(line)
|
||||
return 0
|
||||
|
||||
|
||||
def _header_table_column_count(header: str) -> int:
|
||||
for line in header.split("\n"):
|
||||
line = line.strip()
|
||||
if not line or "---" in line:
|
||||
continue
|
||||
count = _table_row_column_count(line)
|
||||
if count > 0:
|
||||
return count
|
||||
return 0
|
||||
|
||||
|
||||
def _split_ends_with_paragraph_break(split: str) -> bool:
|
||||
trimmed = split.rstrip(" \t\r")
|
||||
return trimmed.endswith("\n\n") or trimmed.endswith("\r\n\r\n")
|
||||
|
||||
|
||||
def header_column_mismatch(headers: str, next_unit: str) -> bool:
|
||||
header_cols = _header_table_column_count(headers)
|
||||
row_cols = _first_table_row_column_count(next_unit)
|
||||
return header_cols > 0 and row_cols > 0 and header_cols != row_cols
|
||||
|
||||
|
||||
# 定义Hook状态数据结构
|
||||
class HeaderTracker(BaseModel):
|
||||
@@ -70,10 +131,28 @@ class HeaderTracker(BaseModel):
|
||||
header_hook_configs: List[HeaderTrackerHook] = Field(default=DEFAULT_CONFIGS)
|
||||
active_headers: Dict[int, str] = Field(default_factory=dict)
|
||||
ended_headers: set[int] = Field(default_factory=set)
|
||||
pending_extend: Dict[int, bool] = Field(default_factory=dict)
|
||||
pending_table_break: bool = Field(default=False)
|
||||
header_ended_this_unit: bool = Field(default=False)
|
||||
|
||||
def _clear_table_header(self) -> None:
|
||||
self.ended_headers.add(_MARKDOWN_TABLE_PRIORITY)
|
||||
self.active_headers.pop(_MARKDOWN_TABLE_PRIORITY, None)
|
||||
self.pending_extend.pop(_MARKDOWN_TABLE_PRIORITY, None)
|
||||
|
||||
def update(self, split: str) -> Dict[int, str]:
|
||||
"""检测当前split中的表头开始/结束,更新Hook状态"""
|
||||
new_headers: Dict[int, str] = {}
|
||||
self.header_ended_this_unit = False
|
||||
|
||||
if self.pending_table_break:
|
||||
self.pending_table_break = False
|
||||
if _MARKDOWN_TABLE_PRIORITY in self.active_headers:
|
||||
if _first_table_row_column_count(split) > 0:
|
||||
self._clear_table_header()
|
||||
self.header_ended_this_unit = True
|
||||
else:
|
||||
self._clear_table_header()
|
||||
|
||||
# 1. 检查是否有表头结束标记
|
||||
for config in self.header_hook_configs:
|
||||
@@ -82,8 +161,31 @@ class HeaderTracker(BaseModel):
|
||||
):
|
||||
self.ended_headers.add(config.priority)
|
||||
del self.active_headers[config.priority]
|
||||
self.pending_extend.pop(config.priority, None)
|
||||
|
||||
# 2. 检查是否有新的表头开始标记(只处理未活跃且未结束的)
|
||||
# 1b. \n\n 分块会吞掉表间空行:段尾 \n\n 或列数变化时结束表头追踪
|
||||
if (
|
||||
_MARKDOWN_TABLE_PRIORITY in self.active_headers
|
||||
and not self.pending_extend.get(_MARKDOWN_TABLE_PRIORITY)
|
||||
):
|
||||
if _split_ends_with_paragraph_break(split):
|
||||
self.pending_table_break = True
|
||||
else:
|
||||
header = self.active_headers[_MARKDOWN_TABLE_PRIORITY]
|
||||
row_cols = _first_table_row_column_count(split)
|
||||
header_cols = _header_table_column_count(header)
|
||||
if row_cols > 0 and header_cols > 0 and row_cols != header_cols:
|
||||
self._clear_table_header()
|
||||
self.header_ended_this_unit = True
|
||||
|
||||
# 2. 空表头行:用首个数据行补全列名(与 Go header_tracker 一致)
|
||||
for priority in list(self.pending_extend.keys()):
|
||||
if priority in self.active_headers and _TABLE_ROW_PATTERN.search(split):
|
||||
sep = _extract_separator_line(self.active_headers[priority])
|
||||
self.active_headers[priority] = split + sep
|
||||
self.pending_extend.pop(priority, None)
|
||||
|
||||
# 3. 检查是否有新的表头开始标记(只处理未活跃且未结束的)
|
||||
for config in self.header_hook_configs:
|
||||
if (
|
||||
config.priority not in self.active_headers
|
||||
@@ -94,8 +196,10 @@ class HeaderTracker(BaseModel):
|
||||
header = config.extract_header_fn(match)
|
||||
self.active_headers[config.priority] = header
|
||||
new_headers[config.priority] = header
|
||||
if _is_empty_table_header_row(header):
|
||||
self.pending_extend[config.priority] = True
|
||||
|
||||
# 3. 检查是否所有活跃表头都已结束(清空结束标记)
|
||||
# 4. 检查是否所有活跃表头都已结束(清空结束标记)
|
||||
if not self.active_headers:
|
||||
self.ended_headers.clear()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user