feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+12
View File
@@ -16,6 +16,7 @@ from pydantic import BaseModel, Field, PrivateAttr
from docreader.splitter.header_hook import (
HeaderTracker,
header_column_mismatch,
)
from docreader.utils.split import split_by_char, split_by_sep
@@ -225,6 +226,16 @@ class TextSplitter(BaseModel, Generic[T]):
# Update header tracking with current split
self.header_hook.update(split)
if self.header_hook.header_ended_this_unit and len(cur_chunk) > 0:
chunks.append(
(
cur_chunk[0][0],
cur_chunk[-1][1],
"".join([c[2] for c in cur_chunk]),
)
)
cur_chunk = []
cur_len = 0
cur_headers = self.header_hook.get_headers()
cur_headers_len = self.len_function(cur_headers)
@@ -276,6 +287,7 @@ class TextSplitter(BaseModel, Generic[T]):
cur_headers
and split_len + cur_headers_len < self.chunk_size
and cur_headers not in split
and not header_column_mismatch(cur_headers, split)
):
next_start = cur_chunk[0][0] if cur_chunk else cur_start