feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing. Optional odl-hybrid sidecar stays local-build only and is excluded from default dev-start and full profiles.
This commit is contained in:
@@ -16,6 +16,7 @@ from pydantic import BaseModel, Field, PrivateAttr
|
||||
|
||||
from docreader.splitter.header_hook import (
|
||||
HeaderTracker,
|
||||
header_column_mismatch,
|
||||
)
|
||||
from docreader.utils.split import split_by_char, split_by_sep
|
||||
|
||||
@@ -225,6 +226,16 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
|
||||
# Update header tracking with current split
|
||||
self.header_hook.update(split)
|
||||
if self.header_hook.header_ended_this_unit and len(cur_chunk) > 0:
|
||||
chunks.append(
|
||||
(
|
||||
cur_chunk[0][0],
|
||||
cur_chunk[-1][1],
|
||||
"".join([c[2] for c in cur_chunk]),
|
||||
)
|
||||
)
|
||||
cur_chunk = []
|
||||
cur_len = 0
|
||||
cur_headers = self.header_hook.get_headers()
|
||||
cur_headers_len = self.len_function(cur_headers)
|
||||
|
||||
@@ -276,6 +287,7 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
cur_headers
|
||||
and split_len + cur_headers_len < self.chunk_size
|
||||
and cur_headers not in split
|
||||
and not header_column_mismatch(cur_headers, split)
|
||||
):
|
||||
next_start = cur_chunk[0][0] if cur_chunk else cur_start
|
||||
|
||||
|
||||
Reference in New Issue
Block a user