7b1bb1054f
Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end.
This change addresses the parse, transport, and task-scheduling layers:
docreader (parse + transport):
- Parallelize per-page scanned rendering across processes (forkserver/fork),
with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not
thread-safe so we fan out across processes. Configurable via
DOCREADER_PDF_RENDER_PARALLELISM.
- Add server-streaming ReadStream RPC: emit one meta frame then one frame per
image, so documents with many page images are no longer capped by the unary
gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over
the 50MB cap) and memory is bounded on both ends. Unary Read is kept for
backward compatibility; the Go production reader switches to ReadStream.
VLM:
- Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise
the default 90s -> 180s so dense scanned-page OCR does not time out with
"context deadline exceeded".
Async task queues:
- Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so
a single large document cannot saturate the shared worker pool and block
user-facing document parsing:
image:multimodal -> "multimodal"
chunk:extract -> "graph"
question:generation -> "question"
- Register the new queues in the server weight map and the cancel inspector's
scanned-queue set (so cancelling a knowledge still purges its pending tasks).
110 lines
4.2 KiB
Python
110 lines
4.2 KiB
Python
"""Optional PDF engine backed by LiteParse (LlamaIndex, MIT).
|
|
|
|
LiteParse is a fast Rust/PDFium text extractor that performs spatial reading-order
|
|
reconstruction natively (multi-column aware) and is considerably faster than the
|
|
Python text path. It is exposed as a *selectable* engine (``liteparse``) rather
|
|
than replacing the builtin engine, so users can opt in per knowledge base.
|
|
|
|
Scope/limitations (documented intentionally):
|
|
* Text-first engine: it returns reading-order plain text, not figures. Scanned
|
|
pages carry no text layer, so for image-dominated PDFs we fall back to the
|
|
builtin scanned renderer (page -> JPEG, OCR'd by the Go App) to stay robust.
|
|
* docreader never runs OCR itself; OCR/VLM remain Go-side responsibilities.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from docreader.models.document import Document
|
|
from docreader.parser.base_parser import BaseParser
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# If the extracted text averages fewer characters per page than this, the PDF is
|
|
# treated as scanned/image-dominated and routed to the builtin image renderer.
|
|
_MIN_CHARS_PER_PAGE = 20
|
|
# If at least this fraction of sampled pages are image-dominated, the PDF is
|
|
# scanned (even when it carries a garbled OCR text layer) and is routed to the
|
|
# builtin image renderer rather than trusting the low-quality text.
|
|
_SCANNED_PAGE_FRACTION = 0.5
|
|
|
|
|
|
def liteparse_available(_overrides=None):
|
|
"""Engine availability probe used by the registry/UI."""
|
|
try:
|
|
import liteparse # noqa: F401
|
|
except Exception as e: # pragma: no cover - depends on install
|
|
return False, f"liteparse 未安装: {e}"
|
|
return True, ""
|
|
|
|
|
|
class LiteParseParser(BaseParser):
|
|
"""Parse a PDF with LiteParse, falling back to scanned rendering when empty."""
|
|
|
|
def parse_into_text(self, content: bytes) -> Document:
|
|
import liteparse
|
|
|
|
from docreader.parser.pdf_parser import (
|
|
PDFScannedParser,
|
|
estimate_scanned_fraction,
|
|
)
|
|
|
|
# Image-dominated PDFs (incl. ones with a garbled OCR text layer) carry
|
|
# no trustworthy text; render them as images for Go-side OCR instead.
|
|
try:
|
|
scanned_frac = estimate_scanned_fraction(content)
|
|
except Exception:
|
|
scanned_frac = 0.0
|
|
if scanned_frac >= _SCANNED_PAGE_FRACTION:
|
|
logger.info(
|
|
"LiteParseParser: %s is image-dominated (%.0f%% scanned pages); "
|
|
"using builtin scanned renderer",
|
|
self.file_name,
|
|
scanned_frac * 100,
|
|
)
|
|
return PDFScannedParser(
|
|
file_name=self.file_name, file_type=self.file_type
|
|
).parse_into_text(content)
|
|
|
|
engine = liteparse.LiteParse(ocr_enabled=False, quiet=True)
|
|
result = engine.parse(content)
|
|
page_count = int(result.num_pages)
|
|
|
|
page_texts = []
|
|
for i in range(page_count):
|
|
page = result.get_page(i)
|
|
page_texts.append((getattr(page, "text", "") or "").strip())
|
|
|
|
doc_text = (getattr(result, "text", "") or "").strip()
|
|
if not doc_text:
|
|
doc_text = "\n\n".join(t for t in page_texts if t)
|
|
|
|
# Image-dominated / scanned PDFs yield little to no text: defer to the
|
|
# builtin scanned renderer so the Go App can OCR the page images.
|
|
if page_count and len(doc_text) < _MIN_CHARS_PER_PAGE * page_count:
|
|
logger.info(
|
|
"LiteParseParser: %s looks scanned (%d chars / %d pages); "
|
|
"falling back to builtin scanned renderer",
|
|
self.file_name,
|
|
len(doc_text),
|
|
page_count,
|
|
)
|
|
return PDFScannedParser(
|
|
file_name=self.file_name, file_type=self.file_type
|
|
).parse_into_text(content)
|
|
|
|
logger.info(
|
|
"LiteParseParser: %s -> %d pages, content_len=%d",
|
|
self.file_name,
|
|
page_count,
|
|
len(doc_text),
|
|
)
|
|
return Document(
|
|
content=doc_text,
|
|
images={},
|
|
metadata={
|
|
"page_count": page_count,
|
|
"image_source_type": "pdf_text_layer",
|
|
"parser_engine": "liteparse",
|
|
},
|
|
)
|