From 7b1bb1054fe6a80c640133a716d6343004b0211b Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 2 Jun 2026 09:50:09 +0800 Subject: [PATCH] feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks). --- .env.example | 16 +- docreader/config.py | 23 +- docreader/main.py | 159 +++- docreader/parser/liteparse_parser.py | 109 +++ docreader/parser/pdf_parser.py | 890 ++++++++++++++++-- docreader/parser/registry.py | 9 + docreader/proto/docreader.pb.go | 263 +++++- docreader/proto/docreader.proto | 25 + docreader/proto/docreader_grpc.pb.go | 58 +- docreader/proto/docreader_pb2.py | 30 +- docreader/proto/docreader_pb2.pyi | 29 + docreader/proto/docreader_pb2_grpc.py | 51 +- docreader/pyproject.toml | 1 + docreader/scripts/parse_local.py | 119 +++ docreader/tests/test_pdf_router.py | 191 ++++ docreader/uv.lock | 33 + internal/application/service/extract.go | 2 +- .../service/knowledge_post_process.go | 2 +- .../application/service/knowledge_process.go | 2 +- .../infrastructure/docparser/grpc_parser.go | 69 +- internal/models/vlm/remote_api.go | 22 +- internal/models/vlm/weknoracloud.go | 2 +- internal/router/task.go | 9 +- internal/router/task_inspector.go | 12 +- internal/types/task.go | 23 + scripts/dev.sh | 33 +- 26 files changed, 1994 insertions(+), 188 deletions(-) create mode 100644 docreader/parser/liteparse_parser.py create mode 100644 docreader/scripts/parse_local.py create mode 100644 docreader/tests/test_pdf_router.py diff --git a/.env.example b/.env.example index f8578cf9..8523e5fd 100644 --- a/.env.example +++ b/.env.example @@ -552,8 +552,22 @@ DOCREADER_TRANSPORT=grpc # 设为 0 可关闭对应限流;默认 1 更适合普通单机或 GPU 资源有限的部署 # DOCREADER_MARKITDOWN_MAX_WORKERS=1 # DOCREADER_PDF_RENDER_MAX_WORKERS=1 +# 单个 PDF 内部「扫描页渲染」的并行进程数。pdfium 非线程安全,故按进程并行, +# 每个进程各自打开文档渲染分到的页。大体量扫描件串行渲染在 CPU 受限容器上可能 +# 耗时数小时,这是缩短该耗时的主要手段(实测 874 页 117s→30s/4 进程,→17s/8 进程)。 +# 默认随 CPU 核数自动取 min(4, cpu)。注意:容器需分配足够 CPU 才能吃到加速。 +# DOCREADER_PDF_RENDER_PARALLELISM=4 # DOCREADER_PDF_RENDER_DPI=200 -# DOCREADER_PDF_JPEG_QUALITY=90 +# DOCREADER_PDF_JPEG_QUALITY=85 +# 渲染页图的最大长边像素(防止超大页面 PDF 渲染出 100+MP 图、撑爆 gRPC 消息上限) +# 调小可进一步减小图片体积;过小会影响 OCR 识别(密集中文建议 >=1600) +# DOCREADER_PDF_RENDER_MAX_EDGE=2000 + +# VLM(视觉模型)单次 HTTP 请求的整体超时时间(秒)。 +# 扫描件整页 OCR(全文+版式抽取)在慢端点上很容易超过默认值, +# 出现 "context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 时调大本值。 +# 默认 180。 +# VLM_HTTP_TIMEOUT_SECONDS=180 # 如果使用Weaviate作为向量存储,需要配置以下参数 # 注意:容器内访问请使用 service:port(不要用 localhost,也不要用宿主机映射端口) diff --git a/docreader/config.py b/docreader/config.py index c9530722..30091c02 100644 --- a/docreader/config.py +++ b/docreader/config.py @@ -56,8 +56,10 @@ class DocReaderConfig: docx_max_pages: int markitdown_max_workers: int pdf_render_max_workers: int + pdf_render_parallelism: int pdf_render_dpi: int pdf_jpeg_quality: int + pdf_render_max_edge: int # Proxy external_http_proxy: str @@ -80,8 +82,23 @@ def load_config() -> DocReaderConfig: docx_max_pages = _get_int(["DOCREADER_DOCX_MAX_PAGES"], 0) markitdown_max_workers = _get_int(["DOCREADER_MARKITDOWN_MAX_WORKERS"], 1) pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1) + # Intra-document render parallelism: how many worker processes render the + # scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so + # page rendering is fanned out across processes (each opens its own + # document). A large scanned PDF rendered serially can take >1h on + # CPU-constrained containers; this is the main lever to cut that wall time. + # Default scales with CPU count but is capped so we don't oversubscribe. + _cpu = os.cpu_count() or 1 + pdf_render_parallelism = _get_int( + ["DOCREADER_PDF_RENDER_PARALLELISM"], max(1, min(4, _cpu)) + ) pdf_render_dpi = _get_int(["DOCREADER_PDF_RENDER_DPI"], 200) - pdf_jpeg_quality = _get_int(["DOCREADER_PDF_JPEG_QUALITY"], 90) + pdf_jpeg_quality = _get_int(["DOCREADER_PDF_JPEG_QUALITY"], 85) + # Cap the long edge (px) of rendered/extracted page images. Without this, + # PDFs declaring very large page boxes render to 100+ MP JPEGs that blow the + # gRPC message limit (and are far higher-res than OCR needs). ~2000px keeps + # dense CJK text legible for OCR while keeping page images well under ~1MB. + pdf_render_max_edge = _get_int(["DOCREADER_PDF_RENDER_MAX_EDGE"], 2000) external_http_proxy = _get_str( ["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], "" @@ -101,8 +118,10 @@ def load_config() -> DocReaderConfig: docx_max_pages=docx_max_pages, markitdown_max_workers=markitdown_max_workers, pdf_render_max_workers=pdf_render_max_workers, + pdf_render_parallelism=pdf_render_parallelism, pdf_render_dpi=pdf_render_dpi, pdf_jpeg_quality=pdf_jpeg_quality, + pdf_render_max_edge=pdf_render_max_edge, external_http_proxy=external_http_proxy, external_https_proxy=external_https_proxy, image_output_dir=image_output_dir, @@ -121,8 +140,10 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]: "DOCREADER_DOCX_MAX_PAGES": cfg.docx_max_pages, "DOCREADER_MARKITDOWN_MAX_WORKERS": cfg.markitdown_max_workers, "DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers, + "DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism, "DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi, "DOCREADER_PDF_JPEG_QUALITY": cfg.pdf_jpeg_quality, + "DOCREADER_PDF_RENDER_MAX_EDGE": cfg.pdf_render_max_edge, "DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy, "DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy, "DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir, diff --git a/docreader/main.py b/docreader/main.py index 6f915a50..51f2cd0c 100644 --- a/docreader/main.py +++ b/docreader/main.py @@ -21,6 +21,8 @@ from docreader.proto.docreader_pb2 import ( ReadRequest, ReadResponse, ImageRef, + ReadStreamMeta, + ReadStreamResponse, ListEnginesResponse, ParserEngineInfo, ) @@ -104,49 +106,93 @@ def _resolve_images( return "", refs +def _mime_for_ref(ref_path: str) -> tuple[str, str]: + """Return (filename, mime_type) for an image reference path.""" + mime_map = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + } + fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png" + ext = os.path.splitext(fname)[1].lower() + return fname, mime_map.get(ext, "application/octet-stream") + + +def _iter_image_refs(images: dict): + """Yield ImageRef one at a time, freeing each source entry as we go. + + Used by the streaming RPC so we never hold every decoded image plus its + base64 source in memory simultaneously (the inline path's peak-memory and + message-size problem for large scanned PDFs). + """ + import base64 + + for ref_path in list(images.keys()): + b64data = images.pop(ref_path) + try: + img_bytes = base64.b64decode(b64data) + except Exception: + img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data + del b64data + fname, mime = _mime_for_ref(ref_path) + yield ImageRef( + filename=fname, + original_ref=ref_path, + mime_type=mime, + image_data=img_bytes, + ) + + class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer): def __init__(self): super().__init__() self.parser = Parser() + def _parse_request(self, request: ReadRequest): + """Run the parser for a ReadRequest, returning (result, source_desc). + + Shared by the unary Read and streaming ReadStream RPCs. + """ + cfg = request.config + parser_engine = cfg.parser_engine if cfg else "" + engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {} + + if request.url: + logger.info("Read(URL): url=%s", request.url) + result = self.parser.parse_url( + request.url, + request.title, + parser_engine=parser_engine, + engine_overrides=engine_overrides, + ) + return result, request.url + + file_type = request.file_type or os.path.splitext(request.file_name)[1][1:] + logger.info( + "Read(File): file=%s, type=%s, size=%d bytes", + request.file_name, + file_type, + len(request.file_content), + ) + result = self.parser.parse_file( + request.file_name, + file_type, + request.file_content, + parser_engine=parser_engine, + engine_overrides=engine_overrides, + ) + return result, request.file_name + def Read(self, request: ReadRequest, context): """Unified read: file mode (file_content set) or URL mode (url set).""" request_id = request.request_id or str(uuid.uuid4()) - is_url = bool(request.url) with request_id_context(request_id): try: - cfg = request.config - parser_engine = cfg.parser_engine if cfg else "" - engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {} - - if is_url: - logger.info("Read(URL): url=%s", request.url) - result = self.parser.parse_url( - request.url, - request.title, - parser_engine=parser_engine, - engine_overrides=engine_overrides, - ) - source_desc = request.url - else: - file_type = ( - request.file_type or os.path.splitext(request.file_name)[1][1:] - ) - logger.info( - "Read(File): file=%s, type=%s, size=%d bytes", - request.file_name, - file_type, - len(request.file_content), - ) - result = self.parser.parse_file( - request.file_name, - file_type, - request.file_content, - parser_engine=parser_engine, - engine_overrides=engine_overrides, - ) - source_desc = request.file_name + result, source_desc = self._parse_request(request) if not result or not result.content: error_msg = f"Failed to parse: {source_desc}" @@ -177,6 +223,55 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer): logger.info("Traceback: %s", traceback.format_exc()) return ReadResponse(error=str(e)) + def ReadStream(self, request: ReadRequest, context): + """Streaming read: yields one meta frame, then one frame per image. + + Each frame is a small, independent gRPC message, so documents with many + page images (large scanned PDFs) are returned without hitting the unary + message-size cap, and neither side has to hold the whole payload at once. + """ + request_id = request.request_id or str(uuid.uuid4()) + + with request_id_context(request_id): + _c = to_valid_utf8_text + try: + result, source_desc = self._parse_request(request) + except Exception as e: + logger.error("Error reading document: %s", e) + logger.info("Traceback: %s", traceback.format_exc()) + yield ReadStreamResponse(meta=ReadStreamMeta(error=str(e))) + return + + if not result or not result.content: + error_msg = f"Failed to parse: {source_desc}" + logger.error(error_msg) + yield ReadStreamResponse(meta=ReadStreamMeta(error=error_msg)) + return + + images = result.images or {} + image_count = len(images) + yield ReadStreamResponse( + meta=ReadStreamMeta( + markdown_content=_c(result.content), + image_dir_path="", + metadata={k: _c(str(v)) for k, v in result.metadata.items()} + if result.metadata + else {}, + image_count=image_count, + ) + ) + + sent = 0 + for ref in _iter_image_refs(images): + yield ReadStreamResponse(image=ref) + sent += 1 + + logger.info( + "ReadStream response: content_len=%d, images=%d", + len(result.content), + sent, + ) + def ListEngines(self, request, context): overrides = dict(getattr(request, "config_overrides", None) or {}) engines_data = registry.list_engines(overrides=overrides or None) diff --git a/docreader/parser/liteparse_parser.py b/docreader/parser/liteparse_parser.py new file mode 100644 index 00000000..e5991e33 --- /dev/null +++ b/docreader/parser/liteparse_parser.py @@ -0,0 +1,109 @@ +"""Optional PDF engine backed by LiteParse (LlamaIndex, MIT). + +LiteParse is a fast Rust/PDFium text extractor that performs spatial reading-order +reconstruction natively (multi-column aware) and is considerably faster than the +Python text path. It is exposed as a *selectable* engine (``liteparse``) rather +than replacing the builtin engine, so users can opt in per knowledge base. + +Scope/limitations (documented intentionally): + * Text-first engine: it returns reading-order plain text, not figures. Scanned + pages carry no text layer, so for image-dominated PDFs we fall back to the + builtin scanned renderer (page -> JPEG, OCR'd by the Go App) to stay robust. + * docreader never runs OCR itself; OCR/VLM remain Go-side responsibilities. +""" + +import logging + +from docreader.models.document import Document +from docreader.parser.base_parser import BaseParser + +logger = logging.getLogger(__name__) + +# If the extracted text averages fewer characters per page than this, the PDF is +# treated as scanned/image-dominated and routed to the builtin image renderer. +_MIN_CHARS_PER_PAGE = 20 +# If at least this fraction of sampled pages are image-dominated, the PDF is +# scanned (even when it carries a garbled OCR text layer) and is routed to the +# builtin image renderer rather than trusting the low-quality text. +_SCANNED_PAGE_FRACTION = 0.5 + + +def liteparse_available(_overrides=None): + """Engine availability probe used by the registry/UI.""" + try: + import liteparse # noqa: F401 + except Exception as e: # pragma: no cover - depends on install + return False, f"liteparse 未安装: {e}" + return True, "" + + +class LiteParseParser(BaseParser): + """Parse a PDF with LiteParse, falling back to scanned rendering when empty.""" + + def parse_into_text(self, content: bytes) -> Document: + import liteparse + + from docreader.parser.pdf_parser import ( + PDFScannedParser, + estimate_scanned_fraction, + ) + + # Image-dominated PDFs (incl. ones with a garbled OCR text layer) carry + # no trustworthy text; render them as images for Go-side OCR instead. + try: + scanned_frac = estimate_scanned_fraction(content) + except Exception: + scanned_frac = 0.0 + if scanned_frac >= _SCANNED_PAGE_FRACTION: + logger.info( + "LiteParseParser: %s is image-dominated (%.0f%% scanned pages); " + "using builtin scanned renderer", + self.file_name, + scanned_frac * 100, + ) + return PDFScannedParser( + file_name=self.file_name, file_type=self.file_type + ).parse_into_text(content) + + engine = liteparse.LiteParse(ocr_enabled=False, quiet=True) + result = engine.parse(content) + page_count = int(result.num_pages) + + page_texts = [] + for i in range(page_count): + page = result.get_page(i) + page_texts.append((getattr(page, "text", "") or "").strip()) + + doc_text = (getattr(result, "text", "") or "").strip() + if not doc_text: + doc_text = "\n\n".join(t for t in page_texts if t) + + # Image-dominated / scanned PDFs yield little to no text: defer to the + # builtin scanned renderer so the Go App can OCR the page images. + if page_count and len(doc_text) < _MIN_CHARS_PER_PAGE * page_count: + logger.info( + "LiteParseParser: %s looks scanned (%d chars / %d pages); " + "falling back to builtin scanned renderer", + self.file_name, + len(doc_text), + page_count, + ) + return PDFScannedParser( + file_name=self.file_name, file_type=self.file_type + ).parse_into_text(content) + + logger.info( + "LiteParseParser: %s -> %d pages, content_len=%d", + self.file_name, + page_count, + len(doc_text), + ) + return Document( + content=doc_text, + images={}, + metadata={ + "page_count": page_count, + "image_source_type": "pdf_text_layer", + "parser_engine": "liteparse", + }, + ) diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py index fc080ff7..33b14207 100644 --- a/docreader/parser/pdf_parser.py +++ b/docreader/parser/pdf_parser.py @@ -1,18 +1,100 @@ -from docreader.models.document import Document -from docreader.config import CONFIG -from docreader.parser.base_parser import BaseParser -from docreader.parser.chain_parser import FirstParser -from docreader.parser.concurrency import parser_worker_limit -from docreader.parser.markitdown_parser import MarkitdownParser +"""PDF parsing with per-page routing between native text and scanned images. + +Design (aligned with how MinerU / Docling / DeepDoc route PDFs): + +* The dominant signal for "this page is scanned" is the **image-area coverage + ratio** (image bounding-box area / page area), not the raw character count. + A scanned page is essentially one big image covering the whole page, even + when it carries a (often low-quality) embedded OCR text layer. Trusting that + embedded text layer is what produced garbled RAG content before. +* Pages are classified independently so hybrid PDFs (some native, some scanned) + are handled correctly. Native pages contribute their text layer; scanned + pages are rendered to JPEG and tagged ``image_source_type=scanned_pdf`` so the + Go App performs OCR/VLM on them (docreader itself never runs OCR). + +No external services (e.g. MinerU) are required: the builtin engine is fully +self-sufficient using pypdfium2 + the Go-side OCR that already exists. +""" -import io -import os import base64 +import io import logging +import os +import statistics + +from docreader.config import CONFIG +from docreader.models.document import Document +from docreader.parser.base_parser import BaseParser +from docreader.parser.concurrency import parser_worker_limit logger = logging.getLogger(__name__) +def _env_float(name: str, default: float) -> float: + v = os.environ.get(name) + if v is None or not str(v).strip(): + return default + try: + return float(v) + except ValueError: + return default + + +def _env_int(name: str, default: int) -> int: + v = os.environ.get(name) + if v is None or not str(v).strip(): + return default + try: + return int(str(v).strip()) + except ValueError: + return default + + +def _env_bool(name: str, default: bool) -> bool: + v = os.environ.get(name) + if v is None or not str(v).strip(): + return default + return str(v).strip().lower() in {"1", "true", "yes", "y", "on"} + + +# A page whose image objects cover at least this fraction of the page area is +# treated as scanned (image-dominated). Native digital pages measure ~0.0-0.05; +# scanned pages measure ~1.0+, so 0.5 leaves a wide safety margin. +SCAN_IMAGE_AREA_RATIO = _env_float("DOCREADER_PDF_SCAN_IMAGE_RATIO", 0.5) +# Below this many characters a page is considered to have no usable text layer. +SCAN_MIN_CHARS_PER_PAGE = _env_int("DOCREADER_PDF_SCAN_MIN_CHARS", 10) +# A near-empty-text page is only rendered as an image if it actually contains +# some image content (avoids rendering genuinely blank pages). +_LOW_TEXT_IMAGE_RATIO = 0.1 + +# --- Embedded figure extraction (text pages) ------------------------------ +# Native pages can embed figures/charts. We surface them as image references so +# the Go App can OCR/caption them (docreader does not caption). Logos, icons, +# watermarks and tiny decorations are filtered out by size, page-area share and +# cross-page repetition. +EXTRACT_EMBEDDED_IMAGES = _env_bool("DOCREADER_PDF_EXTRACT_EMBEDDED_IMAGES", True) +# Minimum pixel width AND height for an embedded image to be kept. +EMBED_MIN_PIXELS = _env_int("DOCREADER_PDF_EMBED_MIN_PIXELS", 80) +# Minimum share of the page area for an embedded image to be kept. +EMBED_MIN_AREA_RATIO = _env_float("DOCREADER_PDF_EMBED_MIN_AREA_RATIO", 0.01) +# An identical image appearing on at least this fraction of text pages is +# treated as a running logo/watermark and dropped. +EMBED_REPEAT_PAGE_FRAC = _env_float("DOCREADER_PDF_EMBED_REPEAT_PAGE_FRAC", 0.5) +# Hard cap on the number of embedded images extracted per document. +EMBED_MAX_IMAGES = _env_int("DOCREADER_PDF_EMBED_MAX_IMAGES", 50) + +# --- Layout-aware text extraction (native text pages) --------------------- +# Reconstruct reading order with a geometric XY-cut so multi-column pages are +# linearised column-by-column instead of line-interleaved. +LAYOUT_ORDERING = _env_bool("DOCREADER_PDF_LAYOUT_ORDERING", True) +# Promote visually larger lines to markdown headings (font-size proxy = rect +# height relative to the page's median line height). +DETECT_HEADINGS = _env_bool("DOCREADER_PDF_DETECT_HEADINGS", True) +# Drop invisible (render-mode 3), off-page and degenerate text — a cheap guard +# against hidden-text prompt injection and OCR artefacts. +FILTER_HIDDEN_TEXT = _env_bool("DOCREADER_PDF_FILTER_HIDDEN_TEXT", True) + + def _close_pdfium_resource(resource) -> None: close = getattr(resource, "close", None) if close: @@ -23,27 +105,612 @@ def _normalize_image_quality(quality: int) -> int: return min(95, max(1, quality)) -class PDFScannedParser(BaseParser): - """Fallback parser for scanned PDFs. - - If the primary parser extracts no text (e.g. Markitdown on a scanned PDF), - this parser converts each page into a JPEG image. The Go App will then - perform OCR on the extracted images. +def _classify_page(image_area_ratio: float, text_len: int) -> str: + """Classify a page as ``"scanned"`` or ``"text"``. + + Image-area coverage is the primary signal; a sparse text layer combined with + some image content is the secondary signal. """ - + if image_area_ratio >= SCAN_IMAGE_AREA_RATIO: + return "scanned" + if text_len < SCAN_MIN_CHARS_PER_PAGE and image_area_ratio >= _LOW_TEXT_IMAGE_RATIO: + return "scanned" + return "text" + + +def _page_image_area_ratio(page, raw) -> float: + """Return the fraction of the page area covered by image objects. + + Overlapping images can push the ratio above 1.0; callers only compare it + against a threshold so that is harmless. + """ + width, height = page.get_size() + page_area = float(width) * float(height) + if page_area <= 0: + return 0.0 + + image_area = 0.0 + for obj in page.get_objects(): + try: + if obj.type == raw.FPDF_PAGEOBJ_IMAGE: + left, bottom, right, top = obj.get_bounds() + image_area += abs((right - left) * (top - bottom)) + except Exception: + continue + return image_area / page_area + + +def _extract_page_text(page) -> str: + """Plain top-to-bottom text extraction (fallback path).""" + textpage = None + try: + textpage = page.get_textpage() + return textpage.get_text_range() + finally: + _close_pdfium_resource(textpage) + + +def _collect_invisible_boxes(page, raw) -> list: + """Bounding boxes of invisible (render-mode 3) text objects on the page.""" + boxes: list = [] + try: + for obj in page.get_objects(): + if obj.type != raw.FPDF_PAGEOBJ_TEXT: + continue + try: + mode = raw.FPDFTextObj_GetTextRenderMode(obj.raw) + except Exception: + continue + if mode != raw.FPDF_TEXTRENDERMODE_INVISIBLE: + continue + try: + left, bottom, right, top = obj.get_bounds() + except Exception: + continue + boxes.append( + (min(left, right), min(bottom, top), max(left, right), max(bottom, top)) + ) + except Exception: + return [] + return boxes + + +def _point_in_boxes(x: float, y: float, boxes: list) -> bool: + for x0, y0, x1, y1 in boxes: + if x0 <= x <= x1 and y0 <= y <= y1: + return True + return False + + +def _page_chars(textpage, page, raw) -> tuple: + """Return ``(chars, page_width)`` with hidden/off-page glyphs filtered. + + Working at the glyph level (instead of pdfium rect segments) keeps mixed + CJK + Latin/number lines in their true left-to-right order, which the + rect-level ``get_text_bounded`` API scrambles. + """ + n = textpage.count_chars() + if n <= 0: + return [], 0.0 + width, height = page.get_size() + invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else [] + + chars: list = [] + for i in range(n): + try: + left, bottom, right, top = textpage.get_charbox(i) + except Exception: + continue + ch = textpage.get_text_range(i, 1) + if ch in ("\r", "\n"): + continue + x0, x1 = (left, right) if left <= right else (right, left) + y0, y1 = (bottom, top) if bottom <= top else (top, bottom) + if FILTER_HIDDEN_TEXT: + if x1 < 0 or x0 > width or y1 < 0 or y0 > height: + continue # off-page glyph + if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible): + continue # covered by an invisible text object + chars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch}) + return chars, width + + +def _find_split(items: list, axis: str, min_gap: float): + """Return a coordinate at the widest clean gap on ``axis`` ('x'), or None. + + A "clean" gap means no item interval bridges it — i.e. a full-height column + gutter. Used to detect multi-column layouts. + """ + lo, hi = ("x0", "x1") if axis == "x" else ("y0", "y1") + intervals = sorted(((s[lo], s[hi]) for s in items), key=lambda iv: iv[0]) + cur_end = intervals[0][1] + best_gap, best_cut = 0.0, None + for a, b in intervals[1:]: + gap = a - cur_end + if gap >= min_gap and gap > best_gap: + best_gap, best_cut = gap, cur_end + gap / 2 + if b > cur_end: + cur_end = b + return best_cut + + +def _split_columns(chars: list, scale: float, width: float, depth: int = 0) -> list: + """Split glyphs into reading-order columns at full-height gutters.""" + if len(chars) <= 1 or depth > 10: + return [chars] + min_gap = max(scale * 2.5, width * 0.04) + cut = _find_split(chars, "x", min_gap) + if cut is None: + return [chars] + left = [c for c in chars if (c["x0"] + c["x1"]) / 2 < cut] + right = [c for c in chars if (c["x0"] + c["x1"]) / 2 >= cut] + if not left or not right: + return [chars] + return _split_columns(left, scale, width, depth + 1) + _split_columns( + right, scale, width, depth + 1 + ) + + +def _group_lines(chars: list) -> list: + """Group a column's glyphs into lines (top-to-bottom, glyphs sorted by x).""" + if not chars: + return [] + heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0] + med_h = statistics.median(heights) if heights else 1.0 + + ordered = sorted(chars, key=lambda c: -(c["y0"] + c["y1"]) / 2) + lines: list = [] + cur: list = [] + ref = None + for c in ordered: + yc = (c["y0"] + c["y1"]) / 2 + if ref is None or abs(yc - ref) <= 0.5 * med_h: + cur.append(c) + ref = yc if ref is None else ref + else: + lines.append(cur) + cur = [c] + ref = yc + if cur: + lines.append(cur) + + out: list = [] + for ln in lines: + ln_sorted = sorted(ln, key=lambda c: c["x0"]) + text = "".join(c["ch"] for c in ln_sorted).strip() + if not text: + continue + hs = [c["y1"] - c["y0"] for c in ln_sorted if c["y1"] - c["y0"] > 0] + out.append({"h": statistics.median(hs) if hs else med_h, "text": text}) + return out + + +def _segments_to_markdown(lines: list) -> str: + """Render merged lines to text, promoting visually large lines to headings.""" + if not lines: + return "" + body = statistics.median([ln["h"] for ln in lines]) + + def level(ln) -> int: + txt = ln["text"] + if not DETECT_HEADINGS or body <= 0 or len(txt) > 80: + return 0 + if txt[-1:] in ".。!!??,,;;::": + return 0 + r = ln["h"] / body + if r >= 2.0: + return 1 + if r >= 1.6: + return 2 + if r >= 1.35: + return 3 + return 0 + + levels = [level(ln) for ln in lines] + # If too many lines qualify, the font sizes are too uniform/noisy to trust. + if sum(1 for x in levels if x) > max(1, int(0.4 * len(lines))): + levels = [0] * len(lines) + + out = [] + for ln, lv in zip(lines, levels): + out.append(("#" * lv + " " + ln["text"]) if lv else ln["text"]) + return "\n".join(out) + + +def _extract_layout_text(page, raw) -> str: + """Layout-aware extraction: reading order + headings + hidden-text filter. + + Falls back to plain extraction on any failure so a single odd page never + breaks the document. + """ + textpage = None + try: + textpage = page.get_textpage() + chars, width = _page_chars(textpage, page, raw) + if not chars: + return "" + heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0] + scale = (statistics.median(heights) if heights else 1.0) or 1.0 + blocks = [] + for col in _split_columns(chars, scale, width): + md = _segments_to_markdown(_group_lines(col)) + if md: + blocks.append(md) + return "\n".join(blocks) + except Exception: + logger.debug("layout extraction failed; using plain text", exc_info=True) + return _extract_page_text(page) + finally: + _close_pdfium_resource(textpage) + + +def _effective_scale(page, scale: float, max_edge: int) -> float: + """Reduce ``scale`` so the rendered long edge never exceeds ``max_edge`` px. + + Some scanned PDFs declare enormous page boxes; rendering those at the raw + DPI scale produces 100+ MP JPEGs that exceed the gRPC message limit and are + far higher resolution than OCR needs. + """ + if max_edge <= 0: + return scale + width, height = page.get_size() + longest_pt = max(float(width), float(height)) + if longest_pt <= 0: + return scale + return min(scale, max_edge / longest_pt) + + +def _render_page_to_jpeg(page, scale: float, quality: int, max_edge: int = 0) -> bytes: + bitmap = None + try: + bitmap = page.render(scale=_effective_scale(page, scale, max_edge)) + img_obj = bitmap.to_pil() + if img_obj.mode != "RGB": + img_obj = img_obj.convert("RGB") + buf = io.BytesIO() + img_obj.save(buf, format="JPEG", quality=quality, optimize=True) + return buf.getvalue() + finally: + _close_pdfium_resource(bitmap) + + +# --- Parallel scanned-page rendering -------------------------------------- +# pdfium is NOT thread-safe (concurrent get_page on one document crashes), so +# we parallelise across *processes*: each worker opens its own PdfDocument from +# a temp file and renders an assigned slice of pages. This turns the serial +# per-page render (the dominant cost for big scanned PDFs — hours on +# CPU-constrained containers) into a near-linear speedup. + +# Per-worker document handle, populated by the pool initializer. +_WORKER_RENDER_DOC = None + + +def _render_pool_init(pdf_path: str) -> None: + global _WORKER_RENDER_DOC + import pypdfium2 as pdfium + + with open(pdf_path, "rb") as f: + _WORKER_RENDER_DOC = pdfium.PdfDocument(f.read()) + + +def _render_pool_task(args): + index, scale, quality, max_edge = args + page = _WORKER_RENDER_DOC[index] + try: + return index, _render_page_to_jpeg(page, scale, quality, max_edge) + finally: + _close_pdfium_resource(page) + + +def _select_mp_context(): + """Pick the safest available multiprocessing start method. + + ``forkserver`` forks workers from a clean, single-threaded server process, + avoiding the fork-in-a-multithreaded-process hazards of the gRPC server. + Falls back to ``fork`` and finally returns ``None`` (serial) when neither + is available (e.g. Windows/dev). + """ + import multiprocessing as mp + + for method in ("forkserver", "fork"): + try: + return mp.get_context(method) + except ValueError: + continue + return None + + +def _render_pages_parallel( + content: bytes, indices: list, scale: float, quality: int, max_edge: int, workers: int +) -> dict | None: + """Render ``indices`` in parallel. Returns ``{index: jpeg_bytes}`` or None. + + Returns None to signal the caller to fall back to serial rendering (when + parallelism is disabled, only one page is requested, or no usable + multiprocessing start method exists). + """ + if workers <= 1 or len(indices) <= 1: + return None + ctx = _select_mp_context() + if ctx is None: + return None + + import tempfile + from concurrent.futures import ProcessPoolExecutor + + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + prefix="docreader_render_", suffix=".pdf", delete=False + ) as tmp: + tmp.write(content) + tmp_path = tmp.name + + max_workers = min(workers, len(indices)) + tasks = [(i, scale, quality, max_edge) for i in indices] + result: dict = {} + with ProcessPoolExecutor( + max_workers=max_workers, + mp_context=ctx, + initializer=_render_pool_init, + initargs=(tmp_path,), + ) as ex: + for index, jpeg in ex.map(_render_pool_task, tasks, chunksize=4): + result[index] = jpeg + return result + except Exception: + logger.warning( + "parallel page rendering failed; falling back to serial", + exc_info=True, + ) + return None + finally: + if tmp_path: + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _render_scanned_pages( + pdf, content: bytes, indices: list, scale: float, quality: int, max_edge: int +) -> dict: + """Render the given (scanned) page indices to JPEG bytes. + + Tries process-parallel rendering first (big win for large scanned PDFs), + transparently falling back to serial rendering on the already-open ``pdf`` + handle when parallelism is unavailable or fails. + """ + parallel = _render_pages_parallel( + content, indices, scale, quality, max_edge, CONFIG.pdf_render_parallelism + ) + if parallel is not None: + return parallel + + out: dict = {} + for i in indices: + page = pdf[i] + try: + out[i] = _render_page_to_jpeg(page, scale, quality, max_edge) + finally: + _close_pdfium_resource(page) + return out + + +def _select_embedded_images( + meta: list, + num_text_pages: int, + *, + min_pixels: int = EMBED_MIN_PIXELS, + min_area_ratio: float = EMBED_MIN_AREA_RATIO, + repeat_frac: float = EMBED_REPEAT_PAGE_FRAC, + max_images: int = EMBED_MAX_IMAGES, +) -> list: + """Decide which embedded-image candidates to keep (pure function). + + ``meta`` is a list of dicts with keys ``page``, ``width``, ``height``, + ``area_ratio`` and ``hash``. Returns the indices (into ``meta``) to keep, + after filtering by size, page-area share, cross-page repetition (logos / + watermarks), exact in-page duplicates and a hard count cap. + """ + from collections import defaultdict + + hash_pages = defaultdict(set) + for m in meta: + hash_pages[m["hash"]].add(m["page"]) + + repeat_threshold = max(2, int(num_text_pages * repeat_frac)) if num_text_pages else 2 + banned = {h for h, pages in hash_pages.items() if len(pages) >= repeat_threshold} + + kept: list = [] + seen = set() + for idx, m in enumerate(meta): + if m["area_ratio"] < min_area_ratio: + continue + if m["width"] < min_pixels or m["height"] < min_pixels: + continue + if m["hash"] in banned: + continue + key = (m["page"], m["hash"]) + if key in seen: + continue + seen.add(key) + kept.append(idx) + if len(kept) >= max_images: + break + return kept + + +def _extract_embedded_images(pdf, classes, raw, base_name: str, quality: int) -> dict: + """Extract filtered embedded figures from native text pages. + + Returns ``{page_index: [(ref_path, base64_jpeg, y_top), ...]}`` ordered so + callers can place figures after the page text in top-to-bottom order. + """ + import hashlib + + text_indices = [i for i, c in enumerate(classes) if c == "text"] + if not text_indices: + return {} + + candidates: list = [] # parallel to meta; holds heavy pixel data + meta: list = [] + for i in text_indices: + page = pdf[i] + try: + width, height = page.get_size() + page_area = float(width) * float(height) + if page_area <= 0: + continue + for obj in page.get_objects(): + if obj.type != raw.FPDF_PAGEOBJ_IMAGE: + continue + try: + left, bottom, right, top = obj.get_bounds() + except Exception: + continue + area_ratio = abs((right - left) * (top - bottom)) / page_area + if area_ratio < EMBED_MIN_AREA_RATIO: + continue # cheap skip before decoding (logos/decorations) + try: + pil = obj.get_bitmap().to_pil() + except Exception: + continue + content_hash = hashlib.md5(pil.tobytes()).hexdigest() + candidates.append((i, top, pil)) + meta.append( + { + "page": i, + "width": pil.width, + "height": pil.height, + "area_ratio": area_ratio, + "hash": content_hash, + } + ) + finally: + _close_pdfium_resource(page) + + kept_idx = _select_embedded_images(meta, len(text_indices)) + if not kept_idx: + return {} + + from collections import defaultdict + + result: dict = defaultdict(list) + per_page_count: dict = defaultdict(int) + max_edge = CONFIG.pdf_render_max_edge + for idx in kept_idx: + page_i, y_top, pil = candidates[idx] + if pil.mode not in ("RGB", "L"): + pil = pil.convert("RGB") + if max_edge > 0 and max(pil.size) > max_edge: + ratio = max_edge / max(pil.size) + pil = pil.resize( + (max(1, int(pil.width * ratio)), max(1, int(pil.height * ratio))) + ) + buf = io.BytesIO() + pil.save(buf, format="JPEG", quality=quality, optimize=True) + per_page_count[page_i] += 1 + fname = f"{base_name}_p{page_i+1}_img{per_page_count[page_i]}.jpg" + ref_path = f"images/{fname}" + result[page_i].append( + (ref_path, base64.b64encode(buf.getvalue()).decode("utf-8"), y_top) + ) + + # Top-to-bottom within each page (PDF y grows upward, so larger y first). + for page_i in result: + result[page_i].sort(key=lambda item: item[2], reverse=True) + return result + + +def estimate_scanned_fraction(content: bytes, sample: int = 12) -> float: + """Return the fraction of (sampled) pages that look image-dominated. + + Used by alternative engines (e.g. liteparse) that lack image-object access + to decide whether a PDF is scanned, applying the same image-area signal the + builtin router uses. Samples up to ``sample`` pages for speed on big PDFs. + """ + import pypdfium2 as pdfium + import pypdfium2.raw as pdfium_r + + pdf = pdfium.PdfDocument(content) + try: + page_count = len(pdf) + if page_count <= 0: + return 0.0 + step = max(1, page_count // sample) + indices = list(range(0, page_count, step)) + scanned = 0 + for i in indices: + page = pdf[i] + try: + ratio = _page_image_area_ratio(page, pdfium_r) + finally: + _close_pdfium_resource(page) + if ratio >= SCAN_IMAGE_AREA_RATIO: + scanned += 1 + return scanned / len(indices) if indices else 0.0 + finally: + _close_pdfium_resource(pdf) + + +def _strip_repeating_lines(texts: list, classes: list) -> list: + """Remove running headers/footers that repeat across most text pages. + + Conservative: only the first/last non-empty line of each text page is a + candidate, the line must be short, and it must appear on at least 60% of the + text pages (and there must be enough pages to judge). Mirrors DeepDoc's + cross-page "garbage set" idea without risking removal of real content. + """ + from collections import Counter + + text_indices = [i for i, c in enumerate(classes) if c == "text"] + if len(text_indices) < 4: + return list(texts) + + counter: Counter = Counter() + for i in text_indices: + lines = [ln.strip() for ln in texts[i].splitlines() if ln.strip()] + if not lines: + continue + for edge in {lines[0], lines[-1]}: + if len(edge) <= 80: + counter[edge] += 1 + + threshold = max(2, int(len(text_indices) * 0.6)) + repeating = {line for line, count in counter.items() if count >= threshold} + if not repeating: + return list(texts) + + cleaned = [] + for i, text in enumerate(texts): + if classes[i] != "text": + cleaned.append(text) + continue + kept = [ln for ln in text.splitlines() if ln.strip() not in repeating] + cleaned.append("\n".join(kept)) + return cleaned + + +class PDFScannedParser(BaseParser): + """Render every PDF page to a JPEG image. + + Used as a robust last-resort fallback and for image-only PDFs. The Go App + performs OCR on the extracted page images. + """ + def parse_into_text(self, content: bytes) -> Document: import pypdfium2 as pdfium images = {} markdown_lines = [] - base_name = os.path.splitext(self.file_name or "document")[0] - + logger.info( "PDFScannedParser: Rendering PDF pages to JPEG images for %s", self.file_name, ) - + try: with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers): pdf = pdfium.PdfDocument(content) @@ -52,57 +719,160 @@ class PDFScannedParser(BaseParser): scale = max(1, CONFIG.pdf_render_dpi) / 72 quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality) - for i in range(page_count): - page = pdf[i] - bitmap = None - try: - bitmap = page.render(scale=scale) - img_obj = bitmap.to_pil() - if img_obj.mode != "RGB": - img_obj = img_obj.convert("RGB") - - img_byte_arr = io.BytesIO() - img_obj.save( - img_byte_arr, - format="JPEG", - quality=quality, - optimize=True, - ) - img_bytes = img_byte_arr.getvalue() - finally: - _close_pdfium_resource(bitmap) - _close_pdfium_resource(page) - - page_filename = f"{base_name}_page_{i+1}.jpg" - ref_path = f"images/{page_filename}" - - markdown_lines.append(f"![{page_filename}]({ref_path})") - images[ref_path] = base64.b64encode(img_bytes).decode("utf-8") + rendered = _render_scanned_pages( + pdf, + content, + list(range(page_count)), + scale, + quality, + CONFIG.pdf_render_max_edge, + ) finally: _close_pdfium_resource(pdf) - + + for i in range(page_count): + page_filename = f"{base_name}_page_{i+1}.jpg" + ref_path = f"images/{page_filename}" + markdown_lines.append(f"![{page_filename}]({ref_path})") + images[ref_path] = base64.b64encode(rendered[i]).decode("utf-8") + text = "\n\n".join(markdown_lines) return Document( - content=text, - images=images, + content=text, + images=images, metadata={ "image_source_type": "scanned_pdf", - "page_count": page_count - } + "page_count": page_count, + }, ) except Exception as e: logger.exception("PDFScannedParser failed to parse PDF: %s", e) raise e -class PDFParser(FirstParser): - """PDF Parser using chain of responsibility pattern - - Attempts to parse PDF files using multiple parser backends in order: - 1. MinerUParser - Primary parser for PDF documents (if enabled) - 2. MarkitdownParser - Fallback parser if MinerU fails - 3. PDFScannedParser - Final fallback for scanned PDFs - - The first successful parser result will be returned. + +class PDFParser(BaseParser): + """Per-page router between native text extraction and scanned rendering. + + For each page: + * native text page -> keep its text layer (fast, pypdfium2) + * scanned page -> render to JPEG, tag ``image_source_type=scanned_pdf`` + so the Go App OCRs it + + Hybrid documents interleave both in reading order. On any unexpected error + the parser falls back to rendering all pages as images (safe last resort). """ - # Parser classes to try in order (chain of responsibility pattern) - _parser_cls = (MarkitdownParser, PDFScannedParser) + + def parse_into_text(self, content: bytes) -> Document: + try: + return self._route(content) + except Exception: + logger.exception( + "PDFParser: per-page routing failed for %s; " + "falling back to full image rendering", + self.file_name, + ) + return PDFScannedParser( + file_name=self.file_name, file_type=self.file_type + ).parse_into_text(content) + + def _route(self, content: bytes) -> Document: + import pypdfium2 as pdfium + import pypdfium2.raw as pdfium_r + + base_name = os.path.splitext(self.file_name or "document")[0] + scale = max(1, CONFIG.pdf_render_dpi) / 72 + quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality) + + pdf = pdfium.PdfDocument(content) + images: dict = {} + try: + page_count = len(pdf) + + # Pass 1: cheap text extraction + image-area classification. + texts: list = [] + classes: list = [] + for i in range(page_count): + page = pdf[i] + try: + plain = _extract_page_text(page) + ratio = _page_image_area_ratio(page, pdfium_r) + cls = _classify_page(ratio, len(plain.strip())) + # Layout reconstruction only pays off (and is only spent) on + # native text pages; scanned pages are rendered, not read. + if cls == "text" and LAYOUT_ORDERING: + text = _extract_layout_text(page, pdfium_r) or plain + else: + text = plain + finally: + _close_pdfium_resource(page) + texts.append(text) + classes.append(cls) + + texts = _strip_repeating_lines(texts, classes) + scanned_indices = [i for i, c in enumerate(classes) if c == "scanned"] + + # Pass 2: render only the scanned pages (heavy work, rate-limited). + if scanned_indices: + with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers): + rendered = _render_scanned_pages( + pdf, + content, + scanned_indices, + scale, + quality, + CONFIG.pdf_render_max_edge, + ) + for i, img_bytes in rendered.items(): + ref_path = f"images/{base_name}_page_{i+1}.jpg" + images[ref_path] = base64.b64encode(img_bytes).decode("utf-8") + + # Pass 3: extract embedded figures from native text pages so the Go + # App can OCR/caption them (logos/watermarks/tiny images filtered). + embedded: dict = {} + if EXTRACT_EMBEDDED_IMAGES: + embedded = _extract_embedded_images( + pdf, classes, pdfium_r, base_name, quality + ) + for refs in embedded.values(): + for ref_path, b64, _y in refs: + images[ref_path] = b64 + finally: + _close_pdfium_resource(pdf) + + # Assemble markdown in reading order. + embedded_count = 0 + blocks = [] + for i in range(page_count): + if classes[i] == "scanned": + page_filename = f"{base_name}_page_{i+1}.jpg" + blocks.append(f"![{page_filename}](images/{page_filename})") + else: + stripped = texts[i].strip() + if stripped: + blocks.append(stripped) + for ref_path, _b64, _y in embedded.get(i, []): + fname = os.path.basename(ref_path) + blocks.append(f"![{fname}]({ref_path})") + embedded_count += 1 + + content_text = "\n\n".join(blocks).strip() + + metadata = { + "page_count": page_count, + "scanned_page_count": len(scanned_indices), + "text_page_count": page_count - len(scanned_indices), + "embedded_image_count": embedded_count, + "image_source_type": "scanned_pdf" if scanned_indices else "pdf_text_layer", + } + + logger.info( + "PDFParser: %s -> %d pages (%d scanned, %d text), " + "embedded_images=%d, content_len=%d", + self.file_name, + page_count, + len(scanned_indices), + page_count - len(scanned_indices), + embedded_count, + len(content_text), + ) + return Document(content=content_text, images=images, metadata=metadata) diff --git a/docreader/parser/registry.py b/docreader/parser/registry.py index 18580b32..f51b9be7 100644 --- a/docreader/parser/registry.py +++ b/docreader/parser/registry.py @@ -7,6 +7,7 @@ from docreader.parser.docx2_parser import Docx2Parser from docreader.parser.excel_parser import ExcelParser from docreader.parser.image_parser import ImageParser from docreader.parser.markdown_parser import MarkdownParser +from docreader.parser.liteparse_parser import LiteParseParser, liteparse_available from docreader.parser.markitdown_parser import MarkitdownParser from docreader.parser.pdf_parser import PDFParser @@ -149,6 +150,14 @@ def _build_default_registry() -> ParserEngineRegistry: description="MarkItDown 解析引擎(微软 MarkItDown 库)", ) + reg.register( + "liteparse", + {"pdf": LiteParseParser}, + description="LiteParse 解析引擎(快速空间阅读顺序,适合数字版 PDF)", + check_available=liteparse_available, + unavailable_hint="liteparse 未安装", + ) + # NOTE: Engine listing is managed by Go-side engine registry # (docparser.ListAllEngines). The Python list_engines method is kept for # backward compatibility with the gRPC ListEngines RPC but the Go app diff --git a/docreader/proto/docreader.pb.go b/docreader/proto/docreader.pb.go index 751ae9b5..f071eecb 100644 --- a/docreader/proto/docreader.pb.go +++ b/docreader/proto/docreader.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 -// protoc v6.33.4 +// protoc-gen-go v1.36.7 +// protoc v5.29.3 // source: docreader.proto package proto @@ -318,6 +318,167 @@ func (x *ReadResponse) GetError() string { return "" } +// Metadata header for a streamed read. Sent exactly once, before any image. +type ReadStreamMeta struct { + state protoimpl.MessageState `protogen:"open.v1"` + MarkdownContent string `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"` + ImageDirPath string `protobuf:"bytes,2,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"` + Metadata map[string]string `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` + ImageCount uint32 `protobuf:"varint,5,opt,name=image_count,json=imageCount,proto3" json:"image_count,omitempty"` // best-effort total image count (0 if unknown) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadStreamMeta) Reset() { + *x = ReadStreamMeta{} + mi := &file_docreader_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadStreamMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadStreamMeta) ProtoMessage() {} + +func (x *ReadStreamMeta) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadStreamMeta.ProtoReflect.Descriptor instead. +func (*ReadStreamMeta) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{4} +} + +func (x *ReadStreamMeta) GetMarkdownContent() string { + if x != nil { + return x.MarkdownContent + } + return "" +} + +func (x *ReadStreamMeta) GetImageDirPath() string { + if x != nil { + return x.ImageDirPath + } + return "" +} + +func (x *ReadStreamMeta) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ReadStreamMeta) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *ReadStreamMeta) GetImageCount() uint32 { + if x != nil { + return x.ImageCount + } + return 0 +} + +// One frame of a ReadStream. The first frame carries `meta`; every subsequent +// frame carries a single `image`. +type ReadStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Payload: + // + // *ReadStreamResponse_Meta + // *ReadStreamResponse_Image + Payload isReadStreamResponse_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadStreamResponse) Reset() { + *x = ReadStreamResponse{} + mi := &file_docreader_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadStreamResponse) ProtoMessage() {} + +func (x *ReadStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadStreamResponse.ProtoReflect.Descriptor instead. +func (*ReadStreamResponse) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{5} +} + +func (x *ReadStreamResponse) GetPayload() isReadStreamResponse_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *ReadStreamResponse) GetMeta() *ReadStreamMeta { + if x != nil { + if x, ok := x.Payload.(*ReadStreamResponse_Meta); ok { + return x.Meta + } + } + return nil +} + +func (x *ReadStreamResponse) GetImage() *ImageRef { + if x != nil { + if x, ok := x.Payload.(*ReadStreamResponse_Image); ok { + return x.Image + } + } + return nil +} + +type isReadStreamResponse_Payload interface { + isReadStreamResponse_Payload() +} + +type ReadStreamResponse_Meta struct { + Meta *ReadStreamMeta `protobuf:"bytes,1,opt,name=meta,proto3,oneof"` +} + +type ReadStreamResponse_Image struct { + Image *ImageRef `protobuf:"bytes,2,opt,name=image,proto3,oneof"` +} + +func (*ReadStreamResponse_Meta) isReadStreamResponse_Payload() {} + +func (*ReadStreamResponse_Image) isReadStreamResponse_Payload() {} + type ListEnginesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` ConfigOverrides map[string]string `protobuf:"bytes,1,rep,name=config_overrides,json=configOverrides,proto3" json:"config_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` @@ -327,7 +488,7 @@ type ListEnginesRequest struct { func (x *ListEnginesRequest) Reset() { *x = ListEnginesRequest{} - mi := &file_docreader_proto_msgTypes[4] + mi := &file_docreader_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -339,7 +500,7 @@ func (x *ListEnginesRequest) String() string { func (*ListEnginesRequest) ProtoMessage() {} func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message { - mi := &file_docreader_proto_msgTypes[4] + mi := &file_docreader_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -352,7 +513,7 @@ func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEnginesRequest.ProtoReflect.Descriptor instead. func (*ListEnginesRequest) Descriptor() ([]byte, []int) { - return file_docreader_proto_rawDescGZIP(), []int{4} + return file_docreader_proto_rawDescGZIP(), []int{6} } func (x *ListEnginesRequest) GetConfigOverrides() map[string]string { @@ -375,7 +536,7 @@ type ParserEngineInfo struct { func (x *ParserEngineInfo) Reset() { *x = ParserEngineInfo{} - mi := &file_docreader_proto_msgTypes[5] + mi := &file_docreader_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -387,7 +548,7 @@ func (x *ParserEngineInfo) String() string { func (*ParserEngineInfo) ProtoMessage() {} func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message { - mi := &file_docreader_proto_msgTypes[5] + mi := &file_docreader_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -400,7 +561,7 @@ func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ParserEngineInfo.ProtoReflect.Descriptor instead. func (*ParserEngineInfo) Descriptor() ([]byte, []int) { - return file_docreader_proto_rawDescGZIP(), []int{5} + return file_docreader_proto_rawDescGZIP(), []int{7} } func (x *ParserEngineInfo) GetName() string { @@ -447,7 +608,7 @@ type ListEnginesResponse struct { func (x *ListEnginesResponse) Reset() { *x = ListEnginesResponse{} - mi := &file_docreader_proto_msgTypes[6] + mi := &file_docreader_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -459,7 +620,7 @@ func (x *ListEnginesResponse) String() string { func (*ListEnginesResponse) ProtoMessage() {} func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message { - mi := &file_docreader_proto_msgTypes[6] + mi := &file_docreader_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -472,7 +633,7 @@ func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEnginesResponse.ProtoReflect.Descriptor instead. func (*ListEnginesResponse) Descriptor() ([]byte, []int) { - return file_docreader_proto_rawDescGZIP(), []int{6} + return file_docreader_proto_rawDescGZIP(), []int{8} } func (x *ListEnginesResponse) GetEngines() []*ParserEngineInfo { @@ -520,7 +681,21 @@ const file_docreader_proto_rawDesc = "" + "\x05error\x18\x05 \x01(\tR\x05error\x1a;\n" + "\rMetadataEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb7\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x9a\x02\n" + + "\x0eReadStreamMeta\x12)\n" + + "\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x12$\n" + + "\x0eimage_dir_path\x18\x02 \x01(\tR\fimageDirPath\x12C\n" + + "\bmetadata\x18\x03 \x03(\v2'.docreader.ReadStreamMeta.MetadataEntryR\bmetadata\x12\x14\n" + + "\x05error\x18\x04 \x01(\tR\x05error\x12\x1f\n" + + "\vimage_count\x18\x05 \x01(\rR\n" + + "imageCount\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"}\n" + + "\x12ReadStreamResponse\x12/\n" + + "\x04meta\x18\x01 \x01(\v2\x19.docreader.ReadStreamMetaH\x00R\x04meta\x12+\n" + + "\x05image\x18\x02 \x01(\v2\x13.docreader.ImageRefH\x00R\x05imageB\t\n" + + "\apayload\"\xb7\x01\n" + "\x12ListEnginesRequest\x12]\n" + "\x10config_overrides\x18\x01 \x03(\v22.docreader.ListEnginesRequest.ConfigOverridesEntryR\x0fconfigOverrides\x1aB\n" + "\x14ConfigOverridesEntry\x12\x10\n" + @@ -534,9 +709,11 @@ const file_docreader_proto_rawDesc = "" + "\tavailable\x18\x04 \x01(\bR\tavailable\x12-\n" + "\x12unavailable_reason\x18\x05 \x01(\tR\x11unavailableReason\"L\n" + "\x13ListEnginesResponse\x125\n" + - "\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines2\x96\x01\n" + + "\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines2\xdf\x01\n" + "\tDocReader\x129\n" + - "\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n" + + "\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" + + "\n" + + "ReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x000\x01\x12N\n" + "\vListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3" var ( @@ -551,35 +728,43 @@ func file_docreader_proto_rawDescGZIP() []byte { return file_docreader_proto_rawDescData } -var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_docreader_proto_goTypes = []any{ (*ReadConfig)(nil), // 0: docreader.ReadConfig (*ReadRequest)(nil), // 1: docreader.ReadRequest (*ImageRef)(nil), // 2: docreader.ImageRef (*ReadResponse)(nil), // 3: docreader.ReadResponse - (*ListEnginesRequest)(nil), // 4: docreader.ListEnginesRequest - (*ParserEngineInfo)(nil), // 5: docreader.ParserEngineInfo - (*ListEnginesResponse)(nil), // 6: docreader.ListEnginesResponse - nil, // 7: docreader.ReadConfig.ParserEngineOverridesEntry - nil, // 8: docreader.ReadResponse.MetadataEntry - nil, // 9: docreader.ListEnginesRequest.ConfigOverridesEntry + (*ReadStreamMeta)(nil), // 4: docreader.ReadStreamMeta + (*ReadStreamResponse)(nil), // 5: docreader.ReadStreamResponse + (*ListEnginesRequest)(nil), // 6: docreader.ListEnginesRequest + (*ParserEngineInfo)(nil), // 7: docreader.ParserEngineInfo + (*ListEnginesResponse)(nil), // 8: docreader.ListEnginesResponse + nil, // 9: docreader.ReadConfig.ParserEngineOverridesEntry + nil, // 10: docreader.ReadResponse.MetadataEntry + nil, // 11: docreader.ReadStreamMeta.MetadataEntry + nil, // 12: docreader.ListEnginesRequest.ConfigOverridesEntry } var file_docreader_proto_depIdxs = []int32{ - 7, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry - 0, // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig - 2, // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef - 8, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry - 9, // 4: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry - 5, // 5: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo - 1, // 6: docreader.DocReader.Read:input_type -> docreader.ReadRequest - 4, // 7: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest - 3, // 8: docreader.DocReader.Read:output_type -> docreader.ReadResponse - 6, // 9: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse - 8, // [8:10] is the sub-list for method output_type - 6, // [6:8] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 9, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry + 0, // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig + 2, // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef + 10, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry + 11, // 4: docreader.ReadStreamMeta.metadata:type_name -> docreader.ReadStreamMeta.MetadataEntry + 4, // 5: docreader.ReadStreamResponse.meta:type_name -> docreader.ReadStreamMeta + 2, // 6: docreader.ReadStreamResponse.image:type_name -> docreader.ImageRef + 12, // 7: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry + 7, // 8: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo + 1, // 9: docreader.DocReader.Read:input_type -> docreader.ReadRequest + 1, // 10: docreader.DocReader.ReadStream:input_type -> docreader.ReadRequest + 6, // 11: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest + 3, // 12: docreader.DocReader.Read:output_type -> docreader.ReadResponse + 5, // 13: docreader.DocReader.ReadStream:output_type -> docreader.ReadStreamResponse + 8, // 14: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse + 12, // [12:15] is the sub-list for method output_type + 9, // [9:12] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_docreader_proto_init() } @@ -587,13 +772,17 @@ func file_docreader_proto_init() { if File_docreader_proto != nil { return } + file_docreader_proto_msgTypes[5].OneofWrappers = []any{ + (*ReadStreamResponse_Meta)(nil), + (*ReadStreamResponse_Image)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)), NumEnums: 0, - NumMessages: 10, + NumMessages: 13, NumExtensions: 0, NumServices: 1, }, diff --git a/docreader/proto/docreader.proto b/docreader/proto/docreader.proto index 0d803cac..f1f8a441 100644 --- a/docreader/proto/docreader.proto +++ b/docreader/proto/docreader.proto @@ -6,6 +6,13 @@ option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto"; service DocReader { rpc Read(ReadRequest) returns (ReadResponse) {} + // ReadStream is the streaming counterpart of Read. It first emits one + // ReadStreamResponse carrying the parse metadata (markdown / metadata / + // error), then emits one message per image. This keeps every gRPC message + // small so large scanned PDFs (hundreds of page images, far exceeding the + // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and + // with bounded memory on both ends. + rpc ReadStream(ReadRequest) returns (stream ReadStreamResponse) {} rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {} } @@ -44,6 +51,24 @@ message ReadResponse { string error = 5; } +// Metadata header for a streamed read. Sent exactly once, before any image. +message ReadStreamMeta { + string markdown_content = 1; + string image_dir_path = 2; + map metadata = 3; + string error = 4; + uint32 image_count = 5; // best-effort total image count (0 if unknown) +} + +// One frame of a ReadStream. The first frame carries `meta`; every subsequent +// frame carries a single `image`. +message ReadStreamResponse { + oneof payload { + ReadStreamMeta meta = 1; + ImageRef image = 2; + } +} + message ListEnginesRequest { map config_overrides = 1; } diff --git a/docreader/proto/docreader_grpc.pb.go b/docreader/proto/docreader_grpc.pb.go index 4a163a87..abb5bbc8 100644 --- a/docreader/proto/docreader_grpc.pb.go +++ b/docreader/proto/docreader_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.4 +// - protoc v5.29.3 // source: docreader.proto package proto @@ -20,6 +20,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( DocReader_Read_FullMethodName = "/docreader.DocReader/Read" + DocReader_ReadStream_FullMethodName = "/docreader.DocReader/ReadStream" DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines" ) @@ -28,6 +29,13 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type DocReaderClient interface { Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) + // ReadStream is the streaming counterpart of Read. It first emits one + // ReadStreamResponse carrying the parse metadata (markdown / metadata / + // error), then emits one message per image. This keeps every gRPC message + // small so large scanned PDFs (hundreds of page images, far exceeding the + // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and + // with bounded memory on both ends. + ReadStream(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadStreamResponse], error) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) } @@ -49,6 +57,25 @@ func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grp return out, nil } +func (c *docReaderClient) ReadStream(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadStreamResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DocReader_ServiceDesc.Streams[0], DocReader_ReadStream_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ReadRequest, ReadStreamResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DocReader_ReadStreamClient = grpc.ServerStreamingClient[ReadStreamResponse] + func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ListEnginesResponse) @@ -64,6 +91,13 @@ func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesReques // for forward compatibility. type DocReaderServer interface { Read(context.Context, *ReadRequest) (*ReadResponse, error) + // ReadStream is the streaming counterpart of Read. It first emits one + // ReadStreamResponse carrying the parse metadata (markdown / metadata / + // error), then emits one message per image. This keeps every gRPC message + // small so large scanned PDFs (hundreds of page images, far exceeding the + // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and + // with bounded memory on both ends. + ReadStream(*ReadRequest, grpc.ServerStreamingServer[ReadStreamResponse]) error ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) mustEmbedUnimplementedDocReaderServer() } @@ -78,6 +112,9 @@ type UnimplementedDocReaderServer struct{} func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Read not implemented") } +func (UnimplementedDocReaderServer) ReadStream(*ReadRequest, grpc.ServerStreamingServer[ReadStreamResponse]) error { + return status.Errorf(codes.Unimplemented, "method ReadStream not implemented") +} func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented") } @@ -120,6 +157,17 @@ func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(inte return interceptor(ctx, in, info, handler) } +func _DocReader_ReadStream_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DocReaderServer).ReadStream(m, &grpc.GenericServerStream[ReadRequest, ReadStreamResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DocReader_ReadStreamServer = grpc.ServerStreamingServer[ReadStreamResponse] + func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListEnginesRequest) if err := dec(in); err != nil { @@ -154,6 +202,12 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{ Handler: _DocReader_ListEngines_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ReadStream", + Handler: _DocReader_ReadStream_Handler, + ServerStreams: true, + }, + }, Metadata: "docreader.proto", } diff --git a/docreader/proto/docreader_pb2.py b/docreader/proto/docreader_pb2.py index 68a61b46..8e3d16e0 100644 --- a/docreader/proto/docreader_pb2.py +++ b/docreader/proto/docreader_pb2.py @@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\x96\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd2\x01\n\x0eReadStreamMeta\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\x16\n\x0eimage_dir_path\x18\x02 \x01(\t\x12\x39\n\x08metadata\x18\x03 \x03(\x0b\x32\'.docreader.ReadStreamMeta.MetadataEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x13\n\x0bimage_count\x18\x05 \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x12ReadStreamResponse\x12)\n\x04meta\x18\x01 \x01(\x0b\x32\x19.docreader.ReadStreamMetaH\x00\x12$\n\x05image\x18\x02 \x01(\x0b\x32\x13.docreader.ImageRefH\x00\x42\t\n\x07payload\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\xdf\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\nReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x00\x30\x01\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,6 +36,8 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001' _globals['_READRESPONSE_METADATAENTRY']._loaded_options = None _globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001' + _globals['_READSTREAMMETA_METADATAENTRY']._loaded_options = None + _globals['_READSTREAMMETA_METADATAENTRY']._serialized_options = b'8\001' _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001' _globals['_READCONFIG']._serialized_start=31 @@ -50,14 +52,20 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_READRESPONSE']._serialized_end=721 _globals['_READRESPONSE_METADATAENTRY']._serialized_start=674 _globals['_READRESPONSE_METADATAENTRY']._serialized_end=721 - _globals['_LISTENGINESREQUEST']._serialized_start=724 - _globals['_LISTENGINESREQUEST']._serialized_end=878 - _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=824 - _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=878 - _globals['_PARSERENGINEINFO']._serialized_start=880 - _globals['_PARSERENGINEINFO']._serialized_end=1000 - _globals['_LISTENGINESRESPONSE']._serialized_start=1002 - _globals['_LISTENGINESRESPONSE']._serialized_end=1069 - _globals['_DOCREADER']._serialized_start=1072 - _globals['_DOCREADER']._serialized_end=1222 + _globals['_READSTREAMMETA']._serialized_start=724 + _globals['_READSTREAMMETA']._serialized_end=934 + _globals['_READSTREAMMETA_METADATAENTRY']._serialized_start=674 + _globals['_READSTREAMMETA_METADATAENTRY']._serialized_end=721 + _globals['_READSTREAMRESPONSE']._serialized_start=936 + _globals['_READSTREAMRESPONSE']._serialized_end=1048 + _globals['_LISTENGINESREQUEST']._serialized_start=1051 + _globals['_LISTENGINESREQUEST']._serialized_end=1205 + _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=1151 + _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=1205 + _globals['_PARSERENGINEINFO']._serialized_start=1207 + _globals['_PARSERENGINEINFO']._serialized_end=1327 + _globals['_LISTENGINESRESPONSE']._serialized_start=1329 + _globals['_LISTENGINESRESPONSE']._serialized_end=1396 + _globals['_DOCREADER']._serialized_start=1399 + _globals['_DOCREADER']._serialized_end=1622 # @@protoc_insertion_point(module_scope) diff --git a/docreader/proto/docreader_pb2.pyi b/docreader/proto/docreader_pb2.pyi index 53477a66..e10b4e53 100644 --- a/docreader/proto/docreader_pb2.pyi +++ b/docreader/proto/docreader_pb2.pyi @@ -74,6 +74,35 @@ class ReadResponse(_message.Message): error: str def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ... +class ReadStreamMeta(_message.Message): + __slots__ = ("markdown_content", "image_dir_path", "metadata", "error", "image_count") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int] + IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + ERROR_FIELD_NUMBER: _ClassVar[int] + IMAGE_COUNT_FIELD_NUMBER: _ClassVar[int] + markdown_content: str + image_dir_path: str + metadata: _containers.ScalarMap[str, str] + error: str + image_count: int + def __init__(self, markdown_content: _Optional[str] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ..., image_count: _Optional[int] = ...) -> None: ... + +class ReadStreamResponse(_message.Message): + __slots__ = ("meta", "image") + META_FIELD_NUMBER: _ClassVar[int] + IMAGE_FIELD_NUMBER: _ClassVar[int] + meta: ReadStreamMeta + image: ImageRef + def __init__(self, meta: _Optional[_Union[ReadStreamMeta, _Mapping]] = ..., image: _Optional[_Union[ImageRef, _Mapping]] = ...) -> None: ... + class ListEnginesRequest(_message.Message): __slots__ = ("config_overrides",) class ConfigOverridesEntry(_message.Message): diff --git a/docreader/proto/docreader_pb2_grpc.py b/docreader/proto/docreader_pb2_grpc.py index c3808c21..8b9323c9 100644 --- a/docreader/proto/docreader_pb2_grpc.py +++ b/docreader/proto/docreader_pb2_grpc.py @@ -5,7 +5,7 @@ import warnings from docreader.proto import docreader_pb2 as docreader__pb2 -GRPC_GENERATED_VERSION = '1.78.0' +GRPC_GENERATED_VERSION = '1.80.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -39,6 +39,11 @@ class DocReaderStub(object): request_serializer=docreader__pb2.ReadRequest.SerializeToString, response_deserializer=docreader__pb2.ReadResponse.FromString, _registered_method=True) + self.ReadStream = channel.unary_stream( + '/docreader.DocReader/ReadStream', + request_serializer=docreader__pb2.ReadRequest.SerializeToString, + response_deserializer=docreader__pb2.ReadStreamResponse.FromString, + _registered_method=True) self.ListEngines = channel.unary_unary( '/docreader.DocReader/ListEngines', request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString, @@ -55,6 +60,18 @@ class DocReaderServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ReadStream(self, request, context): + """ReadStream is the streaming counterpart of Read. It first emits one + ReadStreamResponse carrying the parse metadata (markdown / metadata / + error), then emits one message per image. This keeps every gRPC message + small so large scanned PDFs (hundreds of page images, far exceeding the + unary message-size cap) can be returned without RESOURCE_EXHAUSTED and + with bounded memory on both ends. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def ListEngines(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) @@ -69,6 +86,11 @@ def add_DocReaderServicer_to_server(servicer, server): request_deserializer=docreader__pb2.ReadRequest.FromString, response_serializer=docreader__pb2.ReadResponse.SerializeToString, ), + 'ReadStream': grpc.unary_stream_rpc_method_handler( + servicer.ReadStream, + request_deserializer=docreader__pb2.ReadRequest.FromString, + response_serializer=docreader__pb2.ReadStreamResponse.SerializeToString, + ), 'ListEngines': grpc.unary_unary_rpc_method_handler( servicer.ListEngines, request_deserializer=docreader__pb2.ListEnginesRequest.FromString, @@ -112,6 +134,33 @@ class DocReader(object): metadata, _registered_method=True) + @staticmethod + def ReadStream(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/docreader.DocReader/ReadStream', + docreader__pb2.ReadRequest.SerializeToString, + docreader__pb2.ReadStreamResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def ListEngines(request, target, diff --git a/docreader/pyproject.toml b/docreader/pyproject.toml index 35a2c50e..77c03571 100644 --- a/docreader/pyproject.toml +++ b/docreader/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "grpcio>=1.78.0", "grpcio-health-checking>=1.78.0", "grpcio-tools>=1.78.0", + "liteparse>=2.0.4", "lxml>=6.1.0", "markitdown[docx,pdf,xls,xlsx]>=0.1.3", "pillow>=12.0.0", diff --git a/docreader/scripts/parse_local.py b/docreader/scripts/parse_local.py new file mode 100644 index 00000000..a5372f3d --- /dev/null +++ b/docreader/scripts/parse_local.py @@ -0,0 +1,119 @@ +"""本地解析调试脚本:直接调用 Parser 解析本地文件,不经过 gRPC 服务。 + +用法(在仓库根目录 WeKnora 下执行): + PYTHONPATH=. docreader/.venv/bin/python docreader/scripts/parse_local.py <文件路径> [--engine markitdown] [--out out.md] + +示例: + PYTHONPATH=. docreader/.venv/bin/python docreader/scripts/parse_local.py docreader/testdata/test.md + PYTHONPATH=. docreader/.venv/bin/python docreader/scripts/parse_local.py ~/Desktop/demo.pdf --out /tmp/demo.md +""" + +import argparse +import logging +import os +import sys +import time + +from docreader.parser import Parser + + +def main() -> int: + parser = argparse.ArgumentParser(description="解析本地文件并输出 markdown") + parser.add_argument("path", help="待解析的本地文件路径") + parser.add_argument( + "--engine", + default="", + help="解析引擎名称(builtin / markitdown),留空使用内置引擎", + ) + parser.add_argument( + "--type", + default="", + help="文件类型(如 pdf/docx/md),留空则按扩展名推断", + ) + parser.add_argument( + "--out", + default="", + help="将完整 markdown 写入该文件,并把图片导出到同目录的 images/ 下", + ) + parser.add_argument( + "--scanned", + action="store_true", + help="跳过 markitdown 文本抽取,直接把 PDF 每页渲染成图片(扫描件用,避免 pdfminer 卡死)", + ) + parser.add_argument( + "--log-level", + default="INFO", + help="日志级别(DEBUG/INFO/WARNING/ERROR)", + ) + args = parser.parse_args() + + logging.basicConfig( + level=getattr(logging, args.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + stream=sys.stderr, + ) + + if not os.path.isfile(args.path): + print(f"文件不存在: {args.path}", file=sys.stderr) + return 1 + + file_name = os.path.basename(args.path) + file_type = args.type or os.path.splitext(file_name)[1].lstrip(".") + with open(args.path, "rb") as f: + content = f.read() + + started = time.monotonic() + if args.scanned: + from docreader.parser.pdf_parser import PDFScannedParser + + doc = PDFScannedParser(file_name=file_name, file_type=file_type).parse_into_text( + content + ) + else: + doc = Parser().parse_file( + file_name=file_name, + file_type=file_type, + content=content, + parser_engine=args.engine or None, + ) + elapsed = time.monotonic() - started + + print("=" * 60, file=sys.stderr) + print(f"file : {file_name}", file=sys.stderr) + print(f"type : {file_type}", file=sys.stderr) + print(f"engine : {args.engine or 'builtin'}", file=sys.stderr) + print(f"scanned : {args.scanned}", file=sys.stderr) + print(f"content_len: {len(doc.content)}", file=sys.stderr) + print(f"images : {len(doc.images)}", file=sys.stderr) + print(f"metadata : {doc.metadata}", file=sys.stderr) + print(f"elapsed : {elapsed:.2f}s", file=sys.stderr) + print("=" * 60, file=sys.stderr) + + if args.out: + import base64 + + out_dir = os.path.dirname(os.path.abspath(args.out)) + os.makedirs(out_dir, exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + f.write(doc.content) + if doc.images: + img_root = os.path.join(out_dir, "images") + os.makedirs(img_root, exist_ok=True) + for ref_path, b64data in doc.images.items(): + try: + raw = base64.b64decode(b64data) + except Exception: + raw = b64data if isinstance(b64data, bytes) else b64data.encode() + dest = os.path.join(out_dir, ref_path) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as imgf: + imgf.write(raw) + print(f"已写入: {args.out}(图片导出到 {out_dir}/images/)", file=sys.stderr) + else: + print(doc.content) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.py new file mode 100644 index 00000000..b07ee36c --- /dev/null +++ b/docreader/tests/test_pdf_router.py @@ -0,0 +1,191 @@ +import io +import unittest + +from PIL import Image + +from docreader.parser.pdf_parser import ( + PDFParser, + _classify_page, + _group_lines, + _point_in_boxes, + _segments_to_markdown, + _select_embedded_images, + _split_columns, + _strip_repeating_lines, +) + + +def _char(ch, x0, x1, y0, y1): + return {"x0": x0, "x1": x1, "y0": y0, "y1": y1, "ch": ch} + + +def _line(text, h): + return {"text": text, "h": h} + + +def _make_image_only_pdf(num_pages: int = 2) -> bytes: + buf = io.BytesIO() + pages = [Image.new("RGB", (64, 64), color) for color in ("white", "black")] + pages = (pages * ((num_pages // 2) + 1))[:num_pages] + pages[0].save(buf, format="PDF", save_all=True, append_images=pages[1:]) + return buf.getvalue() + + +class ClassifyPageTest(unittest.TestCase): + def test_full_page_image_is_scanned_even_with_text(self): + # Scanned newspaper: image covers the page, embedded OCR text exists. + self.assertEqual(_classify_page(2.0, 1620), "scanned") + # Scanned UN doc with garbled OCR text layer (ratio ~1.0). + self.assertEqual(_classify_page(1.0, 1500), "scanned") + + def test_native_text_page_is_text(self): + self.assertEqual(_classify_page(0.01, 673), "text") + self.assertEqual(_classify_page(0.0, 1200), "text") + + def test_sparse_text_with_image_is_scanned(self): + self.assertEqual(_classify_page(0.3, 2), "scanned") + + def test_blank_page_is_text(self): + # No image, no text -> not rendered as an image. + self.assertEqual(_classify_page(0.0, 0), "text") + + +class StripRepeatingLinesTest(unittest.TestCase): + def test_removes_repeated_header_footer(self): + header = "ACME CONFIDENTIAL" + texts = [f"{header}\nbody page {i}\npage {i} footer" for i in range(6)] + # Make the footer identical across pages so it is detected. + texts = [f"{header}\nbody page {i}\nshared footer" for i in range(6)] + classes = ["text"] * 6 + cleaned = _strip_repeating_lines(texts, classes) + for page in cleaned: + self.assertNotIn(header, page) + self.assertNotIn("shared footer", page) + self.assertIn("body page 0", cleaned[0]) + + def test_keeps_lines_when_too_few_pages(self): + texts = ["HEADER\nbody"] * 2 + classes = ["text"] * 2 + self.assertEqual(_strip_repeating_lines(texts, classes), texts) + + +class SelectEmbeddedImagesTest(unittest.TestCase): + def _fig(self, page, h="fig", w=200, ht=200, area=0.2): + return {"page": page, "width": w, "height": ht, "area_ratio": area, "hash": h} + + def test_keeps_real_figure(self): + meta = [self._fig(0, "a")] + self.assertEqual(_select_embedded_images(meta, 1), [0]) + + def test_drops_tiny_and_small_images(self): + meta = [ + self._fig(0, "tiny_area", area=0.001), # too small a share + self._fig(0, "tiny_px", w=20, ht=20), # too few pixels + ] + self.assertEqual(_select_embedded_images(meta, 1), []) + + def test_drops_repeated_logo_watermark(self): + # Same hash on 5 of 6 text pages -> running logo/watermark. + meta = [self._fig(p, "logo") for p in range(5)] + meta.append(self._fig(5, "unique")) + kept = _select_embedded_images(meta, 6) + kept_hashes = {meta[i]["hash"] for i in kept} + self.assertNotIn("logo", kept_hashes) + self.assertIn("unique", kept_hashes) + + def test_dedups_identical_image_on_same_page(self): + meta = [self._fig(0, "dup"), self._fig(0, "dup")] + self.assertEqual(len(_select_embedded_images(meta, 1)), 1) + + def test_respects_max_images_cap(self): + meta = [self._fig(i, f"h{i}") for i in range(10)] + self.assertEqual(len(_select_embedded_images(meta, 10, max_images=3)), 3) + + +class ReadingOrderTest(unittest.TestCase): + def test_single_column_stays_single(self): + # One column of glyphs at x~100, no full-height gutter. + chars = [_char("a", 100, 110, 700 - i * 12, 712 - i * 12) for i in range(5)] + cols = _split_columns(chars, scale=12.0, width=600.0) + self.assertEqual(len(cols), 1) + + def test_two_columns_split_left_to_right(self): + # Left column x~50-150, right column x~400-500, wide empty gutter between. + left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)] + right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)] + cols = _split_columns(left + right, scale=12.0, width=600.0) + self.assertEqual(len(cols), 2) + # Reading order: left column before right column. + self.assertEqual(cols[0][0]["ch"], "L") + self.assertEqual(cols[1][0]["ch"], "R") + + def test_group_lines_orders_by_y_then_x(self): + # Two visual lines; within a line glyphs given out of x-order. + chars = [ + _char("B", 120, 130, 700, 712), + _char("A", 100, 110, 700, 712), # same line, left of B + _char("C", 100, 110, 680, 692), # next line down + ] + lines = _group_lines(chars) + self.assertEqual([ln["text"] for ln in lines], ["AB", "C"]) + + +class HeadingDetectionTest(unittest.TestCase): + def test_promotes_large_line_to_heading(self): + lines = [_line("Big Title", 24.0)] + [_line(f"body {i}", 10.0) for i in range(6)] + md = _segments_to_markdown(lines) + self.assertTrue(md.startswith("# Big Title")) + self.assertIn("\nbody 0", md) + + def test_does_not_promote_when_sizes_uniform(self): + lines = [_line(f"line {i}", 10.0) for i in range(6)] + md = _segments_to_markdown(lines) + self.assertNotIn("#", md) + + def test_skips_sentence_like_long_lines(self): + # Large but ends with a period and is long -> body text, not a heading. + lines = [_line("x" * 90 + ".", 30.0)] + [_line("y", 10.0) for _ in range(6)] + md = _segments_to_markdown(lines) + self.assertFalse(md.startswith("#")) + + +class HiddenTextFilterTest(unittest.TestCase): + def test_point_in_boxes(self): + boxes = [(0.0, 0.0, 10.0, 10.0)] + self.assertTrue(_point_in_boxes(5.0, 5.0, boxes)) + self.assertFalse(_point_in_boxes(20.0, 5.0, boxes)) + + +class PDFRouterIntegrationTest(unittest.TestCase): + def test_image_only_pdf_routes_to_scanned(self): + pdf_bytes = _make_image_only_pdf(2) + doc = PDFParser(file_name="imgonly.pdf", file_type="pdf").parse_into_text( + pdf_bytes + ) + + self.assertEqual(doc.metadata["image_source_type"], "scanned_pdf") + self.assertEqual(doc.metadata["page_count"], 2) + self.assertEqual(doc.metadata["scanned_page_count"], 2) + self.assertEqual(len(doc.images), 2) + self.assertIn("images/imgonly_page_1.jpg", doc.images) + self.assertIn("![imgonly_page_1.jpg](images/imgonly_page_1.jpg)", doc.content) + # JPEG magic bytes after decoding. + import base64 + + self.assertTrue( + base64.b64decode(doc.images["images/imgonly_page_1.jpg"]).startswith( + b"\xff\xd8" + ) + ) + + def test_malformed_pdf_raises_after_fallback(self): + # Routing fails to open the PDF, falls back to full rendering which also + # fails on garbage input; the error surfaces to the caller. + with self.assertRaises(Exception): + PDFParser(file_name="broken.pdf", file_type="pdf").parse_into_text( + b"not a pdf" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docreader/uv.lock b/docreader/uv.lock index dfc19e1c..fbba524e 100644 --- a/docreader/uv.lock +++ b/docreader/uv.lock @@ -463,6 +463,7 @@ dependencies = [ { name = "grpcio" }, { name = "grpcio-health-checking" }, { name = "grpcio-tools" }, + { name = "liteparse" }, { name = "lxml" }, { name = "markitdown", extra = ["docx", "pdf", "xls", "xlsx"] }, { name = "pillow" }, @@ -483,6 +484,7 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.78.0" }, { name = "grpcio-health-checking", specifier = ">=1.78.0" }, { name = "grpcio-tools", specifier = ">=1.78.0" }, + { name = "liteparse", specifier = ">=2.0.4" }, { name = "lxml", specifier = ">=6.1.0" }, { name = "markitdown", extras = ["docx", "pdf", "xls", "xlsx"], specifier = ">=0.1.3" }, { name = "pillow", specifier = ">=12.0.0" }, @@ -784,6 +786,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" }, ] +[[package]] +name = "liteparse" +version = "2.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/e7/ecf68643604a59247a0a7b2f8c73bee7415ea99e0165bb32e2838ddd0d3f/liteparse-2.0.4.tar.gz", hash = "sha256:17f6119f38e80b956c1ce3dc998ea7b0a8e80777ce1f49178f2b14bb17b35a9c", size = 115487, upload-time = "2026-05-30T06:32:12.351Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/b0/4f5007a52ef13679437a892a06ea58448b825de7ea78276e19b9d7fb9dcb/liteparse-2.0.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6df1e1199ffbeb2191bb64d7fcbff6af6bdfd1592973e0ad67a82eb09d377c08", size = 13027870, upload-time = "2026-05-30T06:31:11.022Z" }, + { url = "https://files.pythonhosted.org/packages/83/2f/c7977a2d6f376e31c8c465ee010c238e27e06cbb2c3200d63f41983e40db/liteparse-2.0.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:35a72946b965d3b6b87a602051919e7ce243da15ad143d301152fb5e8cd0f6d2", size = 13149255, upload-time = "2026-05-30T06:31:13.636Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9d/e7f1a1b8cb14ac867b1220fdb0c87bfe07b86c69bf98578573ab37b1a103/liteparse-2.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:c1bbc8b7206b8bfbf7aabc5341d2cf851b7464641d58375bd218b4e1dd3517f9", size = 11115466, upload-time = "2026-05-30T06:31:16.201Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2d/be89a429a6a6bc78ce8d620974a4f8fbe9f566ea3592a2f1da8dc6bdda4a/liteparse-2.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:537ab6347a384f81980e48cc181d6cd33fc6ad2b7478e3db61350076744d952e", size = 11029024, upload-time = "2026-05-30T06:31:19.11Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c8/7429622d86bf00ceaec95bf211adf1c9a7bdf46f8c2cd806685f9c02c0f1/liteparse-2.0.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:feae0c530197130cb38f176d718eeae639d9091264aa5f954835986c59470813", size = 13028074, upload-time = "2026-05-30T06:31:21.725Z" }, + { url = "https://files.pythonhosted.org/packages/21/d0/a97174ae281d353251994ed080c8855ea9b0b5d81a60ab3b6b065e911c49/liteparse-2.0.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:2012a3a9b5a3f7e13ce34b5a770158971da43bb9d266c7c5a3ea62bdda7ca851", size = 13148977, upload-time = "2026-05-30T06:31:24.498Z" }, + { url = "https://files.pythonhosted.org/packages/60/48/f41ebe428d8d8d70c53ddd47523baa7300c5cc96e404417d7af25578be01/liteparse-2.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:2d05d10f0d14b1beb34ef8c5e9a14d6cc966adf19f60c7ea1ec5717adc4c986f", size = 11115791, upload-time = "2026-05-30T06:31:26.961Z" }, + { url = "https://files.pythonhosted.org/packages/ea/58/be78c7c47147aeb1350d475336c6c2e17d5aa513be9244e9d95a170ced34/liteparse-2.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87680616fae276b04ace6e5fc5e4e0c93980391b0d46c2d66d72c0742a3cb19e", size = 11026045, upload-time = "2026-05-30T06:31:29.405Z" }, + { url = "https://files.pythonhosted.org/packages/86/1f/105ccdd9bc4608a836fe409394d68e8765e699fa7393c2f2f464c612057f/liteparse-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:414599a922aa51f567fa939183929579d1668ef74846fe25f7f46742bb31fcd8", size = 13022571, upload-time = "2026-05-30T06:31:32.321Z" }, + { url = "https://files.pythonhosted.org/packages/58/9f/4bf4e9b112b47025ae085503fe9cbf13631673ffc41bfb864a3091285c22/liteparse-2.0.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:efdaa6b471084a1f4594574555eb6abb5f85de25f2155c8d539542239eacaa56", size = 13146871, upload-time = "2026-05-30T06:31:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f0/bf10611e409732bd4e19f0fc0faf3194040e8e09bb75a166ee126d09b70f/liteparse-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:fb67326ba957388214762acea35d24cf0d1230ae6a2fe1fdeaf74024e92e3c40", size = 11116682, upload-time = "2026-05-30T06:31:36.992Z" }, + { url = "https://files.pythonhosted.org/packages/6e/b5/02ed5fff6418fdc970688190eab4470f4f9c116f4de1e39a7deea0d9968a/liteparse-2.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8f02c6e0d8f71da671a3527d52d8f1e2c42fddebf81d1b4931c3d035e4ec1e6a", size = 11025231, upload-time = "2026-05-30T06:31:39.696Z" }, + { url = "https://files.pythonhosted.org/packages/be/d7/b4633483502940d43d583f8057e0aed68b9091087a86d021f8bd7558ba0b/liteparse-2.0.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:924e3f669341e22e625b13d08535644d1dfd779bc6781e4ab6f6e54ea90a53d6", size = 13022754, upload-time = "2026-05-30T06:31:42.434Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e0/3938561ad66d4a216922c8e1e6a878f63df82ce5f00f15a935f779fb7c5b/liteparse-2.0.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:137f169002f3abe21e3dd2e6781fbd86841096a0f3b0162afc1fd64eb21fa607", size = 13146432, upload-time = "2026-05-30T06:31:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/e6/7f/a2017df8031677d7940ad1ce33640219aa28defae4a8171844ea8bed68ca/liteparse-2.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:098fba3ecb2337f78426d9e077d1f70bc75871d4387ab8c3774b0cc5d26b890d", size = 11116383, upload-time = "2026-05-30T06:31:49.546Z" }, + { url = "https://files.pythonhosted.org/packages/59/63/b2bb03bc30103e93c87695f63eae3ed007b08796a6cc06ea29acace54c4a/liteparse-2.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8aeaf821151aaaa854294f3499d64264dbea7d10e682fa9a2443f9177cd444c6", size = 11024196, upload-time = "2026-05-30T06:31:52.091Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c1/6dedc6b4325aa8de3249694123a74bc9506e0d65a28c85aa5fad4bfdea5c/liteparse-2.0.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:d2efbaf7453d2bedc86db51b2b808078567817d7fc537122389b65a317927902", size = 13022936, upload-time = "2026-05-30T06:31:54.619Z" }, + { url = "https://files.pythonhosted.org/packages/7a/04/7e7c3a8edd01c9904b6eef76bf4a008f987a5df64b8334c61e742861ac84/liteparse-2.0.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:34c53d9cefa35f77dc67a19a875e6dca32b4f35006c2015a22eb30c9c810653b", size = 13146821, upload-time = "2026-05-30T06:31:57.284Z" }, + { url = "https://files.pythonhosted.org/packages/66/f4/da191e881cad5941dc0065782497eb81027bc3f48ac0a3143deab094be33/liteparse-2.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:6546ee0359dc56eebd9f45008bb59708118c234140ecf466f6c7121d9161d9e4", size = 11114558, upload-time = "2026-05-30T06:31:59.799Z" }, + { url = "https://files.pythonhosted.org/packages/16/59/c554f376c0bdd1bf4c313ac5d77a34817740f021ab6ada9d3226a23fa4b6/liteparse-2.0.4-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:7c02d0bb31cd5aefa3297ce6e58388abd6f3e109c62ac0fdeef07d8eac4b769e", size = 13023454, upload-time = "2026-05-30T06:32:02.464Z" }, + { url = "https://files.pythonhosted.org/packages/89/96/04c595ab45162d81bc73218870d1459560428c3f40957e594a6c1c5ea2be/liteparse-2.0.4-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:acdf3c76cb3215f8d389a935b6b68007fac2ffa9ce0b681dd53650b69d580521", size = 13146859, upload-time = "2026-05-30T06:32:05.175Z" }, + { url = "https://files.pythonhosted.org/packages/97/9c/59cdd88ebc6c27312ea6cbd0a894002e78b6f8a3dead2b2bf60d7febba85/liteparse-2.0.4-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4cf31cb3987df1190e59b73d9f10976e538ff577f41c40281fd14b84fe4f9da1", size = 13030767, upload-time = "2026-05-30T06:32:07.9Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ae/9b85e510ddb390ed63b407851d412152b7006487d06703d931f6a0b1414e/liteparse-2.0.4-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a1f9cb9c24f2df0d4f71ddd66ddb474bfdec8a434ecc1428b791f83aab2a688b", size = 13152103, upload-time = "2026-05-30T06:32:10.457Z" }, +] + [[package]] name = "lxml" version = "6.1.0" diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index 209dc11a..b184d050 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -113,7 +113,7 @@ func NewChunkExtractTask( if err != nil { return false, err } - task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.MaxRetry(3)) + task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.Queue(types.QueueGraph), asynq.MaxRetry(3)) info, err := client.Enqueue(task) if err != nil { logger.Errorf(ctx, "failed to enqueue task: %v", err) diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index 3507a661..2d842156 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -525,7 +525,7 @@ func (s *KnowledgePostProcessService) enqueueQuestionGenerationTasks( continue } - task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) + task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue(types.QueueQuestion), asynq.MaxRetry(3)) if _, err := s.taskEnqueuer.Enqueue(task); err != nil { logger.Warnf(ctx, "[KnowledgePostProcess] Failed to enqueue question generation batch %d for %s: %v", batchIndex-1, payload.KnowledgeID, err) continue diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 66098974..a80c238d 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3343,7 +3343,7 @@ func (s *knowledgeService) enqueueImageMultimodalTasks( continue } - task := asynq.NewTask(types.TypeImageMultimodal, payloadBytes) + task := asynq.NewTask(types.TypeImageMultimodal, payloadBytes, asynq.Queue(types.QueueMultimodal)) if _, err := s.task.Enqueue(task); err != nil { logger.Warnf(ctx, "Failed to enqueue image multimodal task for %s: %v", img.ServingURL, err) } else { diff --git a/internal/infrastructure/docparser/grpc_parser.go b/internal/infrastructure/docparser/grpc_parser.go index 61d2d15b..0478e2de 100644 --- a/internal/infrastructure/docparser/grpc_parser.go +++ b/internal/infrastructure/docparser/grpc_parser.go @@ -3,6 +3,7 @@ package docparser import ( "context" "fmt" + "io" "os" "strconv" "sync" @@ -125,11 +126,52 @@ func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) ( }, } - resp, err := client.Read(ctx, protoReq) + // Use the streaming RPC so documents with many page images (large scanned + // PDFs) are not capped by the unary message-size limit. The meta frame + // arrives first, followed by one frame per image. + stream, err := client.ReadStream(ctx, protoReq) if err != nil { - return nil, fmt.Errorf("gRPC Read failed: %w", err) + return nil, fmt.Errorf("gRPC ReadStream failed: %w", err) } - return fromProtoReadResponse(resp), nil + + result := &types.ReadResult{} + gotMeta := false + for { + frame, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + return nil, fmt.Errorf("gRPC ReadStream recv failed: %w", recvErr) + } + + if meta := frame.GetMeta(); meta != nil { + gotMeta = true + result.MarkdownContent = meta.GetMarkdownContent() + result.ImageDirPath = meta.GetImageDirPath() + result.Metadata = meta.GetMetadata() + result.Error = meta.GetError() + if n := meta.GetImageCount(); n > 0 { + result.ImageRefs = make([]types.ImageRef, 0, n) + } + continue + } + + if img := frame.GetImage(); img != nil { + result.ImageRefs = append(result.ImageRefs, types.ImageRef{ + Filename: img.GetFilename(), + OriginalRef: img.GetOriginalRef(), + MimeType: img.GetMimeType(), + StorageKey: img.GetStorageKey(), + ImageData: img.GetImageData(), + }) + } + } + + if !gotMeta { + return nil, fmt.Errorf("gRPC ReadStream returned no metadata frame") + } + return result, nil } func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) { @@ -157,24 +199,3 @@ func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[stri } return result, nil } - -func fromProtoReadResponse(resp *proto.ReadResponse) *types.ReadResult { - result := &types.ReadResult{ - MarkdownContent: resp.GetMarkdownContent(), - ImageDirPath: resp.GetImageDirPath(), - Metadata: resp.GetMetadata(), - Error: resp.GetError(), - } - - for _, ref := range resp.GetImageRefs() { - result.ImageRefs = append(result.ImageRefs, types.ImageRef{ - Filename: ref.GetFilename(), - OriginalRef: ref.GetOriginalRef(), - MimeType: ref.GetMimeType(), - StorageKey: ref.GetStorageKey(), - ImageData: ref.GetImageData(), - }) - } - - return result -} diff --git a/internal/models/vlm/remote_api.go b/internal/models/vlm/remote_api.go index 03feba4f..46747bbf 100644 --- a/internal/models/vlm/remote_api.go +++ b/internal/models/vlm/remote_api.go @@ -5,6 +5,8 @@ import ( "encoding/base64" "fmt" "net/http" + "os" + "strconv" "strings" "time" @@ -15,11 +17,27 @@ import ( ) const ( - defaultTimeout = 90 * time.Second + // defaultTimeout is the fallback HTTP timeout for a single VLM request. + // Dense scanned-PDF OCR (full-page text + layout extraction) can take well + // over a minute on slow endpoints, so this is intentionally generous and + // can be raised further via VLM_HTTP_TIMEOUT_SECONDS. + defaultTimeout = 180 * time.Second defaultMaxToks = 5000 defaultTemp = float32(0.1) ) +// vlmHTTPTimeout returns the HTTP client timeout for VLM requests, read from +// the VLM_HTTP_TIMEOUT_SECONDS env var when set (and positive), falling back to +// defaultTimeout otherwise. Shared by all OpenAI-compatible VLM backends. +func vlmHTTPTimeout() time.Duration { + if v := strings.TrimSpace(os.Getenv("VLM_HTTP_TIMEOUT_SECONDS")); v != "" { + if secs, err := strconv.Atoi(v); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + } + return defaultTimeout +} + // RemoteAPIVLM implements VLM via an OpenAI-compatible chat completions API. type RemoteAPIVLM struct { modelName string @@ -54,7 +72,7 @@ func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) { apiCfg.BaseURL = config.BaseURL } } - httpClient := &http.Client{Timeout: defaultTimeout} + httpClient := &http.Client{Timeout: vlmHTTPTimeout()} // 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers) if len(config.CustomHeaders) > 0 { diff --git a/internal/models/vlm/weknoracloud.go b/internal/models/vlm/weknoracloud.go index bc9b5ad0..98428586 100644 --- a/internal/models/vlm/weknoracloud.go +++ b/internal/models/vlm/weknoracloud.go @@ -51,7 +51,7 @@ func NewWeKnoraCloudVLM(config *Config) (*WeKnoraCloudVLM, error) { appID: config.AppID, apiKey: config.AppSecret, baseURL: strings.TrimRight(config.BaseURL, "/"), - client: &http.Client{Timeout: defaultTimeout}, + client: &http.Client{Timeout: vlmHTTPTimeout()}, }, nil } diff --git a/internal/router/task.go b/internal/router/task.go index 505e6a4b..61fced8d 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -139,9 +139,12 @@ func NewAsynqServer() *asynq.Server { asynq.Config{ Concurrency: concurrency, Queues: map[string]int{ - "critical": 6, // Highest priority queue - "default": 3, // Default priority queue - "low": 1, // Lowest priority queue + types.QueueCritical: 6, // Highest priority queue + types.QueueDefault: 3, // Default priority queue + types.QueueLow: 1, // Lowest priority queue + types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks + types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks + types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks }, RetryDelayFunc: asynqRetryDelayFunc, }, diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index 013720f2..600c42b8 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -47,7 +47,17 @@ type knowledgeIDProbe struct { // queuesScanned is the fixed set of queue names this codebase enqueues // into. Kept tight on purpose — we never scan user-defined queues. -var queuesScanned = []string{"default", "critical", "low"} +// MUST include every queue any cancelable task type can land in; the +// multimodal queue is required here so cancelling a knowledge also purges +// its (potentially hundreds of) pending image:multimodal tasks. +var queuesScanned = []string{ + types.QueueDefault, + types.QueueCritical, + types.QueueLow, + types.QueueMultimodal, + types.QueueGraph, + types.QueueQuestion, +} // taskTypesForKnowledgeCancel lists every asynq task type that carries // a knowledge_id in its payload and should be cancelable. The set is diff --git a/internal/types/task.go b/internal/types/task.go index 257ee3b0..fba6a3cd 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -1,5 +1,28 @@ package types +// Asynq queue names. MUST stay in sync with the Queues weight map in +// router.NewAsynqServer — a task enqueued to a queue that the server does not +// list will never be consumed. +const ( + QueueCritical = "critical" + QueueDefault = "default" + QueueLow = "low" + // QueueMultimodal isolates high-volume, slow VLM image tasks (OCR + caption) + // so a single large scanned PDF (hundreds–thousands of page images) cannot + // saturate the shared worker pool and block user-facing document parsing in + // the default queue. + QueueMultimodal = "multimodal" + // QueueGraph isolates high-volume, slow graph-extraction tasks (one per + // chunk, LLM-backed, only when Neo4j is enabled). Same rationale as + // QueueMultimodal: a large document must not flood the default queue. + QueueGraph = "graph" + // QueueQuestion isolates high-volume, slow question-generation tasks (one + // per 20-chunk batch, LLM-backed). Keeps a large document's hundreds of + // question batches from starving the lightweight tasks in the low queue + // (summary, deletes, wiki ingest). + QueueQuestion = "question" +) + const ( TypeChunkExtract = "chunk:extract" TypeDocumentProcess = "document:process" // 文档处理任务 diff --git a/scripts/dev.sh b/scripts/dev.sh index 1d37abc9..4103cf96 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -55,7 +55,7 @@ show_help() { echo "用法: $0 [命令] [选项]" echo "" echo "命令:" - echo " start 启动基础设施服务(postgres, redis, docreader)" + echo " start 启动基础设施服务(postgres, redis, docreader, langfuse)" echo " stop 停止所有服务" echo " restart 重启所有服务" echo " logs 查看服务日志" @@ -65,12 +65,14 @@ show_help() { echo " help 显示此帮助信息" echo "" echo "可选 Profile(用于 start 命令):" - echo " --minio 启动 MinIO 对象存储" - echo " --qdrant 启动 Qdrant 向量数据库" - echo " --neo4j 启动 Neo4j 图数据库" - echo " --jaeger 启动 Jaeger 链路追踪" - echo " --dex 启动 Dex(OIDC 身份认证)" - echo " --full 启动所有可选服务" + echo " --minio 启动 MinIO 对象存储" + echo " --qdrant 启动 Qdrant 向量数据库" + echo " --neo4j 启动 Neo4j 图数据库" + echo " --jaeger 启动 Jaeger 链路追踪" + echo " --dex 启动 Dex(OIDC 身份认证)" + echo " --langfuse 启动 Langfuse(默认已开启)" + echo " --no-langfuse 不启动 Langfuse" + echo " --full 启动所有可选服务" echo "" echo "示例:" echo " $0 start # 启动基础服务" @@ -121,8 +123,10 @@ start_services() { # 解析 profile 参数 shift # 移除 "start" 命令本身 - PROFILES="--profile full" - ENABLED_SERVICES="" + # 默认启动基础设施(postgres / redis / docreader)+ langfuse, + # 其余可选服务通过 --minio / --qdrant / --neo4j / --jaeger / --dex / --full 按需开启。 + PROFILES="--profile langfuse" + ENABLED_SERVICES="langfuse" while [ $# -gt 0 ]; do case "$1" in @@ -146,6 +150,14 @@ start_services() { PROFILES="$PROFILES --profile dex" ENABLED_SERVICES="$ENABLED_SERVICES dex" ;; + --langfuse) + PROFILES="$PROFILES --profile langfuse" + ENABLED_SERVICES="$ENABLED_SERVICES langfuse" + ;; + --no-langfuse) + PROFILES="${PROFILES//--profile langfuse/}" + ENABLED_SERVICES="${ENABLED_SERVICES//langfuse/}" + ;; --full) PROFILES="--profile full" ENABLED_SERVICES="minio qdrant neo4j jaeger dex" @@ -185,6 +197,9 @@ start_services() { if [[ "$ENABLED_SERVICES" == *"dex"* ]]; then echo " - Dex: localhost:5556" fi + if [[ "$ENABLED_SERVICES" == *"langfuse"* ]]; then + echo " - Langfuse: http://localhost:${LANGFUSE_WEB_PORT:-3000}" + fi echo "" log_info "接下来的步骤:"