Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Browse Source 
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docreader/parser/web_parser.py
+137 -29
View File
@@ -1,9 +1,11 @@
import asyncio
import logging
import re
from dataclasses import dataclass
from typing import Optional
from lxml.etree import XPath
from playwright.async_api import async_playwright
from playwright.async_api import Page, async_playwright
from trafilatura import extract, utils, xpaths
from docreader.config import CONFIG
@@ -15,6 +17,14 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
_GOTO_TIMEOUT_MS = 30_000
_NETWORK_IDLE_TIMEOUT_MS = 10_000
_SPA_WAIT_TIMEOUT_MS = 15_000
# Minimum visible characters before treating an SPA shell as "rendered".
_SPA_MIN_TEXT_LEN = 80
# Minimum visible characters for Playwright text fallback when trafilatura fails.
_MIN_FALLBACK_TEXT_LEN = 50
# Monkey-patch trafilatura internals to better support WeChat Official Account
# articles, whose images live on `mmbiz.qpic.cn` without a standard file
# extension and whose main content sits inside `#js_content` /
@@ -40,6 +50,78 @@ except (AttributeError, ImportError) as e:
)
@dataclass(frozen=True)
class _ScrapeResult:
html: str
visible_text: str
page_title: str
def extract_markdown_from_html(html: str) -> Optional[str]:
"""Run trafilatura on HTML; return markdown or None if nothing extracted."""
if not html or not html.strip():
return None
md_text = extract(
html,
output_format="markdown",
with_metadata=True,
include_images=True,
include_tables=True,
include_links=True,
)
if not md_text or not md_text.strip():
return None
return md_text
def build_visible_text_fallback(visible_text: str, page_title: str = "") -> Optional[str]:
"""Build markdown from Playwright-visible text when trafilatura finds no article body."""
text = (visible_text or "").strip()
if len(text) < _MIN_FALLBACK_TEXT_LEN:
return None
title = (page_title or "").strip()
if title and not text.startswith(title):
return f"# {title}\n\n{text}"
return text
async def wait_for_rendered_content(page: Page) -> None:
"""Wait for SPA/JS pages beyond the initial HTML shell."""
try:
await page.wait_for_load_state("networkidle", timeout=_NETWORK_IDLE_TIMEOUT_MS)
logger.info("Network idle after navigation")
except Exception:
logger.info("Network idle wait timed out, continuing")
try:
await page.wait_for_function(
"""(minLen) => {
const root = document.querySelector('#app')
|| document.querySelector('main')
|| document.body;
return ((root?.innerText || '').trim().length >= minLen);
}""",
arg=_SPA_MIN_TEXT_LEN,
timeout=_SPA_WAIT_TIMEOUT_MS,
)
logger.info("SPA/root visible text reached minimum length")
except Exception:
logger.info("SPA text wait timed out, using current DOM")
async def read_visible_text(page: Page) -> str:
"""Prefer #app/main innerText, then fall back to body."""
return await page.evaluate(
"""() => {
const root = document.querySelector('#app')
|| document.querySelector('main')
|| document.querySelector('[role="main"]')
|| document.body;
return (root?.innerText || '').trim();
}"""
)
class StdWebParser(BaseParser):
"""Standard web page parser using Playwright and Trafilatura.
@@ -61,16 +143,17 @@ class StdWebParser(BaseParser):
super().__init__(file_name=title, **kwargs)
logger.info(f"Initialized WebParser with title: {title}")
async def scrape(self, url: str) -> str:
async def scrape(self, url: str) -> _ScrapeResult:
"""Scrape web page content using Playwright.
Args:
url: The URL of the web page to scrape
Returns:
HTML content of the web page as string, empty string on error
HTML, visible text, and document title; empty fields on hard failure
"""
logger.info(f"Starting web page scraping for URL: {url}")
empty = _ScrapeResult(html="", visible_text="", page_title="")
try:
async with async_playwright() as p:
kwargs = {}
@@ -83,30 +166,42 @@ class StdWebParser(BaseParser):
logger.info(f"Navigating to URL: {url}")
try:
# Navigate to URL with 30 second timeout
await page.goto(url, timeout=30000)
await page.goto(
url,
timeout=_GOTO_TIMEOUT_MS,
wait_until="domcontentloaded",
)
logger.info("Initial page load complete")
except Exception as e:
logger.error(f"Error navigating to URL: {str(e)}")
await browser.close()
return ""
return empty
logger.info("Retrieving page HTML content")
# Get the full HTML content of the page
await wait_for_rendered_content(page)
page_title = await page.title()
visible_text = await read_visible_text(page)
content = await page.content()
logger.info(f"Retrieved {len(content)} bytes of HTML content")
logger.info(
"Retrieved %d bytes HTML, %d chars visible text, title=%r",
len(content),
len(visible_text),
page_title[:80] if page_title else "",
)
await browser.close()
logger.info("Browser closed")
# Return raw HTML content for further processing
logger.info("Successfully retrieved HTML content")
return content
return _ScrapeResult(
html=content,
visible_text=visible_text,
page_title=page_title or "",
)
except Exception as e:
logger.error(f"Failed to scrape web page: {str(e)}")
# Return empty string on error
return ""
return empty
def parse_into_text(self, content: bytes) -> Document:
"""Parse web page content into a Document object.
@@ -117,36 +212,49 @@ class StdWebParser(BaseParser):
Returns:
Document object containing the parsed markdown content
"""
# Decode bytes to get the URL string
url = endecode.decode_bytes(content)
logger.info(f"Scraping web page: {url}")
# Run async scraping in sync context
chtml = asyncio.run(self.scrape(url))
# Extract clean content from HTML using Trafilatura
# Convert to markdown format with metadata, images, tables, and links
md_text = extract(
chtml,
output_format="markdown",
with_metadata=True,
include_images=True,
include_tables=True,
include_links=True,
)
scrape_result = asyncio.run(self.scrape(url))
if not scrape_result.html and not scrape_result.visible_text:
logger.error("Failed to scrape web page (no HTML or visible text)")
return Document(content=f"Error parsing web page: {url}")
md_text = extract_markdown_from_html(scrape_result.html)
if not md_text:
md_text = build_visible_text_fallback(
scrape_result.visible_text,
scrape_result.page_title,
)
if md_text:
logger.info(
"Trafilatura empty; using Playwright visible-text fallback (%d chars)",
len(md_text),
)
if not md_text:
logger.error("Failed to parse web page")
return Document(content=f"Error parsing web page: {url}")
# Extract title from trafilatura metadata output (e.g. "title: xxx" line)
metadata = {}
title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
if title_match:
extracted_title = title_match.group(1).strip()
if extracted_title:
metadata["title"] = extracted_title
logger.info(f"Extracted article title from trafilatura: {extracted_title}")
logger.info(
f"Extracted article title from trafilatura: {extracted_title}"
)
elif scrape_result.page_title:
metadata["title"] = scrape_result.page_title.strip()
logger.info(
"Using page title from Playwright: %s", metadata["title"]
)
else:
logger.info(f"No title found in trafilatura output, first 200 chars: {md_text[:200]!r}")
logger.info(
"No title found in trafilatura output, first 200 chars: %r",
md_text[:200],
)
return Document(content=md_text, metadata=metadata)