From 13301ca026c06158fefcc03878a9419ddc6b6ceb Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 20 May 2026 22:50:49 +0800 Subject: [PATCH] feat(parser): enhance web parser with improved image extension handling and XPath prioritization - Added a regex pattern for image file extensions to the utils module for better image detection. - Updated the BODY_XPATH in the xpaths module to prioritize matching specific content structures in web pages. - These changes aim to improve the accuracy and efficiency of content extraction from web pages using the StdWebParser class. --- docreader/parser/web_parser.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py index c34bc722..fb46fb4d 100644 --- a/docreader/parser/web_parser.py +++ b/docreader/parser/web_parser.py @@ -2,8 +2,9 @@ import asyncio import logging import re +from lxml.etree import XPath from playwright.async_api import async_playwright -from trafilatura import extract +from trafilatura import extract, utils, xpaths from docreader.config import CONFIG from docreader.models.document import Document @@ -14,6 +15,30 @@ from docreader.utils import endecode logger = logging.getLogger(__name__) +# Monkey-patch trafilatura internals to better support WeChat Official Account +# articles, whose images live on `mmbiz.qpic.cn` without a standard file +# extension and whose main content sits inside `#js_content` / +# `.rich_media_content`. Trafilatura's `utils.IMAGE_EXTENSION` and +# `xpaths.BODY_XPATH` are internal APIs, so we guard the patch and skip +# silently if they are renamed/removed in a future release. +try: + _WECHAT_IMAGE_EXTENSION = re.compile( + r"[^\s]+\.(avif|bmp|gif|hei[cf]|jpe?g|png|webp)(\b|$)|" # Standard extensions + r"mmbiz\.qpic\.cn/[^\s]*wx_fmt=(jpeg|jpg|png|gif|webp)" # WeChat query format + ) + utils.IMAGE_EXTENSION = _WECHAT_IMAGE_EXTENSION + + _WECHAT_BODY_XPATH = XPath( + '(.//*[@id="js_content" or contains(@class, "rich_media_content")])[1]' + ) + _wechat_xpath_str = str(_WECHAT_BODY_XPATH) + if not any(str(x) == _wechat_xpath_str for x in xpaths.BODY_XPATH): + xpaths.BODY_XPATH.insert(0, _WECHAT_BODY_XPATH) +except (AttributeError, ImportError) as e: + logger.warning( + "Failed to patch trafilatura internals for WeChat support: %s", e + ) + class StdWebParser(BaseParser): """Standard web page parser using Playwright and Trafilatura.