feat(parser): enhance web parser with improved image extension handling and XPath prioritization

- Added a regex pattern for image file extensions to the utils module for better image detection.
- Updated the BODY_XPATH in the xpaths module to prioritize matching specific content structures in web pages.
- These changes aim to improve the accuracy and efficiency of content extraction from web pages using the StdWebParser class.
This commit is contained in:
wizardchen
2026-05-20 22:50:49 +08:00
committed by lyingbug
parent 5c0243cd92
commit 13301ca026
+26 -1
View File
@@ -2,8 +2,9 @@ import asyncio
import logging
import re
from lxml.etree import XPath
from playwright.async_api import async_playwright
from trafilatura import extract
from trafilatura import extract, utils, xpaths
from docreader.config import CONFIG
from docreader.models.document import Document
@@ -14,6 +15,30 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
# Monkey-patch trafilatura internals to better support WeChat Official Account
# articles, whose images live on `mmbiz.qpic.cn` without a standard file
# extension and whose main content sits inside `#js_content` /
# `.rich_media_content`. Trafilatura's `utils.IMAGE_EXTENSION` and
# `xpaths.BODY_XPATH` are internal APIs, so we guard the patch and skip
# silently if they are renamed/removed in a future release.
try:
_WECHAT_IMAGE_EXTENSION = re.compile(
r"[^\s]+\.(avif|bmp|gif|hei[cf]|jpe?g|png|webp)(\b|$)|" # Standard extensions
r"mmbiz\.qpic\.cn/[^\s]*wx_fmt=(jpeg|jpg|png|gif|webp)" # WeChat query format
)
utils.IMAGE_EXTENSION = _WECHAT_IMAGE_EXTENSION
_WECHAT_BODY_XPATH = XPath(
'(.//*[@id="js_content" or contains(@class, "rich_media_content")])[1]'
)
_wechat_xpath_str = str(_WECHAT_BODY_XPATH)
if not any(str(x) == _wechat_xpath_str for x in xpaths.BODY_XPATH):
xpaths.BODY_XPATH.insert(0, _WECHAT_BODY_XPATH)
except (AttributeError, ImportError) as e:
logger.warning(
"Failed to patch trafilatura internals for WeChat support: %s", e
)
class StdWebParser(BaseParser):
"""Standard web page parser using Playwright and Trafilatura.