fix(excel): filter out image function strings from parsed content

WPS embeds images in cells using =DISPIMG("ID",mode) which may appear as plain text prefixed with "_xlfn." when processed by openpyxl. These function strings pollute the RAG-indexed document content, causing incorrect AI recall.

Add a regex filter (_is_image_function) that detects _xlfn.DISPIMG(...) and _xlfn.IMAGE(...) patterns and excludes them during row processing in ExcelParser.parse_into_text().

Refs: https://github.com/Tencent/WeKnora/issues/1779
This commit is contained in:
zhouxun
2026-06-24 11:40:46 +08:00
committed by lyingbug
parent fa8e2bb087
commit 4db727ac69
2 changed files with 111 additions and 1 deletions
+18 -1
View File
@@ -6,6 +6,7 @@ structured Document objects with text content and chunks. It supports multiple
sheets and handles various Excel formats using pandas.
"""
import logging
import re
from io import BytesIO
from typing import List
@@ -24,6 +25,22 @@ from docreader.parser.xlsx_repair import repair_xlsx_bytes
logger = logging.getLogger(__name__)
# Pattern to detect Excel image function strings that should be excluded from
# parsed text content. WPS uses =DISPIMG("ID",mode) to embed images in cells;
# when opened by other tools the formula may appear as plain text prefixed with
# "_xlfn." or "=". Office 365 uses =_xlfn.IMAGE(url, ...) similarly.
# The _xlfn. prefix is optional — WPS may omit it (e.g. =DISPIMG("ID",1)).
_IMAGE_FUNC_RE = re.compile(
r"^=?(_xlfn\.)?(DISPIMG|IMAGE)\(", re.IGNORECASE
)
def _is_image_function(value: object) -> bool:
"""Return True if *value* looks like an embedded-image function string."""
if not isinstance(value, str):
return False
return _IMAGE_FUNC_RE.match(value) is not None
class ExcelParser(BaseParser):
"""Parser for Excel files (.xlsx, .xls).
@@ -81,7 +98,7 @@ class ExcelParser(BaseParser):
page_content = []
# Build key-value pairs for non-null values
for k, v in row.items():
if pd.notna(v): # Skip NaN/null values
if pd.notna(v) and not _is_image_function(v):
page_content.append(f"{k}: {v}")
# Skip rows with no valid content
+93
View File
@@ -181,6 +181,99 @@ class XlsxMergeFillTest(unittest.TestCase):
self.assertIn("D: D10", chunks[9])
class ExcelImageFilterTest(unittest.TestCase):
"""Tests for filtering embedded image function strings (#1779)."""
def _xlsx_with_image_functions(self) -> bytes:
"""Create an XLSX where image functions are stored as text values.
WPS embeds images using =DISPIMG("ID",1) which may appear as plain
text (not a formula) in some export scenarios.
"""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
ws["A3"] = "Bob"
ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
ws["A4"] = "Charlie"
ws["B4"] = "real data"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def test_dispimg_text_values_are_excluded(self):
"""Image function strings stored as text must not appear in output."""
document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("_xlfn", document.content)
# Real data must still be present
self.assertIn("Alice", document.content)
self.assertIn("Bob", document.content)
self.assertIn("Charlie", document.content)
self.assertIn("real data", document.content)
def test_dispimg_with_equals_prefix(self):
"""=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
# Stored as text with = prefix (some exporters do this)
ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
bio = io.BytesIO()
wb.save(bio)
# Note: openpyxl treats strings starting with = as formulas,
# so data_only=True in fill_merged_cells_xlsx will turn them to None.
# This test verifies the formula path also works correctly.
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertIn("Alice", document.content)
def test_image_function_variations(self):
"""Various image function patterns should all be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
ws["A2"] = 'DISPIMG("ID_002",1)' # no prefix at all
ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
ws["A5"] = "Normal text"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("IMAGE(", document.content)
self.assertIn("Normal text", document.content)
def test_real_world_wps_dispimg(self):
"""Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Product"
ws["B1"] = "Image"
ws["C1"] = "Price"
ws["A2"] = "Basic"
# This is the exact format shown in the issue screenshot
ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
ws["C2"] = "39.9"
ws["A3"] = "Pro"
ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
ws["C3"] = "99.9"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("ID_5A60F9ED", document.content)
self.assertIn("Basic", document.content)
self.assertIn("39.9", document.content)
self.assertIn("Pro", document.content)
self.assertIn("99.9", document.content)
class ExcelParserTest(unittest.TestCase):
def test_parse_phantom_shared_strings_workbook(self):
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())