fix(excel): filter out image function strings from parsed content

WPS embeds images in cells using =DISPIMG("ID",mode) which may appear as plain text prefixed with "_xlfn." when processed by openpyxl. These function strings pollute the RAG-indexed document content, causing incorrect AI recall.

Add a regex filter (_is_image_function) that detects _xlfn.DISPIMG(...) and _xlfn.IMAGE(...) patterns and excludes them during row processing in ExcelParser.parse_into_text().

Refs: https://github.com/Tencent/WeKnora/issues/1779
This commit is contained in:
zhouxun
2026-06-24 11:40:46 +08:00
committed by lyingbug
parent fa8e2bb087
commit 4db727ac69
2 changed files with 111 additions and 1 deletions
+93
View File
@@ -181,6 +181,99 @@ class XlsxMergeFillTest(unittest.TestCase):
self.assertIn("D: D10", chunks[9])
class ExcelImageFilterTest(unittest.TestCase):
"""Tests for filtering embedded image function strings (#1779)."""
def _xlsx_with_image_functions(self) -> bytes:
"""Create an XLSX where image functions are stored as text values.
WPS embeds images using =DISPIMG("ID",1) which may appear as plain
text (not a formula) in some export scenarios.
"""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
ws["A3"] = "Bob"
ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
ws["A4"] = "Charlie"
ws["B4"] = "real data"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def test_dispimg_text_values_are_excluded(self):
"""Image function strings stored as text must not appear in output."""
document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("_xlfn", document.content)
# Real data must still be present
self.assertIn("Alice", document.content)
self.assertIn("Bob", document.content)
self.assertIn("Charlie", document.content)
self.assertIn("real data", document.content)
def test_dispimg_with_equals_prefix(self):
"""=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
# Stored as text with = prefix (some exporters do this)
ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
bio = io.BytesIO()
wb.save(bio)
# Note: openpyxl treats strings starting with = as formulas,
# so data_only=True in fill_merged_cells_xlsx will turn them to None.
# This test verifies the formula path also works correctly.
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertIn("Alice", document.content)
def test_image_function_variations(self):
"""Various image function patterns should all be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
ws["A2"] = 'DISPIMG("ID_002",1)' # no prefix at all
ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
ws["A5"] = "Normal text"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("IMAGE(", document.content)
self.assertIn("Normal text", document.content)
def test_real_world_wps_dispimg(self):
"""Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Product"
ws["B1"] = "Image"
ws["C1"] = "Price"
ws["A2"] = "Basic"
# This is the exact format shown in the issue screenshot
ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
ws["C2"] = "39.9"
ws["A3"] = "Pro"
ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
ws["C3"] = "99.9"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("ID_5A60F9ED", document.content)
self.assertIn("Basic", document.content)
self.assertIn("39.9", document.content)
self.assertIn("Pro", document.content)
self.assertIn("99.9", document.content)
class ExcelParserTest(unittest.TestCase):
def test_parse_phantom_shared_strings_workbook(self):
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())