fix(excel): filter out image function strings from parsed content
WPS embeds images in cells using =DISPIMG("ID",mode) which may appear as plain text prefixed with "_xlfn." when processed by openpyxl. These function strings pollute the RAG-indexed document content, causing incorrect AI recall.
Add a regex filter (_is_image_function) that detects _xlfn.DISPIMG(...) and _xlfn.IMAGE(...) patterns and excludes them during row processing in ExcelParser.parse_into_text().
Refs: https://github.com/Tencent/WeKnora/issues/1779
This commit is contained in:
@@ -6,6 +6,7 @@ structured Document objects with text content and chunks. It supports multiple
|
||||
sheets and handles various Excel formats using pandas.
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
@@ -24,6 +25,22 @@ from docreader.parser.xlsx_repair import repair_xlsx_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Pattern to detect Excel image function strings that should be excluded from
|
||||
# parsed text content. WPS uses =DISPIMG("ID",mode) to embed images in cells;
|
||||
# when opened by other tools the formula may appear as plain text prefixed with
|
||||
# "_xlfn." or "=". Office 365 uses =_xlfn.IMAGE(url, ...) similarly.
|
||||
# The _xlfn. prefix is optional — WPS may omit it (e.g. =DISPIMG("ID",1)).
|
||||
_IMAGE_FUNC_RE = re.compile(
|
||||
r"^=?(_xlfn\.)?(DISPIMG|IMAGE)\(", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _is_image_function(value: object) -> bool:
|
||||
"""Return True if *value* looks like an embedded-image function string."""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return _IMAGE_FUNC_RE.match(value) is not None
|
||||
|
||||
|
||||
class ExcelParser(BaseParser):
|
||||
"""Parser for Excel files (.xlsx, .xls).
|
||||
@@ -81,7 +98,7 @@ class ExcelParser(BaseParser):
|
||||
page_content = []
|
||||
# Build key-value pairs for non-null values
|
||||
for k, v in row.items():
|
||||
if pd.notna(v): # Skip NaN/null values
|
||||
if pd.notna(v) and not _is_image_function(v):
|
||||
page_content.append(f"{k}: {v}")
|
||||
|
||||
# Skip rows with no valid content
|
||||
|
||||
@@ -181,6 +181,99 @@ class XlsxMergeFillTest(unittest.TestCase):
|
||||
self.assertIn("D: D10", chunks[9])
|
||||
|
||||
|
||||
class ExcelImageFilterTest(unittest.TestCase):
|
||||
"""Tests for filtering embedded image function strings (#1779)."""
|
||||
|
||||
def _xlsx_with_image_functions(self) -> bytes:
|
||||
"""Create an XLSX where image functions are stored as text values.
|
||||
|
||||
WPS embeds images using =DISPIMG("ID",1) which may appear as plain
|
||||
text (not a formula) in some export scenarios.
|
||||
"""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "Name"
|
||||
ws["B1"] = "Photo"
|
||||
ws["A2"] = "Alice"
|
||||
ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
|
||||
ws["A3"] = "Bob"
|
||||
ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
|
||||
ws["A4"] = "Charlie"
|
||||
ws["B4"] = "real data"
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
return bio.getvalue()
|
||||
|
||||
def test_dispimg_text_values_are_excluded(self):
|
||||
"""Image function strings stored as text must not appear in output."""
|
||||
document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
|
||||
self.assertNotIn("DISPIMG", document.content)
|
||||
self.assertNotIn("_xlfn", document.content)
|
||||
# Real data must still be present
|
||||
self.assertIn("Alice", document.content)
|
||||
self.assertIn("Bob", document.content)
|
||||
self.assertIn("Charlie", document.content)
|
||||
self.assertIn("real data", document.content)
|
||||
|
||||
def test_dispimg_with_equals_prefix(self):
|
||||
"""=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "Name"
|
||||
ws["B1"] = "Photo"
|
||||
ws["A2"] = "Alice"
|
||||
# Stored as text with = prefix (some exporters do this)
|
||||
ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
# Note: openpyxl treats strings starting with = as formulas,
|
||||
# so data_only=True in fill_merged_cells_xlsx will turn them to None.
|
||||
# This test verifies the formula path also works correctly.
|
||||
document = ExcelParser().parse_into_text(bio.getvalue())
|
||||
self.assertNotIn("DISPIMG", document.content)
|
||||
self.assertIn("Alice", document.content)
|
||||
|
||||
def test_image_function_variations(self):
|
||||
"""Various image function patterns should all be filtered."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
|
||||
ws["A2"] = 'DISPIMG("ID_002",1)' # no prefix at all
|
||||
ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
|
||||
ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
|
||||
ws["A5"] = "Normal text"
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
document = ExcelParser().parse_into_text(bio.getvalue())
|
||||
self.assertNotIn("DISPIMG", document.content)
|
||||
self.assertNotIn("IMAGE(", document.content)
|
||||
self.assertIn("Normal text", document.content)
|
||||
|
||||
def test_real_world_wps_dispimg(self):
|
||||
"""Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "Product"
|
||||
ws["B1"] = "Image"
|
||||
ws["C1"] = "Price"
|
||||
ws["A2"] = "Basic"
|
||||
# This is the exact format shown in the issue screenshot
|
||||
ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
|
||||
ws["C2"] = "39.9"
|
||||
ws["A3"] = "Pro"
|
||||
ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
|
||||
ws["C3"] = "99.9"
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
document = ExcelParser().parse_into_text(bio.getvalue())
|
||||
self.assertNotIn("DISPIMG", document.content)
|
||||
self.assertNotIn("ID_5A60F9ED", document.content)
|
||||
self.assertIn("Basic", document.content)
|
||||
self.assertIn("39.9", document.content)
|
||||
self.assertIn("Pro", document.content)
|
||||
self.assertIn("99.9", document.content)
|
||||
|
||||
|
||||
class ExcelParserTest(unittest.TestCase):
|
||||
def test_parse_phantom_shared_strings_workbook(self):
|
||||
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
|
||||
|
||||
Reference in New Issue
Block a user