refactor: Restructure OCR module and centralize config
This commit is contained in:
@@ -10,6 +10,7 @@ This module provides comprehensive Markdown parsing functionality including:
|
||||
The parser uses a pipeline approach to process Markdown content through
|
||||
multiple stages: table formatting -> image processing.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
@@ -28,23 +29,23 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class MarkdownTableUtil:
|
||||
"""Utility class for formatting Markdown tables.
|
||||
|
||||
|
||||
This class standardizes Markdown table formatting by:
|
||||
- Normalizing column alignment markers (e.g., :---, :---:, ---:)
|
||||
- Adding consistent spacing around pipes (|)
|
||||
- Preserving indentation levels
|
||||
- Handling both header rows and data rows
|
||||
|
||||
|
||||
Example:
|
||||
Input: |姓名|年龄|城市|
|
||||
|:---|---:|:---:|
|
||||
|张三|25|北京|
|
||||
|
||||
|
||||
Output: | 姓名 | 年龄 | 城市 |
|
||||
| :--- | ---: | :---: |
|
||||
| 张三 | 25 | 北京 |
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Pattern to match alignment row (e.g., |:---|---:|:---:|)
|
||||
self.align_pattern = re.compile(
|
||||
@@ -59,14 +60,14 @@ class MarkdownTableUtil:
|
||||
|
||||
def format_table(self, content: str) -> str:
|
||||
"""Format all Markdown tables in the content.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown text containing tables
|
||||
|
||||
|
||||
Returns:
|
||||
Formatted Markdown text with standardized table formatting
|
||||
"""
|
||||
|
||||
|
||||
def process_align(match: Match[str]) -> str:
|
||||
"""Process alignment row to standardize format."""
|
||||
# Split by | and remove empty strings
|
||||
@@ -125,10 +126,10 @@ class MarkdownTableUtil:
|
||||
|
||||
class MarkdownTableFormatter(BaseParser):
|
||||
"""Parser for formatting Markdown tables.
|
||||
|
||||
|
||||
This parser standardizes the formatting of all Markdown tables in the
|
||||
document to ensure consistent spacing and alignment markers.
|
||||
|
||||
|
||||
Example:
|
||||
>>> formatter = MarkdownTableFormatter()
|
||||
>>> content = b"|Name|Age|\n|---|---|\n|John|30|"
|
||||
@@ -138,17 +139,17 @@ class MarkdownTableFormatter(BaseParser):
|
||||
| --- | --- |
|
||||
| John | 30 |
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.table_helper = MarkdownTableUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse and format Markdown tables.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
|
||||
Returns:
|
||||
Document with formatted table content
|
||||
"""
|
||||
@@ -161,18 +162,18 @@ class MarkdownTableFormatter(BaseParser):
|
||||
|
||||
class MarkdownImageUtil:
|
||||
"""Utility class for handling images in Markdown.
|
||||
|
||||
|
||||
This class provides functionality to:
|
||||
- Extract base64-encoded images from Markdown
|
||||
- Extract image paths from Markdown
|
||||
- Replace image paths with new URLs
|
||||
- Convert base64 images to binary format
|
||||
|
||||
|
||||
Supported formats:
|
||||
- Base64 embedded images: 
|
||||
- Regular image links: 
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Pattern to match base64 embedded images
|
||||
# Captures: (1) alt text, (2) image format, (3) base64 data
|
||||
@@ -191,15 +192,15 @@ class MarkdownImageUtil:
|
||||
replace: bool = True,
|
||||
) -> Tuple[str, List[str]]:
|
||||
"""Extract image paths from Markdown content.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text containing images
|
||||
path_prefix: Optional prefix to add to image paths
|
||||
replace: Whether to replace image syntax in content
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (processed_text, list_of_image_paths)
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> text, images = util.extract_image("")
|
||||
@@ -213,7 +214,7 @@ class MarkdownImageUtil:
|
||||
"""Replacement function for each image match."""
|
||||
title = match.group(1) # Alt text
|
||||
image_path = match.group(2) # Image path
|
||||
|
||||
|
||||
# Add prefix if specified
|
||||
if path_prefix:
|
||||
image_path = f"{path_prefix}/{image_path}"
|
||||
@@ -238,19 +239,19 @@ class MarkdownImageUtil:
|
||||
replace: bool = True,
|
||||
) -> Tuple[str, Dict[str, bytes]]:
|
||||
"""Extract and decode base64 embedded images from Markdown.
|
||||
|
||||
|
||||
This method finds all base64-encoded images in the Markdown content,
|
||||
decodes them to binary format, generates unique filenames, and
|
||||
optionally replaces them with file path references.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text containing base64 images
|
||||
path_prefix: Optional directory prefix for generated paths
|
||||
replace: Whether to replace base64 syntax with file paths
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (processed_text, dict_of_path_to_bytes)
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> text = ""
|
||||
@@ -294,17 +295,17 @@ class MarkdownImageUtil:
|
||||
|
||||
def replace_path(self, content: str, images: Dict[str, str]) -> str:
|
||||
"""Replace image paths in Markdown with new URLs.
|
||||
|
||||
|
||||
This method is typically used to replace local file paths with
|
||||
uploaded URLs after images have been stored.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text with image references
|
||||
images: Mapping of old paths to new URLs
|
||||
|
||||
|
||||
Returns:
|
||||
Markdown text with updated image URLs
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> content = ""
|
||||
@@ -320,7 +321,7 @@ class MarkdownImageUtil:
|
||||
"""Replacement function for each image match."""
|
||||
title = match.group(1) # Alt text
|
||||
image_path = match.group(2) # Current image path
|
||||
|
||||
|
||||
# Only replace if path exists in mapping
|
||||
if image_path not in images:
|
||||
return match.group(0) # Keep original
|
||||
@@ -328,7 +329,7 @@ class MarkdownImageUtil:
|
||||
content_replace.add(image_path)
|
||||
# Get new URL from mapping
|
||||
image_path = images[image_path]
|
||||
return f""
|
||||
return f"" if image_path else title
|
||||
|
||||
text = self.replace_pattern.sub(repl, content)
|
||||
logger.debug(f"Replaced {len(content_replace)} images in markdown")
|
||||
@@ -348,16 +349,16 @@ class MarkdownImageUtil:
|
||||
|
||||
class MarkdownImageBase64(BaseParser):
|
||||
"""Parser for extracting and uploading base64 images from Markdown.
|
||||
|
||||
|
||||
This parser:
|
||||
1. Extracts base64-encoded images from Markdown content
|
||||
2. Uploads the decoded images to storage
|
||||
3. Replaces base64 data with uploaded URLs
|
||||
4. Returns a Document with clean Markdown and image mappings
|
||||
|
||||
|
||||
Requires:
|
||||
- self.storage: Storage backend for uploading images
|
||||
|
||||
|
||||
Example:
|
||||
>>> parser = MarkdownImageBase64(storage=my_storage)
|
||||
>>> content = b""
|
||||
@@ -365,17 +366,17 @@ class MarkdownImageBase64(BaseParser):
|
||||
>>> print(doc.content)
|
||||

|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Markdown and process base64 images.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
|
||||
Returns:
|
||||
Document with:
|
||||
- content: Markdown with base64 replaced by URLs
|
||||
@@ -411,14 +412,15 @@ class MarkdownImageBase64(BaseParser):
|
||||
|
||||
class MarkdownParser(PipelineParser):
|
||||
"""Complete Markdown parser using pipeline approach.
|
||||
|
||||
|
||||
This parser processes Markdown content through multiple stages:
|
||||
1. MarkdownTableFormatter: Standardizes table formatting
|
||||
2. MarkdownImageBase64: Extracts and uploads base64 images
|
||||
|
||||
|
||||
The pipeline ensures that content flows through each parser in sequence,
|
||||
with each stage's output becoming the next stage's input.
|
||||
"""
|
||||
|
||||
_parser_cls = (MarkdownTableFormatter, MarkdownImageBase64)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user