feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
@@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
|
||||
meaningful chunks for further processing and indexing.
|
||||
"""
|
||||
|
||||
from .csv_parser import CSVParser
|
||||
from .doc_parser import DocParser
|
||||
from .docx2_parser import Docx2Parser
|
||||
from .excel_parser import ExcelParser
|
||||
@@ -21,19 +20,19 @@ from .image_parser import ImageParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .parser import Parser
|
||||
from .pdf_parser import PDFParser
|
||||
from .text_parser import TextParser
|
||||
from .registry import ParserEngineRegistry, registry
|
||||
from .web_parser import WebParser
|
||||
|
||||
# Export public classes and modules
|
||||
__all__ = [
|
||||
"Docx2Parser", # Parser for .docx files (modern Word documents)
|
||||
"DocParser", # Parser for .doc files (legacy Word documents)
|
||||
"PDFParser", # Parser for PDF documents
|
||||
"MarkdownParser", # Parser for Markdown text files
|
||||
"TextParser", # Parser for plain text files
|
||||
"ImageParser", # Parser for images with text content
|
||||
"WebParser", # Parser for web pages
|
||||
"Parser", # Main parser factory that selects the appropriate parser
|
||||
"CSVParser", # Parser for CSV files
|
||||
"ExcelParser", # Parser for Excel files
|
||||
"Docx2Parser",
|
||||
"DocParser",
|
||||
"PDFParser",
|
||||
"MarkdownParser",
|
||||
"ImageParser",
|
||||
"WebParser",
|
||||
"Parser",
|
||||
"ExcelParser",
|
||||
"ParserEngineRegistry",
|
||||
"registry",
|
||||
]
|
||||
|
||||
+22
-968
File diff suppressed because it is too large
Load Diff
@@ -1,387 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import ollama
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageUrl:
|
||||
"""Image URL data structure for caption requests."""
|
||||
|
||||
url: Optional[str] = None
|
||||
detail: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Content:
|
||||
"""Content data structure that can contain text or image URL."""
|
||||
|
||||
type: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
image_url: Optional[ImageUrl] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMessage:
|
||||
"""System message for VLM model requests."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserMessage:
|
||||
"""User message for VLM model requests, can contain multiple content items."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: List[Content] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompletionRequest:
|
||||
"""Request structure for VLM model completion API."""
|
||||
|
||||
model: str
|
||||
temperature: float
|
||||
top_p: float
|
||||
messages: List[Union[SystemMessage, UserMessage]]
|
||||
user: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
"""Model identifier structure."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelsResp:
|
||||
"""Response structure for available models API."""
|
||||
|
||||
data: List[Model] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""Message structure in API response."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Choice:
|
||||
"""Choice structure in API response."""
|
||||
|
||||
message: Optional[Message] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage information in API response."""
|
||||
|
||||
prompt_tokens: Optional[int] = 0
|
||||
total_tokens: Optional[int] = 0
|
||||
completion_tokens: Optional[int] = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptionChatResp:
|
||||
"""Response structure for caption chat API."""
|
||||
|
||||
id: Optional[str] = None
|
||||
created: Optional[int] = None
|
||||
model: Optional[Model] = None
|
||||
object: Optional[str] = None
|
||||
choices: List[Choice] = field(default_factory=list)
|
||||
usage: Optional[Usage] = None
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: dict) -> "CaptionChatResp":
|
||||
"""
|
||||
Parse API response JSON into a CaptionChatResp object.
|
||||
|
||||
Args:
|
||||
json_data: The JSON response from the API
|
||||
|
||||
Returns:
|
||||
A parsed CaptionChatResp object
|
||||
"""
|
||||
logger.info("Parsing CaptionChatResp from JSON")
|
||||
# Manually parse nested fields with safe field extraction
|
||||
choices = []
|
||||
for choice in json_data.get("choices", []):
|
||||
message_data = choice.get("message", {})
|
||||
message = Message(
|
||||
role=message_data.get("role"),
|
||||
content=message_data.get("content"),
|
||||
tool_calls=message_data.get("tool_calls"),
|
||||
)
|
||||
choices.append(Choice(message=message))
|
||||
|
||||
# Handle usage with safe field extraction
|
||||
usage_data = json_data.get("usage", {})
|
||||
usage = None
|
||||
if usage_data:
|
||||
usage = Usage(
|
||||
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
||||
total_tokens=usage_data.get("total_tokens", 0),
|
||||
completion_tokens=usage_data.get("completion_tokens", 0),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
|
||||
)
|
||||
return CaptionChatResp(
|
||||
id=json_data.get("id"),
|
||||
created=json_data.get("created"),
|
||||
model=json_data.get("model"),
|
||||
object=json_data.get("object"),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
def choice_data(self) -> str:
|
||||
"""
|
||||
Extract the content from the first choice in the response.
|
||||
|
||||
Returns:
|
||||
The content string from the first choice, or empty string if no choices
|
||||
"""
|
||||
if (
|
||||
not self.choices
|
||||
or not self.choices[0]
|
||||
or not self.choices[0].message
|
||||
or not self.choices[0].message.content
|
||||
):
|
||||
logger.warning("No choices available in response")
|
||||
return ""
|
||||
logger.info("Retrieving content from first choice")
|
||||
return self.choices[0].message.content
|
||||
|
||||
|
||||
class Caption:
|
||||
"""
|
||||
Service for generating captions for images using a Vision Language Model.
|
||||
Uses an external API to process images and return textual descriptions.
|
||||
"""
|
||||
|
||||
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
|
||||
"""
|
||||
Initialize the Caption service with configuration
|
||||
from parameters or environment variables.
|
||||
"""
|
||||
logger.info("Initializing Caption service")
|
||||
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
|
||||
self.prompt = """简单凝炼的描述图片的主要内容"""
|
||||
# API request timeout in seconds
|
||||
self.timeout = 30
|
||||
|
||||
# Use provided VLM config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
|
||||
# Build completion URL from provided base URL
|
||||
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
|
||||
self.model = vlm_config.get("model_name", "")
|
||||
self.api_key = vlm_config.get("api_key", "")
|
||||
# Interface type: "ollama" or "openai" (default)
|
||||
self.interface_type = vlm_config.get("interface_type", "openai").lower()
|
||||
else:
|
||||
# Fall back to environment variables if config not provided
|
||||
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
|
||||
self.model = CONFIG.vlm_model_name
|
||||
self.api_key = CONFIG.vlm_model_api_key
|
||||
self.interface_type = CONFIG.vlm_interface_type
|
||||
|
||||
# Validate interface type - must be either "ollama" or "openai"
|
||||
if self.interface_type not in ["ollama", "openai"]:
|
||||
logger.warning(
|
||||
f"Unknown interface type: {self.interface_type}, defaulting to openai"
|
||||
)
|
||||
self.interface_type = "openai"
|
||||
|
||||
logger.info(
|
||||
f"Configured with model: {self.model}, "
|
||||
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
|
||||
)
|
||||
|
||||
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
|
||||
"""
|
||||
Call the Caption API to generate a description for the given image.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
CaptionChatResp object if successful, None otherwise
|
||||
"""
|
||||
logger.info("Calling Caption API for image captioning")
|
||||
logger.info(f"Processing image data: {image_data[:50]}...")
|
||||
|
||||
# Route to appropriate API based on interface type
|
||||
if self.interface_type == "ollama":
|
||||
return self._call_ollama_api(image_data)
|
||||
else:
|
||||
return self._call_openai_api(image_data)
|
||||
|
||||
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call Ollama API for image captioning using base64 encoded image data."""
|
||||
|
||||
# Extract host URL by removing the chat completions endpoint
|
||||
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
|
||||
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
|
||||
|
||||
# Initialize Ollama client with host and timeout
|
||||
client = ollama.Client(
|
||||
host=host,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"Calling Ollama API with model: {self.model}")
|
||||
|
||||
# Call Ollama API with base64 encoded image
|
||||
# Prompt: "Briefly describe the main content of the image"
|
||||
response = client.generate(
|
||||
model=self.model,
|
||||
prompt="简单凝炼的描述图片的主要内容",
|
||||
images=[image_base64], # Pass base64 encoded image data
|
||||
options={
|
||||
"temperature": 0.1
|
||||
}, # Low temperature for more deterministic output
|
||||
stream=False,
|
||||
)
|
||||
|
||||
# Construct response object in standard format
|
||||
caption_resp = CaptionChatResp(
|
||||
id="ollama_response",
|
||||
created=int(time.time()),
|
||||
model=Model(id=self.model),
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(message=Message(role="assistant", content=response.response))
|
||||
],
|
||||
)
|
||||
|
||||
logger.info("Successfully received response from Ollama API")
|
||||
return caption_resp
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling Ollama API: {e}")
|
||||
return None
|
||||
|
||||
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call OpenAI-compatible API for image captioning."""
|
||||
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
|
||||
|
||||
# Construct user message with text prompt and base64 encoded image
|
||||
user_msg = UserMessage(
|
||||
role="user",
|
||||
content=[
|
||||
Content(type="text", text=self.prompt),
|
||||
Content(
|
||||
type="image_url",
|
||||
image_url=ImageUrl(
|
||||
url="data:image/png;base64," + image_base64, detail="auto"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Build completion request with model parameters
|
||||
gpt_req = CompletionRequest(
|
||||
model=self.model,
|
||||
temperature=0.3, # Moderate randomness for balanced output
|
||||
top_p=0.8, # Nucleus sampling parameter
|
||||
messages=[user_msg],
|
||||
user="abc",
|
||||
)
|
||||
|
||||
# Set up HTTP headers for the API request
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
# Add authorization header if API key is provided
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Sending request to OpenAI-compatible API with model: {self.model}"
|
||||
)
|
||||
# Send POST request to the API endpoint
|
||||
response = requests.post(
|
||||
self.completion_url,
|
||||
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
# Check for successful response
|
||||
if response.status_code != 200:
|
||||
logger.error(
|
||||
f"OpenAI API returned non-200 status code: {response.status_code}"
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(f"Received from OpenAI with status: {response.status_code}")
|
||||
logger.info("Converting response to CaptionChatResp object")
|
||||
# Parse JSON response into structured object
|
||||
caption_resp = CaptionChatResp.from_json(response.json())
|
||||
|
||||
if caption_resp.usage:
|
||||
logger.info(
|
||||
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
|
||||
f"completion_tokens={caption_resp.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
return caption_resp
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
|
||||
def get_caption(self, image_data: str) -> str:
|
||||
"""
|
||||
Get a caption for the provided image data.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
Caption text as string, or empty string if captioning failed
|
||||
"""
|
||||
logger.info("Getting caption for image")
|
||||
if not image_data or self.completion_url is None:
|
||||
logger.error("Image data is not set")
|
||||
return ""
|
||||
caption_resp = self._call_caption_api(image_data)
|
||||
if caption_resp:
|
||||
caption = caption_resp.choice_data()
|
||||
caption_length = len(caption)
|
||||
logger.info(f"Successfully generated caption of length {caption_length}")
|
||||
logger.info(
|
||||
f"Caption: {caption[:50]}..."
|
||||
if caption_length > 50
|
||||
else f"Caption: {caption}"
|
||||
)
|
||||
return caption
|
||||
logger.warning("Failed to get caption from Caption API")
|
||||
return ""
|
||||
@@ -1,101 +0,0 @@
|
||||
"""
|
||||
CSV Parser Module
|
||||
|
||||
This module provides a parser for CSV (Comma-Separated Values) files.
|
||||
It converts CSV data into a Document with structured chunks, where each row
|
||||
becomes a separate chunk with key-value pairs.
|
||||
"""
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from docreader.models.document import Chunk, Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CSVParser(BaseParser):
|
||||
"""
|
||||
Parser for CSV files that converts tabular data into structured text.
|
||||
|
||||
This parser reads CSV content and transforms each row into a formatted string
|
||||
with column-value pairs. Each row is stored as a separate Chunk in the Document,
|
||||
allowing for granular access to individual records.
|
||||
|
||||
The output format for each row is:
|
||||
"column1: value1, column2: value2, column3: value3\n"
|
||||
|
||||
Usage:
|
||||
parser = CSVParser()
|
||||
with open("data.csv", "rb") as f:
|
||||
document = parser.parse_into_text(f.read())
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse CSV content into a Document with structured chunks.
|
||||
|
||||
Each row in the CSV is converted into a formatted string and stored as
|
||||
a separate Chunk. The chunks maintain sequential order and track their
|
||||
position in the overall document.
|
||||
|
||||
Args:
|
||||
content: Raw bytes content of the CSV file
|
||||
|
||||
Returns:
|
||||
Document: A Document object containing:
|
||||
- content: Full text with all rows concatenated
|
||||
- chunks: List of Chunk objects, one per CSV row
|
||||
|
||||
Note:
|
||||
Bad lines in the CSV are automatically skipped using pandas'
|
||||
on_bad_lines="skip" parameter.
|
||||
"""
|
||||
chunks: List[Chunk] = []
|
||||
text: List[str] = []
|
||||
start, end = 0, 0
|
||||
|
||||
# Read CSV content into a pandas DataFrame, skipping malformed lines
|
||||
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
|
||||
|
||||
# Process each row in the DataFrame
|
||||
for i, (idx, row) in enumerate(df.iterrows()):
|
||||
# Format row as "column: value" pairs separated by commas
|
||||
content_row = (
|
||||
",".join(
|
||||
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
# Update end position for this chunk
|
||||
end += len(content_row)
|
||||
text.append(content_row)
|
||||
|
||||
# Create a chunk for this row with position tracking
|
||||
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
|
||||
# Update start position for next chunk
|
||||
start = end
|
||||
|
||||
return Document(
|
||||
content="".join(text),
|
||||
chunks=chunks,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage: Parse a CSV file and display its content
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
your_file = "/path/to/your/file.csv"
|
||||
parser = CSVParser()
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
# Display full document content
|
||||
logger.error(document.content)
|
||||
|
||||
# Display individual chunks (rows)
|
||||
for chunk in document.chunks:
|
||||
logger.error(chunk.content)
|
||||
@@ -89,11 +89,33 @@ class DocxParser(BaseParser):
|
||||
logger.info(f"Setting max_workers to {max_workers} for document processing")
|
||||
|
||||
try:
|
||||
inline_images: Dict[str, str] = {}
|
||||
|
||||
def _inline_upload(local_path: str) -> str:
|
||||
"""Read temp image file, base64-encode, and return a ref path.
|
||||
|
||||
The Go-side ImageResolver (or main.py _resolve_images) handles
|
||||
actual storage upload from Document.images.
|
||||
"""
|
||||
import base64
|
||||
import uuid as _uuid
|
||||
|
||||
try:
|
||||
with open(local_path, "rb") as f:
|
||||
raw = f.read()
|
||||
ext = os.path.splitext(local_path)[1].lower() or ".png"
|
||||
ref = f"images/{_uuid.uuid4().hex}{ext}"
|
||||
inline_images[ref] = base64.b64encode(raw).decode()
|
||||
return ref
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read temp image %s: %s", local_path, exc)
|
||||
return ""
|
||||
|
||||
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
|
||||
docx_processor = Docx(
|
||||
max_image_size=self.max_image_size,
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
upload_file=self.storage.upload_file,
|
||||
max_image_size=1920,
|
||||
enable_multimodal=True,
|
||||
upload_file=_inline_upload,
|
||||
)
|
||||
all_lines, tables = docx_processor(
|
||||
binary=content,
|
||||
@@ -153,6 +175,7 @@ class DocxParser(BaseParser):
|
||||
f"generated {len(text)} characters of text"
|
||||
)
|
||||
|
||||
image_parts.update(inline_images)
|
||||
return DocumentModel(content=text, images=image_parts)
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOCX document: {str(e)}")
|
||||
|
||||
@@ -5,44 +5,24 @@ import os
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
# Set up logger for this module
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageParser(BaseParser):
|
||||
"""
|
||||
Parser for image files with OCR capability.
|
||||
Extracts text from images and generates captions.
|
||||
"""Parser for standalone image files.
|
||||
|
||||
This parser handles image processing by:
|
||||
1. Uploading the image to storage
|
||||
2. Generating a descriptive caption
|
||||
3. Performing OCR to extract text content
|
||||
4. Returning a combined result with both text and image reference
|
||||
Returns the image as a markdown reference with the raw image data
|
||||
in Document.images so that the Go-side ImageResolver (or main.py's
|
||||
_resolve_images) can handle storage upload.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse image content into markdown text
|
||||
:param content: bytes content of the image
|
||||
:return: Document object
|
||||
"""
|
||||
logger.info(f"Parsing image content, size: {len(content)} bytes")
|
||||
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
|
||||
|
||||
# Get file extension
|
||||
ext = os.path.splitext(self.file_name)[1].lower()
|
||||
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
|
||||
ref_path = f"images/{self.file_name}"
|
||||
|
||||
# Upload image to storage
|
||||
image_url = self.storage.upload_bytes(content, file_ext=ext)
|
||||
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
|
||||
text = f""
|
||||
images = {ref_path: base64.b64encode(content).decode()}
|
||||
|
||||
if not image_url:
|
||||
logger.warning(f"Failed to upload image: {self.file_name}")
|
||||
return Document(content=f"{self.file_name}", images={})
|
||||
|
||||
# Generate markdown text
|
||||
text = f""
|
||||
images = {image_url: base64.b64encode(content).decode()}
|
||||
|
||||
# Create image object and add to map
|
||||
return Document(content=text, images=images)
|
||||
|
||||
@@ -348,23 +348,11 @@ class MarkdownImageUtil:
|
||||
|
||||
|
||||
class MarkdownImageBase64(BaseParser):
|
||||
"""Parser for extracting and uploading base64 images from Markdown.
|
||||
"""Parser for extracting base64 images from Markdown.
|
||||
|
||||
This parser:
|
||||
1. Extracts base64-encoded images from Markdown content
|
||||
2. Uploads the decoded images to storage
|
||||
3. Replaces base64 data with uploaded URLs
|
||||
4. Returns a Document with clean Markdown and image mappings
|
||||
|
||||
Requires:
|
||||
- self.storage: Storage backend for uploading images
|
||||
|
||||
Example:
|
||||
>>> parser = MarkdownImageBase64(storage=my_storage)
|
||||
>>> content = b""
|
||||
>>> doc = parser.parse_into_text(content)
|
||||
>>> print(doc.content)
|
||||

|
||||
Extracts base64-encoded images, replaces them with path references,
|
||||
and returns the raw image data in Document.images for the Go-side
|
||||
ImageResolver (or main.py _resolve_images) to handle storage.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
@@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Markdown and process base64 images.
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
Returns:
|
||||
Document with:
|
||||
- content: Markdown with base64 replaced by URLs
|
||||
- images: Dict mapping URLs to base64 strings
|
||||
"""
|
||||
# Convert byte content to string using universal decoding method
|
||||
text = endecode.decode_bytes(content)
|
||||
# Extract base64 images and replace with temporary paths
|
||||
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
|
||||
|
||||
# Final image mapping: URL -> base64 string (for Document)
|
||||
images: Dict[str, str] = {}
|
||||
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
|
||||
image_replace: Dict[str, str] = {}
|
||||
for ipath, raw_bytes in img_b64.items():
|
||||
images[ipath] = base64.b64encode(raw_bytes).decode()
|
||||
|
||||
logger.debug(f"Uploading {len(img_b64)} images from markdown")
|
||||
# Upload each extracted image to storage
|
||||
for ipath, b64_bytes in img_b64.items():
|
||||
# Get file extension for proper MIME type
|
||||
ext = os.path.splitext(ipath)[1].lower()
|
||||
# Upload binary data and get back URL
|
||||
image_url = self.storage.upload_bytes(b64_bytes, ext)
|
||||
|
||||
# Map temp path to uploaded URL for replacement
|
||||
image_replace[ipath] = image_url
|
||||
# Store base64 string in final images dict
|
||||
images[image_url] = base64.b64encode(b64_bytes).decode()
|
||||
|
||||
# Replace temporary paths with actual uploaded URLs
|
||||
text = self.image_helper.replace_path(text, image_replace)
|
||||
logger.debug("Extracted %d base64 images from markdown", len(images))
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
|
||||
@@ -1,329 +0,0 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Optional
|
||||
|
||||
import markdownify
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StdMinerUParser(BaseParser):
|
||||
"""
|
||||
Standard MinerU Parser for document parsing.
|
||||
|
||||
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
|
||||
with support for tables, formulas, and images extraction.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enable_markdownify: bool = True,
|
||||
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Initialize MinerU parser.
|
||||
|
||||
Args:
|
||||
enable_markdownify: Whether to convert HTML tables to markdown format
|
||||
mineru_endpoint: MinerU API endpoint URL
|
||||
**kwargs: Additional arguments passed to BaseParser
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
# Get MinerU endpoint from environment variable or parameter
|
||||
# Modified: 优先使用传入的参数,否则使用 Config
|
||||
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
|
||||
self.minerU = base_url.rstrip("/") if base_url else ""
|
||||
|
||||
self.enable_markdownify = enable_markdownify
|
||||
# Helper for processing markdown images
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
# Pattern to match base64 encoded images
|
||||
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
|
||||
# Check if MinerU API is available
|
||||
self.enable = self.ping()
|
||||
|
||||
def ping(self, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Check if MinerU API is available.
|
||||
|
||||
Args:
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
True if API is available, False otherwise
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
self.minerU + "/docs", timeout=timeout, allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content into text using MinerU API.
|
||||
|
||||
Args:
|
||||
content: Raw document content in bytes
|
||||
|
||||
Returns:
|
||||
Document object containing parsed text and images
|
||||
"""
|
||||
if not self.enable:
|
||||
logger.debug("MinerU API is not enabled")
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
|
||||
md_content: str = ""
|
||||
images_b64: Dict[str, str] = {}
|
||||
try:
|
||||
# Call MinerU API to parse document
|
||||
response = requests.post(
|
||||
url=self.minerU + "/file_parse",
|
||||
data={
|
||||
"return_md": True, # Return markdown content
|
||||
"return_images": True, # Return extracted images
|
||||
"lang_list": ["ch", "en"], # Support Chinese and English
|
||||
"table_enable": True, # Enable table parsing
|
||||
"formula_enable": True, # Enable formula parsing
|
||||
"parse_method": "auto", # Auto detect parsing method
|
||||
"start_page_id": 0, # Start from first page
|
||||
"end_page_id": 99999, # Parse all pages
|
||||
"backend": "pipeline", # Use pipeline backend
|
||||
"response_format_zip": False, # Return JSON instead of ZIP
|
||||
"return_middle_json": False, # Don't return intermediate JSON
|
||||
"return_model_output": False, # Don't return model output
|
||||
"return_content_list": False, # Don't return content list
|
||||
},
|
||||
files={"files": content},
|
||||
timeout=1000,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()["results"]["files"]
|
||||
md_content = result["md_content"]
|
||||
images_b64 = result.get("images", {})
|
||||
except Exception as e:
|
||||
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
# Convert HTML tables in markdown to markdown table format
|
||||
if self.enable_markdownify:
|
||||
logger.debug("Converting HTML to Markdown")
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
# Filter images that are actually used in markdown content
|
||||
# Some images in images_b64 may not be referenced in md_content
|
||||
# (e.g., images embedded in tables), so we need to filter them
|
||||
for ipath, b64_str in images_b64.items():
|
||||
# Skip images that are not referenced in markdown content
|
||||
if f"images/{ipath}" not in md_content:
|
||||
logger.debug(f"Image {ipath} not used in markdown")
|
||||
continue
|
||||
# Parse base64 image data
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
# Extract image format (e.g., png, jpg)
|
||||
file_ext = match.group(1)
|
||||
# Extract base64 encoded data
|
||||
b64_str = match.group(2)
|
||||
|
||||
# Decode base64 string to bytes
|
||||
image_bytes = endecode.encode_image(b64_str, errors="ignore")
|
||||
if not image_bytes:
|
||||
logger.error("Failed to decode base64 image skip it")
|
||||
continue
|
||||
|
||||
# Upload image to storage and get URL
|
||||
image_url = self.storage.upload_bytes(
|
||||
image_bytes, file_ext=f".{file_ext}"
|
||||
)
|
||||
|
||||
# Store image mapping for later use
|
||||
images[image_url] = b64_str
|
||||
# Prepare replacement mapping for markdown content
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
logger.info(f"Replaced {len(image_replace)} images in markdown")
|
||||
# Replace image paths in markdown with uploaded URLs
|
||||
text = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
logger.info(
|
||||
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
|
||||
)
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
|
||||
class MinerUCloudParser(StdMinerUParser):
|
||||
"""
|
||||
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
|
||||
Uses the /submit -> /status -> /result workflow.
|
||||
"""
|
||||
|
||||
SUBMIT_TIMEOUT = 30
|
||||
POLL_INTERVAL = 2
|
||||
MAX_WAIT_TIME = 600
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content using Cloud MinerU API (Async/Polling).
|
||||
"""
|
||||
if not self.enable:
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
|
||||
|
||||
try:
|
||||
# --- Step 1: Submit Task ---
|
||||
submit_url = f"{self.minerU}/submit"
|
||||
logger.info(f"Submitting task to {submit_url}")
|
||||
|
||||
response = requests.post(
|
||||
url=submit_url,
|
||||
files={"files": content},
|
||||
data={
|
||||
"enable_formula": "true",
|
||||
"enable_table": "true",
|
||||
"layout_model": "doclayout_yolo",
|
||||
"backend": "pipeline",
|
||||
},
|
||||
timeout=self.SUBMIT_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Robust task_id extraction
|
||||
resp_data = response.json()
|
||||
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
|
||||
|
||||
if not task_id:
|
||||
raise ValueError(f"No task_id in response: {resp_data}")
|
||||
|
||||
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
|
||||
|
||||
# --- Step 2: Poll Status ---
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
if time.time() - start_time > self.MAX_WAIT_TIME:
|
||||
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
|
||||
|
||||
try:
|
||||
status_resp = requests.get(
|
||||
f"{self.minerU}/status/{task_id}",
|
||||
timeout=10
|
||||
)
|
||||
status_resp.raise_for_status()
|
||||
status_data = status_resp.json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
continue
|
||||
|
||||
state = status_data.get("status") or status_data.get("state")
|
||||
|
||||
if state in ["done", "success"]:
|
||||
break
|
||||
elif state == "failed":
|
||||
error_msg = status_data.get("error") or "Unknown error"
|
||||
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
|
||||
else:
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
|
||||
# --- Step 3: Get Result ---
|
||||
result_resp = requests.get(
|
||||
f"{self.minerU}/result/{task_id}",
|
||||
timeout=30
|
||||
)
|
||||
result_resp.raise_for_status()
|
||||
result_json = result_resp.json()
|
||||
|
||||
# Normalize result data
|
||||
result_data = result_json.get("result", result_json)
|
||||
|
||||
md_content = result_data.get("md_content", "")
|
||||
images_b64 = result_data.get("images", {})
|
||||
|
||||
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
|
||||
|
||||
# Convert HTML tables
|
||||
if self.enable_markdownify:
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
|
||||
for ipath, b64_str in images_b64.items():
|
||||
if f"images/{ipath}" not in md_content:
|
||||
continue
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
file_ext = match.group(1)
|
||||
b64_str_clean = match.group(2)
|
||||
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
|
||||
if not image_bytes: continue
|
||||
|
||||
if self.storage:
|
||||
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
|
||||
images[image_url] = b64_str_clean
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
if image_replace:
|
||||
md_content = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
return Document(content=md_content, images=images)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
|
||||
class MinerUParser(PipelineParser):
|
||||
"""
|
||||
MinerU Parser with pipeline processing.
|
||||
|
||||
This parser combines StdMinerUParser for document parsing and
|
||||
MarkdownTableFormatter for table formatting in a pipeline.
|
||||
"""
|
||||
|
||||
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
|
||||
# Example usage for testing
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
# Configure your file path and MinerU endpoint
|
||||
your_file = "/path/to/your/file.pdf"
|
||||
|
||||
# Added: 修改为 Localhost 方便测试
|
||||
test_endpoint = "http://localhost:9987"
|
||||
os.environ["MINERU_ENDPOINT"] = test_endpoint
|
||||
|
||||
# Create parser instance
|
||||
# Modified: 传入 endpoint
|
||||
parser = MinerUParser(mineru_endpoint=test_endpoint)
|
||||
|
||||
# Parse PDF file
|
||||
if os.path.exists(your_file):
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
logger.error(document.content)
|
||||
else:
|
||||
print(f"File not found: {your_file}")
|
||||
+42
-138
@@ -1,178 +1,82 @@
|
||||
import logging
|
||||
from typing import Dict, Type
|
||||
from typing import Any, Optional
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.csv_parser import CSVParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
from docreader.parser.text_parser import TextParser
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.parser.web_parser import WebParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""Document parser facade (lightweight version).
|
||||
|
||||
Converts files/URLs to markdown + image references.
|
||||
No chunking, no storage, no OCR, no VLM.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Initialize all parser types - maps file extensions to their corresponding parser classes
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
# Document formats
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
# Image formats - all use the same ImageParser
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
# Alternative markdown extension
|
||||
"markdown": MarkdownParser,
|
||||
# Spreadsheet formats
|
||||
"csv": CSVParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
}
|
||||
self.registry = registry
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
"Parser initialized with engines: %s",
|
||||
", ".join(self.registry.get_engine_names()),
|
||||
)
|
||||
|
||||
def get_parser(self, file_type: str) -> Type[BaseParser]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
# Look up parser by file type (case-insensitive)
|
||||
parser = self.parsers.get(file_type.lower())
|
||||
if not parser:
|
||||
# Raise error if file type is not supported
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
"""Parse file content to markdown."""
|
||||
engine = parser_engine or ""
|
||||
overrides = engine_overrides or {}
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
"Parsing file: %s, type: %s, engine: %s",
|
||||
file_name,
|
||||
file_type,
|
||||
engine or "builtin",
|
||||
)
|
||||
|
||||
# Get appropriate parser class for the file type
|
||||
cls = self.get_parser(file_type)
|
||||
|
||||
# Create parser instance with configuration
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
cls = self.registry.get_parser_class(engine, file_type)
|
||||
logger.info(
|
||||
"Creating %s parser instance for %s file",
|
||||
cls.__name__,
|
||||
file_type,
|
||||
)
|
||||
parser = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
# Execute the parsing process
|
||||
logger.info("Starting to parse file content, size: %d bytes", len(content))
|
||||
result = parser.parse(content)
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for file: {file_name}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for file: %s", file_name)
|
||||
logger.info(
|
||||
"Parsed file %s, content length=%d", file_name, len(result.content)
|
||||
)
|
||||
return result
|
||||
|
||||
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
# Create web parser instance with configuration
|
||||
logger.info("Creating WebParser instance")
|
||||
parser = WebParser(
|
||||
title=title, # Webpage title for metadata
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
)
|
||||
def parse_url(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""Parse content from a URL to markdown."""
|
||||
logger.info("Parsing URL: %s, title: %s", url, title)
|
||||
|
||||
parser = WebParser(title=title)
|
||||
logger.info("Starting to parse URL content")
|
||||
# Parse URL content (encode URL string to bytes as required by parser interface)
|
||||
result = parser.parse(url.encode())
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for url: {url}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for url: {url}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for url: %s", url)
|
||||
logger.info("Parsed url %s, content length=%d", url, len(result.content))
|
||||
return result
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from docreader.parser.chain_parser import FirstParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.mineru_parser import MinerUParser
|
||||
|
||||
|
||||
class PDFParser(FirstParser):
|
||||
@@ -13,4 +12,4 @@ class PDFParser(FirstParser):
|
||||
The first successful parser result will be returned.
|
||||
"""
|
||||
# Parser classes to try in order (chain of responsibility pattern)
|
||||
_parser_cls = (MinerUParser, MarkitdownParser)
|
||||
_parser_cls = (MarkitdownParser,)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import logging
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
|
||||
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BUILTIN_ENGINE = "builtin"
|
||||
|
||||
|
||||
class ParserEngineRegistry:
|
||||
"""Registry for parser engines.
|
||||
|
||||
Each engine maps file extensions to parser classes.
|
||||
When a requested engine doesn't support a file type, the registry
|
||||
falls back to the builtin engine automatically.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
|
||||
self._descriptions: Dict[str, str] = {}
|
||||
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
|
||||
self._unavailable_hint: Dict[str, str] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
name: str,
|
||||
file_types: Dict[str, Type[BaseParser]],
|
||||
description: str = "",
|
||||
check_available: Callable[..., Tuple[bool, str]] | None = None,
|
||||
unavailable_hint: str = "",
|
||||
):
|
||||
self._engines[name] = file_types
|
||||
self._descriptions[name] = description
|
||||
if check_available is not None:
|
||||
self._check_available[name] = check_available
|
||||
self._unavailable_hint[name] = unavailable_hint
|
||||
logger.info(
|
||||
"Registered parser engine '%s' with file types: %s",
|
||||
name,
|
||||
", ".join(file_types.keys()),
|
||||
)
|
||||
|
||||
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
|
||||
"""Resolve parser class for the given engine and file type.
|
||||
|
||||
Falls back to builtin engine when the requested engine doesn't
|
||||
support the file type.
|
||||
"""
|
||||
ft = file_type.lower()
|
||||
|
||||
if engine and engine in self._engines:
|
||||
cls = self._engines[engine].get(ft)
|
||||
if cls:
|
||||
logger.info("Using engine '%s' for file type '%s'", engine, ft)
|
||||
return cls
|
||||
logger.info(
|
||||
"Engine '%s' does not support '%s', falling back to builtin",
|
||||
engine,
|
||||
ft,
|
||||
)
|
||||
|
||||
builtin = self._engines.get(BUILTIN_ENGINE, {})
|
||||
cls = builtin.get(ft)
|
||||
if cls:
|
||||
return cls
|
||||
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
|
||||
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
|
||||
"""Return metadata for all registered engines, including availability.
|
||||
|
||||
Args:
|
||||
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
|
||||
forwarded to each engine's check_available function.
|
||||
"""
|
||||
result = []
|
||||
for name, parsers in self._engines.items():
|
||||
available = True
|
||||
unavailable_reason = ""
|
||||
check = self._check_available.get(name)
|
||||
if check is not None:
|
||||
try:
|
||||
available, unavailable_reason = check(overrides)
|
||||
except Exception as e:
|
||||
available = False
|
||||
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
|
||||
if not available and not unavailable_reason:
|
||||
unavailable_reason = self._unavailable_hint.get(name, "不可用")
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": self._descriptions.get(name, ""),
|
||||
"file_types": sorted(parsers.keys()),
|
||||
"available": available,
|
||||
"unavailable_reason": unavailable_reason,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def get_engine_names(self) -> List[str]:
|
||||
return list(self._engines.keys())
|
||||
|
||||
|
||||
def _build_default_registry() -> ParserEngineRegistry:
|
||||
"""Create and populate the default registry with all known engines."""
|
||||
reg = ParserEngineRegistry()
|
||||
|
||||
_image_types = {
|
||||
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
|
||||
}
|
||||
|
||||
reg.register(
|
||||
BUILTIN_ENGINE,
|
||||
{
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"markdown": MarkdownParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
**_image_types,
|
||||
},
|
||||
description="内置解析引擎",
|
||||
)
|
||||
|
||||
reg.register(
|
||||
"markitdown",
|
||||
{
|
||||
"md": MarkitdownParser,
|
||||
"markdown": MarkitdownParser,
|
||||
"pdf": MarkitdownParser,
|
||||
"docx": MarkitdownParser,
|
||||
"doc": MarkitdownParser,
|
||||
"pptx": MarkitdownParser,
|
||||
"ppt": MarkitdownParser,
|
||||
"xlsx": MarkitdownParser,
|
||||
"xls": MarkitdownParser,
|
||||
"csv": MarkitdownParser,
|
||||
},
|
||||
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
|
||||
)
|
||||
|
||||
# NOTE: Engine listing is managed by Go-side engine registry
|
||||
# (docparser.ListAllEngines). The Python list_engines method is kept for
|
||||
# backward compatibility with the gRPC ListEngines RPC but the Go app
|
||||
# no longer calls it. MinerU engines are handled natively by Go.
|
||||
|
||||
return reg
|
||||
|
||||
|
||||
registry = _build_default_registry()
|
||||
+98
-222
@@ -5,96 +5,70 @@ import os
|
||||
import traceback
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict
|
||||
from typing import Dict, Optional
|
||||
|
||||
from minio import Minio
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
|
||||
"""Read a value from storage_config dict, falling back to env vars."""
|
||||
if storage_config:
|
||||
v = storage_config.get(key, "")
|
||||
if v:
|
||||
return str(v)
|
||||
for ek in env_keys:
|
||||
v = os.environ.get(ek, "")
|
||||
if v:
|
||||
return v
|
||||
return default
|
||||
|
||||
|
||||
class Storage(ABC):
|
||||
"""Abstract base class for object storage operations"""
|
||||
|
||||
@abstractmethod
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to object storage
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to object storage
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CosStorage(Storage):
|
||||
"""Tencent Cloud COS storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize COS storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.region, self.prefix = (
|
||||
self._init_cos_client()
|
||||
)
|
||||
|
||||
def _init_cos_client(self):
|
||||
"""Initialize Tencent Cloud COS client"""
|
||||
try:
|
||||
# Use provided COS config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if self.storage_config and self.storage_config.get("access_key_id") != "":
|
||||
cos_config = self.storage_config
|
||||
secret_id = cos_config.get("access_key_id")
|
||||
secret_key = cos_config.get("secret_access_key")
|
||||
region = cos_config.get("region")
|
||||
bucket_name = cos_config.get("bucket_name")
|
||||
appid = cos_config.get("app_id")
|
||||
prefix = cos_config.get("path_prefix", "")
|
||||
else:
|
||||
# Get COS configuration from environment variables
|
||||
secret_id = CONFIG.cos_secret_id
|
||||
secret_key = CONFIG.cos_secret_key
|
||||
region = CONFIG.cos_region
|
||||
bucket_name = CONFIG.cos_bucket_name
|
||||
appid = CONFIG.cos_app_id
|
||||
prefix = CONFIG.cos_path_prefix
|
||||
|
||||
enable_old_domain = CONFIG.cos_enable_old_domain
|
||||
sc = self.storage_config
|
||||
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
|
||||
region = _cfg(sc, "region", "COS_REGION")
|
||||
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
|
||||
appid = _cfg(sc, "app_id", "COS_APP_ID")
|
||||
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
|
||||
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([secret_id, secret_key, region, bucket_name, appid]):
|
||||
logger.error(
|
||||
"Incomplete COS configuration, missing environment variables"
|
||||
f"secret_id: {secret_id}, secret_key: {secret_key}, "
|
||||
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
|
||||
"Incomplete COS configuration: "
|
||||
"secret_id=%s, region=%s, bucket=%s, appid=%s",
|
||||
bool(secret_id), region, bucket_name, appid,
|
||||
)
|
||||
return None, None, None, None
|
||||
|
||||
# Initialize COS configuration
|
||||
logger.info(
|
||||
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
|
||||
)
|
||||
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
|
||||
config = CosConfig(
|
||||
Appid=appid,
|
||||
Region=region,
|
||||
@@ -102,93 +76,50 @@ class CosStorage(Storage):
|
||||
SecretKey=secret_key,
|
||||
EnableOldDomain=enable_old_domain,
|
||||
)
|
||||
|
||||
# Create client
|
||||
client = CosS3Client(config)
|
||||
return client, bucket_name, region, prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize COS client: {str(e)}")
|
||||
logger.error("Failed to initialize COS client: %s", e)
|
||||
return None, None, None, None
|
||||
|
||||
def _get_download_url(self, bucket_name, region, object_key):
|
||||
"""Generate COS object URL
|
||||
|
||||
Args:
|
||||
bucket_name: Bucket name
|
||||
region: Region
|
||||
object_key: Object key
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to COS: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to COS")
|
||||
self.client.upload_file(
|
||||
Bucket=self.bucket_name,
|
||||
LocalFilePath=file_path,
|
||||
Key=object_key,
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to COS: {file_url}")
|
||||
logger.info("COS upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to COS: {str(e)}")
|
||||
logger.error("COS upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket_name, Body=content, Key=object_key
|
||||
)
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
|
||||
logger.info("COS upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to COS: {str(e)}")
|
||||
logger.error("COS upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
@@ -196,67 +127,34 @@ class CosStorage(Storage):
|
||||
class MinioStorage(Storage):
|
||||
"""MinIO storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize MinIO storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
|
||||
self._init_minio_client()
|
||||
)
|
||||
|
||||
def _init_minio_client(self):
|
||||
"""Initialize MinIO client from environment variables or injected config.
|
||||
|
||||
If storage_config contains valid configuration, prefer those values
|
||||
to override environment variables.
|
||||
"""
|
||||
try:
|
||||
# Get configuration from storage_config with environment variables as fallback
|
||||
# Each field can independently fall back to environment variables
|
||||
access_key = (
|
||||
self.storage_config.get("access_key_id")
|
||||
if self.storage_config and self.storage_config.get("access_key_id")
|
||||
else CONFIG.minio_access_key_id
|
||||
)
|
||||
secret_key = (
|
||||
self.storage_config.get("secret_access_key")
|
||||
if self.storage_config and self.storage_config.get("secret_access_key")
|
||||
else CONFIG.minio_secret_access_key
|
||||
)
|
||||
bucket_name = (
|
||||
self.storage_config.get("bucket_name")
|
||||
if self.storage_config and self.storage_config.get("bucket_name")
|
||||
else CONFIG.minio_bucket_name
|
||||
)
|
||||
path_prefix_raw = (
|
||||
self.storage_config.get("path_prefix")
|
||||
if self.storage_config and self.storage_config.get("path_prefix")
|
||||
else CONFIG.minio_path_prefix
|
||||
)
|
||||
sc = self.storage_config
|
||||
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
|
||||
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
|
||||
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
|
||||
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
|
||||
|
||||
endpoint = CONFIG.minio_endpoint
|
||||
use_ssl = CONFIG.minio_use_ssl
|
||||
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
|
||||
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([endpoint, access_key, secret_key, bucket_name]):
|
||||
logger.error(
|
||||
"Incomplete MinIO configuration, missing environment variables"
|
||||
)
|
||||
logger.error("Incomplete MinIO configuration")
|
||||
return None, None, None, None, None
|
||||
|
||||
# Initialize client
|
||||
client = Minio(
|
||||
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
|
||||
)
|
||||
|
||||
# Ensure bucket exists
|
||||
found = client.bucket_exists(bucket_name)
|
||||
if not found:
|
||||
client.make_bucket(bucket_name)
|
||||
# Set public read policy for the bucket
|
||||
policy = (
|
||||
"{"
|
||||
'"Version":"2012-10-17",'
|
||||
@@ -273,51 +171,26 @@ class MinioStorage(Storage):
|
||||
|
||||
return client, bucket_name, use_ssl, endpoint, path_prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize MinIO client: {str(e)}")
|
||||
logger.error("Failed to initialize MinIO client: %s", e)
|
||||
return None, None, None, None, None
|
||||
|
||||
def _get_download_url(self, object_key: str):
|
||||
"""Construct a public URL for MinIO object.
|
||||
|
||||
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
|
||||
"""
|
||||
# 1. Use public endpoint if provided
|
||||
endpoint = CONFIG.minio_public_endpoint
|
||||
if endpoint:
|
||||
return f"{endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 2. Use SSL if enabled
|
||||
if self.use_ssl:
|
||||
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 3. Use HTTP default
|
||||
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
|
||||
if public_endpoint:
|
||||
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
|
||||
scheme = "https" if self.use_ssl else "http"
|
||||
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to MinIO
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to MinIO: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_name = os.path.basename(file_path)
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to MinIO")
|
||||
with open(file_path, "rb") as file_data:
|
||||
file_size = os.path.getsize(file_path)
|
||||
self.client.put_object(
|
||||
@@ -327,38 +200,22 @@ class MinioStorage(Storage):
|
||||
length=file_size,
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to MinIO
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
self.client.put_object(
|
||||
self.bucket_name or "",
|
||||
object_key,
|
||||
@@ -367,52 +224,67 @@ class MinioStorage(Storage):
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
file_url = self._get_download_url(object_key)
|
||||
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
class LocalStorage(Storage):
|
||||
"""Local file system storage implementation"""
|
||||
"""Local file system storage implementation.
|
||||
|
||||
def __init__(self, storage_config: Dict[str, str] = {}):
|
||||
self.storage_config = storage_config
|
||||
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
|
||||
self.image_dir = os.path.join(base_dir, "images")
|
||||
Saves files under base_dir and returns web-accessible URL paths
|
||||
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
|
||||
"""
|
||||
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
sc = storage_config or {}
|
||||
self.base_dir = (
|
||||
sc.get("base_dir")
|
||||
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
|
||||
)
|
||||
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
|
||||
if path_prefix:
|
||||
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
|
||||
else:
|
||||
self.image_dir = os.path.join(self.base_dir, "images")
|
||||
self.url_prefix = (
|
||||
sc.get("url_prefix")
|
||||
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
|
||||
)
|
||||
os.makedirs(self.image_dir, exist_ok=True)
|
||||
|
||||
def _to_url(self, fpath: str) -> str:
|
||||
if self.url_prefix:
|
||||
rel = os.path.relpath(fpath, self.base_dir)
|
||||
return f"{self.url_prefix}/{rel}"
|
||||
return fpath
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to local storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to local storage: {len(content)} bytes")
|
||||
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fname, "wb") as f:
|
||||
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fpath, "wb") as f:
|
||||
f.write(content)
|
||||
return fname
|
||||
url = self._to_url(fpath)
|
||||
logger.info("Local storage saved: %s -> %s", fpath, url)
|
||||
return url
|
||||
|
||||
|
||||
class Base64Storage(Storage):
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
|
||||
file_ext = file_ext.lstrip(".")
|
||||
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
|
||||
|
||||
|
||||
class DummyStorage(Storage):
|
||||
"""Dummy storage implementation.
|
||||
|
||||
It is used in tests or in environments where object storage is disabled.
|
||||
All upload methods return empty string.
|
||||
"""
|
||||
"""Dummy storage — all uploads return empty string."""
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
return ""
|
||||
@@ -421,26 +293,30 @@ class DummyStorage(Storage):
|
||||
return ""
|
||||
|
||||
|
||||
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
"""Create a storage instance based on configuration or environment variables
|
||||
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
|
||||
"""Create a storage instance based on storage_config dict.
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration dictionary
|
||||
|
||||
Returns:
|
||||
Storage instance
|
||||
The ``provider`` key in storage_config determines the backend:
|
||||
minio, cos, local, base64.
|
||||
Falls back to STORAGE_TYPE env var, then ``local``.
|
||||
"""
|
||||
storage_type = CONFIG.storage_type
|
||||
storage_type = ""
|
||||
if storage_config:
|
||||
storage_type = str(storage_config.get("provider", storage_type)).lower()
|
||||
logger.info(f"Creating {storage_type} storage instance")
|
||||
provider = str(storage_config.get("provider", "")).lower().strip()
|
||||
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
|
||||
storage_type = provider
|
||||
|
||||
if not storage_type:
|
||||
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
|
||||
|
||||
logger.info("Creating %s storage instance", storage_type)
|
||||
|
||||
if storage_type == "minio":
|
||||
return MinioStorage(storage_config)
|
||||
elif storage_type == "cos":
|
||||
return CosStorage(storage_config)
|
||||
elif storage_type == "local":
|
||||
return LocalStorage(storage_config or {})
|
||||
return LocalStorage(storage_config)
|
||||
elif storage_type == "base64":
|
||||
return Base64Storage()
|
||||
return DummyStorage()
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import logging
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = endecode.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return Document(content=text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
Reference in New Issue
Block a user