feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+11 -12
View File
@@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
meaningful chunks for further processing and indexing.
"""
from .csv_parser import CSVParser
from .doc_parser import DocParser
from .docx2_parser import Docx2Parser
from .excel_parser import ExcelParser
@@ -21,19 +20,19 @@ from .image_parser import ImageParser
from .markdown_parser import MarkdownParser
from .parser import Parser
from .pdf_parser import PDFParser
from .text_parser import TextParser
from .registry import ParserEngineRegistry, registry
from .web_parser import WebParser
# Export public classes and modules
__all__ = [
"Docx2Parser", # Parser for .docx files (modern Word documents)
"DocParser", # Parser for .doc files (legacy Word documents)
"PDFParser", # Parser for PDF documents
"MarkdownParser", # Parser for Markdown text files
"TextParser", # Parser for plain text files
"ImageParser", # Parser for images with text content
"WebParser", # Parser for web pages
"Parser", # Main parser factory that selects the appropriate parser
"CSVParser", # Parser for CSV files
"ExcelParser", # Parser for Excel files
"Docx2Parser",
"DocParser",
"PDFParser",
"MarkdownParser",
"ImageParser",
"WebParser",
"Parser",
"ExcelParser",
"ParserEngineRegistry",
"registry",
]
File diff suppressed because it is too large Load Diff
-387
View File
@@ -1,387 +0,0 @@
import json
import logging
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Union
import ollama
import requests
from docreader.config import CONFIG
logger = logging.getLogger(__name__)
@dataclass
class ImageUrl:
"""Image URL data structure for caption requests."""
url: Optional[str] = None
detail: Optional[str] = None
@dataclass
class Content:
"""Content data structure that can contain text or image URL."""
type: Optional[str] = None
text: Optional[str] = None
image_url: Optional[ImageUrl] = None
@dataclass
class SystemMessage:
"""System message for VLM model requests."""
role: Optional[str] = None
content: Optional[str] = None
@dataclass
class UserMessage:
"""User message for VLM model requests, can contain multiple content items."""
role: Optional[str] = None
content: List[Content] = field(default_factory=list)
@dataclass
class CompletionRequest:
"""Request structure for VLM model completion API."""
model: str
temperature: float
top_p: float
messages: List[Union[SystemMessage, UserMessage]]
user: str
@dataclass
class Model:
"""Model identifier structure."""
id: str
@dataclass
class ModelsResp:
"""Response structure for available models API."""
data: List[Model] = field(default_factory=list)
@dataclass
class Message:
"""Message structure in API response."""
role: Optional[str] = None
content: Optional[str] = None
tool_calls: Optional[str] = None
@dataclass
class Choice:
"""Choice structure in API response."""
message: Optional[Message] = None
@dataclass
class Usage:
"""Token usage information in API response."""
prompt_tokens: Optional[int] = 0
total_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
@dataclass
class CaptionChatResp:
"""Response structure for caption chat API."""
id: Optional[str] = None
created: Optional[int] = None
model: Optional[Model] = None
object: Optional[str] = None
choices: List[Choice] = field(default_factory=list)
usage: Optional[Usage] = None
@staticmethod
def from_json(json_data: dict) -> "CaptionChatResp":
"""
Parse API response JSON into a CaptionChatResp object.
Args:
json_data: The JSON response from the API
Returns:
A parsed CaptionChatResp object
"""
logger.info("Parsing CaptionChatResp from JSON")
# Manually parse nested fields with safe field extraction
choices = []
for choice in json_data.get("choices", []):
message_data = choice.get("message", {})
message = Message(
role=message_data.get("role"),
content=message_data.get("content"),
tool_calls=message_data.get("tool_calls"),
)
choices.append(Choice(message=message))
# Handle usage with safe field extraction
usage_data = json_data.get("usage", {})
usage = None
if usage_data:
usage = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
)
logger.info(
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
)
return CaptionChatResp(
id=json_data.get("id"),
created=json_data.get("created"),
model=json_data.get("model"),
object=json_data.get("object"),
choices=choices,
usage=usage,
)
def choice_data(self) -> str:
"""
Extract the content from the first choice in the response.
Returns:
The content string from the first choice, or empty string if no choices
"""
if (
not self.choices
or not self.choices[0]
or not self.choices[0].message
or not self.choices[0].message.content
):
logger.warning("No choices available in response")
return ""
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
class Caption:
"""
Service for generating captions for images using a Vision Language Model.
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
"""
Initialize the Caption service with configuration
from parameters or environment variables.
"""
logger.info("Initializing Caption service")
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
self.prompt = """简单凝炼的描述图片的主要内容"""
# API request timeout in seconds
self.timeout = 30
# Use provided VLM config if available,
# otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
# Build completion URL from provided base URL
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
# Interface type: "ollama" or "openai" (default)
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
# Fall back to environment variables if config not provided
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
self.model = CONFIG.vlm_model_name
self.api_key = CONFIG.vlm_model_api_key
self.interface_type = CONFIG.vlm_interface_type
# Validate interface type - must be either "ollama" or "openai"
if self.interface_type not in ["ollama", "openai"]:
logger.warning(
f"Unknown interface type: {self.interface_type}, defaulting to openai"
)
self.interface_type = "openai"
logger.info(
f"Configured with model: {self.model}, "
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
"""
Call the Caption API to generate a description for the given image.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info("Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50]}...")
# Route to appropriate API based on interface type
if self.interface_type == "ollama":
return self._call_ollama_api(image_data)
else:
return self._call_openai_api(image_data)
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call Ollama API for image captioning using base64 encoded image data."""
# Extract host URL by removing the chat completions endpoint
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
# Initialize Ollama client with host and timeout
client = ollama.Client(
host=host,
timeout=self.timeout,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# Call Ollama API with base64 encoded image
# Prompt: "Briefly describe the main content of the image"
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # Pass base64 encoded image data
options={
"temperature": 0.1
}, # Low temperature for more deterministic output
stream=False,
)
# Construct response object in standard format
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=Model(id=self.model),
object="chat.completion",
choices=[
Choice(message=Message(role="assistant", content=response.response))
],
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
# Construct user message with text prompt and base64 encoded image
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url",
image_url=ImageUrl(
url="data:image/png;base64," + image_base64, detail="auto"
),
),
],
)
# Build completion request with model parameters
gpt_req = CompletionRequest(
model=self.model,
temperature=0.3, # Moderate randomness for balanced output
top_p=0.8, # Nucleus sampling parameter
messages=[user_msg],
user="abc",
)
# Set up HTTP headers for the API request
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
}
# Add authorization header if API key is provided
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(
f"Sending request to OpenAI-compatible API with model: {self.model}"
)
# Send POST request to the API endpoint
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=self.timeout,
)
# Check for successful response
if response.status_code != 200:
logger.error(
f"OpenAI API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(f"Received from OpenAI with status: {response.status_code}")
logger.info("Converting response to CaptionChatResp object")
# Parse JSON response into structured object
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
logger.info(
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
f"completion_tokens={caption_resp.usage.completion_tokens}"
)
return caption_resp
except requests.exceptions.Timeout:
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
return None
except Exception as e:
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
return None
def get_caption(self, image_data: str) -> str:
"""
Get a caption for the provided image data.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
Caption text as string, or empty string if captioning failed
"""
logger.info("Getting caption for image")
if not image_data or self.completion_url is None:
logger.error("Image data is not set")
return ""
caption_resp = self._call_caption_api(image_data)
if caption_resp:
caption = caption_resp.choice_data()
caption_length = len(caption)
logger.info(f"Successfully generated caption of length {caption_length}")
logger.info(
f"Caption: {caption[:50]}..."
if caption_length > 50
else f"Caption: {caption}"
)
return caption
logger.warning("Failed to get caption from Caption API")
return ""
-101
View File
@@ -1,101 +0,0 @@
"""
CSV Parser Module
This module provides a parser for CSV (Comma-Separated Values) files.
It converts CSV data into a Document with structured chunks, where each row
becomes a separate chunk with key-value pairs.
"""
import logging
from io import BytesIO
from typing import List
import pandas as pd
from docreader.models.document import Chunk, Document
from docreader.parser.base_parser import BaseParser
logger = logging.getLogger(__name__)
class CSVParser(BaseParser):
"""
Parser for CSV files that converts tabular data into structured text.
This parser reads CSV content and transforms each row into a formatted string
with column-value pairs. Each row is stored as a separate Chunk in the Document,
allowing for granular access to individual records.
The output format for each row is:
"column1: value1, column2: value2, column3: value3\n"
Usage:
parser = CSVParser()
with open("data.csv", "rb") as f:
document = parser.parse_into_text(f.read())
"""
def parse_into_text(self, content: bytes) -> Document:
"""Parse CSV content into a Document with structured chunks.
Each row in the CSV is converted into a formatted string and stored as
a separate Chunk. The chunks maintain sequential order and track their
position in the overall document.
Args:
content: Raw bytes content of the CSV file
Returns:
Document: A Document object containing:
- content: Full text with all rows concatenated
- chunks: List of Chunk objects, one per CSV row
Note:
Bad lines in the CSV are automatically skipped using pandas'
on_bad_lines="skip" parameter.
"""
chunks: List[Chunk] = []
text: List[str] = []
start, end = 0, 0
# Read CSV content into a pandas DataFrame, skipping malformed lines
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
# Process each row in the DataFrame
for i, (idx, row) in enumerate(df.iterrows()):
# Format row as "column: value" pairs separated by commas
content_row = (
",".join(
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
)
+ "\n"
)
# Update end position for this chunk
end += len(content_row)
text.append(content_row)
# Create a chunk for this row with position tracking
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
# Update start position for next chunk
start = end
return Document(
content="".join(text),
chunks=chunks,
)
if __name__ == "__main__":
# Example usage: Parse a CSV file and display its content
logging.basicConfig(level=logging.DEBUG)
your_file = "/path/to/your/file.csv"
parser = CSVParser()
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
# Display full document content
logger.error(document.content)
# Display individual chunks (rows)
for chunk in document.chunks:
logger.error(chunk.content)
+26 -3
View File
@@ -89,11 +89,33 @@ class DocxParser(BaseParser):
logger.info(f"Setting max_workers to {max_workers} for document processing")
try:
inline_images: Dict[str, str] = {}
def _inline_upload(local_path: str) -> str:
"""Read temp image file, base64-encode, and return a ref path.
The Go-side ImageResolver (or main.py _resolve_images) handles
actual storage upload from Document.images.
"""
import base64
import uuid as _uuid
try:
with open(local_path, "rb") as f:
raw = f.read()
ext = os.path.splitext(local_path)[1].lower() or ".png"
ref = f"images/{_uuid.uuid4().hex}{ext}"
inline_images[ref] = base64.b64encode(raw).decode()
return ref
except Exception as exc:
logger.warning("Failed to read temp image %s: %s", local_path, exc)
return ""
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
docx_processor = Docx(
max_image_size=self.max_image_size,
enable_multimodal=self.enable_multimodal,
upload_file=self.storage.upload_file,
max_image_size=1920,
enable_multimodal=True,
upload_file=_inline_upload,
)
all_lines, tables = docx_processor(
binary=content,
@@ -153,6 +175,7 @@ class DocxParser(BaseParser):
f"generated {len(text)} characters of text"
)
image_parts.update(inline_images)
return DocumentModel(content=text, images=image_parts)
except Exception as e:
logger.error(f"Error parsing DOCX document: {str(e)}")
+9 -29
View File
@@ -5,44 +5,24 @@ import os
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
# Set up logger for this module
logger = logging.getLogger(__name__)
class ImageParser(BaseParser):
"""
Parser for image files with OCR capability.
Extracts text from images and generates captions.
"""Parser for standalone image files.
This parser handles image processing by:
1. Uploading the image to storage
2. Generating a descriptive caption
3. Performing OCR to extract text content
4. Returning a combined result with both text and image reference
Returns the image as a markdown reference with the raw image data
in Document.images so that the Go-side ImageResolver (or main.py's
_resolve_images) can handle storage upload.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse image content into markdown text
:param content: bytes content of the image
:return: Document object
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
# Get file extension
ext = os.path.splitext(self.file_name)[1].lower()
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
ref_path = f"images/{self.file_name}"
# Upload image to storage
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
text = f"![{self.file_name}]({ref_path})"
images = {ref_path: base64.b64encode(content).decode()}
if not image_url:
logger.warning(f"Failed to upload image: {self.file_name}")
return Document(content=f"{self.file_name}", images={})
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
# Create image object and add to map
return Document(content=text, images=images)
+7 -46
View File
@@ -348,23 +348,11 @@ class MarkdownImageUtil:
class MarkdownImageBase64(BaseParser):
"""Parser for extracting and uploading base64 images from Markdown.
"""Parser for extracting base64 images from Markdown.
This parser:
1. Extracts base64-encoded images from Markdown content
2. Uploads the decoded images to storage
3. Replaces base64 data with uploaded URLs
4. Returns a Document with clean Markdown and image mappings
Requires:
- self.storage: Storage backend for uploading images
Example:
>>> parser = MarkdownImageBase64(storage=my_storage)
>>> content = b"![logo](data:image/png;base64,iVBORw0...)"
>>> doc = parser.parse_into_text(content)
>>> print(doc.content)
![logo](https://storage.com/uuid.png)
Extracts base64-encoded images, replaces them with path references,
and returns the raw image data in Document.images for the Go-side
ImageResolver (or main.py _resolve_images) to handle storage.
"""
def __init__(self, **kwargs):
@@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse Markdown and process base64 images.
Args:
content: Raw Markdown content as bytes
Returns:
Document with:
- content: Markdown with base64 replaced by URLs
- images: Dict mapping URLs to base64 strings
"""
# Convert byte content to string using universal decoding method
text = endecode.decode_bytes(content)
# Extract base64 images and replace with temporary paths
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
# Final image mapping: URL -> base64 string (for Document)
images: Dict[str, str] = {}
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
image_replace: Dict[str, str] = {}
for ipath, raw_bytes in img_b64.items():
images[ipath] = base64.b64encode(raw_bytes).decode()
logger.debug(f"Uploading {len(img_b64)} images from markdown")
# Upload each extracted image to storage
for ipath, b64_bytes in img_b64.items():
# Get file extension for proper MIME type
ext = os.path.splitext(ipath)[1].lower()
# Upload binary data and get back URL
image_url = self.storage.upload_bytes(b64_bytes, ext)
# Map temp path to uploaded URL for replacement
image_replace[ipath] = image_url
# Store base64 string in final images dict
images[image_url] = base64.b64encode(b64_bytes).decode()
# Replace temporary paths with actual uploaded URLs
text = self.image_helper.replace_path(text, image_replace)
logger.debug("Extracted %d base64 images from markdown", len(images))
return Document(content=text, images=images)
-329
View File
@@ -1,329 +0,0 @@
import logging
import re
import time
from typing import Dict, Optional
import markdownify
import requests
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class StdMinerUParser(BaseParser):
"""
Standard MinerU Parser for document parsing.
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
with support for tables, formulas, and images extraction.
"""
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
**kwargs,
):
"""
Initialize MinerU parser.
Args:
enable_markdownify: Whether to convert HTML tables to markdown format
mineru_endpoint: MinerU API endpoint URL
**kwargs: Additional arguments passed to BaseParser
"""
super().__init__(**kwargs)
# Get MinerU endpoint from environment variable or parameter
# Modified: 优先使用传入的参数,否则使用 Config
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
self.minerU = base_url.rstrip("/") if base_url else ""
self.enable_markdownify = enable_markdownify
# Helper for processing markdown images
self.image_helper = MarkdownImageUtil()
# Pattern to match base64 encoded images
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
# Check if MinerU API is available
self.enable = self.ping()
def ping(self, timeout: int = 5) -> bool:
"""
Check if MinerU API is available.
Args:
timeout: Request timeout in seconds
Returns:
True if API is available, False otherwise
"""
try:
response = requests.get(
self.minerU + "/docs", timeout=timeout, allow_redirects=True
)
response.raise_for_status()
return True
except Exception:
return False
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content into text using MinerU API.
Args:
content: Raw document content in bytes
Returns:
Document object containing parsed text and images
"""
if not self.enable:
logger.debug("MinerU API is not enabled")
return Document()
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
md_content: str = ""
images_b64: Dict[str, str] = {}
try:
# Call MinerU API to parse document
response = requests.post(
url=self.minerU + "/file_parse",
data={
"return_md": True, # Return markdown content
"return_images": True, # Return extracted images
"lang_list": ["ch", "en"], # Support Chinese and English
"table_enable": True, # Enable table parsing
"formula_enable": True, # Enable formula parsing
"parse_method": "auto", # Auto detect parsing method
"start_page_id": 0, # Start from first page
"end_page_id": 99999, # Parse all pages
"backend": "pipeline", # Use pipeline backend
"response_format_zip": False, # Return JSON instead of ZIP
"return_middle_json": False, # Don't return intermediate JSON
"return_model_output": False, # Don't return model output
"return_content_list": False, # Don't return content list
},
files={"files": content},
timeout=1000,
)
response.raise_for_status()
result = response.json()["results"]["files"]
md_content = result["md_content"]
images_b64 = result.get("images", {})
except Exception as e:
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
return Document()
# Convert HTML tables in markdown to markdown table format
if self.enable_markdownify:
logger.debug("Converting HTML to Markdown")
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
# Filter images that are actually used in markdown content
# Some images in images_b64 may not be referenced in md_content
# (e.g., images embedded in tables), so we need to filter them
for ipath, b64_str in images_b64.items():
# Skip images that are not referenced in markdown content
if f"images/{ipath}" not in md_content:
logger.debug(f"Image {ipath} not used in markdown")
continue
# Parse base64 image data
match = self.base64_pattern.match(b64_str)
if match:
# Extract image format (e.g., png, jpg)
file_ext = match.group(1)
# Extract base64 encoded data
b64_str = match.group(2)
# Decode base64 string to bytes
image_bytes = endecode.encode_image(b64_str, errors="ignore")
if not image_bytes:
logger.error("Failed to decode base64 image skip it")
continue
# Upload image to storage and get URL
image_url = self.storage.upload_bytes(
image_bytes, file_ext=f".{file_ext}"
)
# Store image mapping for later use
images[image_url] = b64_str
# Prepare replacement mapping for markdown content
image_replace[f"images/{ipath}"] = image_url
logger.info(f"Replaced {len(image_replace)} images in markdown")
# Replace image paths in markdown with uploaded URLs
text = self.image_helper.replace_path(md_content, image_replace)
logger.info(
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
)
return Document(content=text, images=images)
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
class MinerUCloudParser(StdMinerUParser):
"""
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
Uses the /submit -> /status -> /result workflow.
"""
SUBMIT_TIMEOUT = 30
POLL_INTERVAL = 2
MAX_WAIT_TIME = 600
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content using Cloud MinerU API (Async/Polling).
"""
if not self.enable:
return Document()
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
try:
# --- Step 1: Submit Task ---
submit_url = f"{self.minerU}/submit"
logger.info(f"Submitting task to {submit_url}")
response = requests.post(
url=submit_url,
files={"files": content},
data={
"enable_formula": "true",
"enable_table": "true",
"layout_model": "doclayout_yolo",
"backend": "pipeline",
},
timeout=self.SUBMIT_TIMEOUT,
)
response.raise_for_status()
# Robust task_id extraction
resp_data = response.json()
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
if not task_id:
raise ValueError(f"No task_id in response: {resp_data}")
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
# --- Step 2: Poll Status ---
start_time = time.time()
while True:
if time.time() - start_time > self.MAX_WAIT_TIME:
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
try:
status_resp = requests.get(
f"{self.minerU}/status/{task_id}",
timeout=10
)
status_resp.raise_for_status()
status_data = status_resp.json()
except requests.RequestException as e:
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
time.sleep(self.POLL_INTERVAL)
continue
state = status_data.get("status") or status_data.get("state")
if state in ["done", "success"]:
break
elif state == "failed":
error_msg = status_data.get("error") or "Unknown error"
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
else:
time.sleep(self.POLL_INTERVAL)
# --- Step 3: Get Result ---
result_resp = requests.get(
f"{self.minerU}/result/{task_id}",
timeout=30
)
result_resp.raise_for_status()
result_json = result_resp.json()
# Normalize result data
result_data = result_json.get("result", result_json)
md_content = result_data.get("md_content", "")
images_b64 = result_data.get("images", {})
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
# Convert HTML tables
if self.enable_markdownify:
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
for ipath, b64_str in images_b64.items():
if f"images/{ipath}" not in md_content:
continue
match = self.base64_pattern.match(b64_str)
if match:
file_ext = match.group(1)
b64_str_clean = match.group(2)
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
if not image_bytes: continue
if self.storage:
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
images[image_url] = b64_str_clean
image_replace[f"images/{ipath}"] = image_url
if image_replace:
md_content = self.image_helper.replace_path(md_content, image_replace)
return Document(content=md_content, images=images)
except Exception as e:
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
return Document()
class MinerUParser(PipelineParser):
"""
MinerU Parser with pipeline processing.
This parser combines StdMinerUParser for document parsing and
MarkdownTableFormatter for table formatting in a pipeline.
"""
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
if __name__ == "__main__":
import os
# Example usage for testing
logging.basicConfig(level=logging.DEBUG)
# Configure your file path and MinerU endpoint
your_file = "/path/to/your/file.pdf"
# Added: 修改为 Localhost 方便测试
test_endpoint = "http://localhost:9987"
os.environ["MINERU_ENDPOINT"] = test_endpoint
# Create parser instance
# Modified: 传入 endpoint
parser = MinerUParser(mineru_endpoint=test_endpoint)
# Parse PDF file
if os.path.exists(your_file):
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
logger.error(document.content)
else:
print(f"File not found: {your_file}")
+42 -138
View File
@@ -1,178 +1,82 @@
import logging
from typing import Dict, Type
from typing import Any, Optional
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
from docreader.parser.csv_parser import CSVParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.pdf_parser import PDFParser
from docreader.parser.text_parser import TextParser
from docreader.parser.registry import registry
from docreader.parser.web_parser import WebParser
logger = logging.getLogger(__name__)
class Parser:
"""
Document parser facade that integrates all specialized parsers.
Provides a unified interface for parsing various document types.
"""Document parser facade (lightweight version).
Converts files/URLs to markdown + image references.
No chunking, no storage, no OCR, no VLM.
"""
def __init__(self):
# Initialize all parser types - maps file extensions to their corresponding parser classes
self.parsers: Dict[str, Type[BaseParser]] = {
# Document formats
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"txt": TextParser,
# Image formats - all use the same ImageParser
"jpg": ImageParser,
"jpeg": ImageParser,
"png": ImageParser,
"gif": ImageParser,
"bmp": ImageParser,
"tiff": ImageParser,
"webp": ImageParser,
# Alternative markdown extension
"markdown": MarkdownParser,
# Spreadsheet formats
"csv": CSVParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
}
self.registry = registry
logger.info(
"Parser initialized with %d parsers: %s",
len(self.parsers),
", ".join(self.parsers.keys()),
"Parser initialized with engines: %s",
", ".join(self.registry.get_engine_names()),
)
def get_parser(self, file_type: str) -> Type[BaseParser]:
"""
Get parser class for the specified file type.
Args:
file_type: The file extension or type identifier
Returns:
Parser class for the file type, or None if unsupported
"""
# Look up parser by file type (case-insensitive)
parser = self.parsers.get(file_type.lower())
if not parser:
# Raise error if file type is not supported
raise ValueError(f"Unsupported file type: {file_type}")
return parser
def parse_file(
self,
file_name: str,
file_type: str,
content: bytes,
config: ChunkingConfig,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""
Parse file content using appropriate parser based on file type.
Args:
file_name: Name of the file being parsed
file_type: Type/extension of the file
content: Raw file content as bytes
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing file: {file_name} with type: {file_type}")
"""Parse file content to markdown."""
engine = parser_engine or ""
overrides = engine_overrides or {}
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
"Parsing file: %s, type: %s, engine: %s",
file_name,
file_type,
engine or "builtin",
)
# Get appropriate parser class for the file type
cls = self.get_parser(file_type)
# Create parser instance with configuration
logger.info(f"Creating parser instance for {file_type} file")
cls = self.registry.get_parser_class(engine, file_type)
logger.info(
"Creating %s parser instance for %s file",
cls.__name__,
file_type,
)
parser = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
**overrides,
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
# Execute the parsing process
logger.info("Starting to parse file content, size: %d bytes", len(content))
result = parser.parse(content)
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for file: {file_name}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for file: %s", file_name)
logger.info(
"Parsed file %s, content length=%d", file_name, len(result.content)
)
return result
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
"""
Parse content from a URL using the WebParser.
Args:
url: URL to parse
title: Title of the webpage (for metadata)
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing URL: {url}, title: {title}")
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
)
# Create web parser instance with configuration
logger.info("Creating WebParser instance")
parser = WebParser(
title=title, # Webpage title for metadata
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
def parse_url(
self,
url: str,
title: str,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""Parse content from a URL to markdown."""
logger.info("Parsing URL: %s, title: %s", url, title)
parser = WebParser(title=title)
logger.info("Starting to parse URL content")
# Parse URL content (encode URL string to bytes as required by parser interface)
result = parser.parse(url.encode())
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for url: {url}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for url: {url}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for url: %s", url)
logger.info("Parsed url %s, content length=%d", url, len(result.content))
return result
+1 -2
View File
@@ -1,6 +1,5 @@
from docreader.parser.chain_parser import FirstParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.mineru_parser import MinerUParser
class PDFParser(FirstParser):
@@ -13,4 +12,4 @@ class PDFParser(FirstParser):
The first successful parser result will be returned.
"""
# Parser classes to try in order (chain of responsibility pattern)
_parser_cls = (MinerUParser, MarkitdownParser)
_parser_cls = (MarkitdownParser,)
+160
View File
@@ -0,0 +1,160 @@
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
from docreader.parser.base_parser import BaseParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.pdf_parser import PDFParser
logger = logging.getLogger(__name__)
BUILTIN_ENGINE = "builtin"
class ParserEngineRegistry:
"""Registry for parser engines.
Each engine maps file extensions to parser classes.
When a requested engine doesn't support a file type, the registry
falls back to the builtin engine automatically.
"""
def __init__(self):
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
self._descriptions: Dict[str, str] = {}
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
self._unavailable_hint: Dict[str, str] = {}
def register(
self,
name: str,
file_types: Dict[str, Type[BaseParser]],
description: str = "",
check_available: Callable[..., Tuple[bool, str]] | None = None,
unavailable_hint: str = "",
):
self._engines[name] = file_types
self._descriptions[name] = description
if check_available is not None:
self._check_available[name] = check_available
self._unavailable_hint[name] = unavailable_hint
logger.info(
"Registered parser engine '%s' with file types: %s",
name,
", ".join(file_types.keys()),
)
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
"""Resolve parser class for the given engine and file type.
Falls back to builtin engine when the requested engine doesn't
support the file type.
"""
ft = file_type.lower()
if engine and engine in self._engines:
cls = self._engines[engine].get(ft)
if cls:
logger.info("Using engine '%s' for file type '%s'", engine, ft)
return cls
logger.info(
"Engine '%s' does not support '%s', falling back to builtin",
engine,
ft,
)
builtin = self._engines.get(BUILTIN_ENGINE, {})
cls = builtin.get(ft)
if cls:
return cls
raise ValueError(f"Unsupported file type: {file_type}")
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
"""Return metadata for all registered engines, including availability.
Args:
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
forwarded to each engine's check_available function.
"""
result = []
for name, parsers in self._engines.items():
available = True
unavailable_reason = ""
check = self._check_available.get(name)
if check is not None:
try:
available, unavailable_reason = check(overrides)
except Exception as e:
available = False
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
if not available and not unavailable_reason:
unavailable_reason = self._unavailable_hint.get(name, "不可用")
result.append(
{
"name": name,
"description": self._descriptions.get(name, ""),
"file_types": sorted(parsers.keys()),
"available": available,
"unavailable_reason": unavailable_reason,
}
)
return result
def get_engine_names(self) -> List[str]:
return list(self._engines.keys())
def _build_default_registry() -> ParserEngineRegistry:
"""Create and populate the default registry with all known engines."""
reg = ParserEngineRegistry()
_image_types = {
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
}
reg.register(
BUILTIN_ENGINE,
{
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"markdown": MarkdownParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
**_image_types,
},
description="内置解析引擎",
)
reg.register(
"markitdown",
{
"md": MarkitdownParser,
"markdown": MarkitdownParser,
"pdf": MarkitdownParser,
"docx": MarkitdownParser,
"doc": MarkitdownParser,
"pptx": MarkitdownParser,
"ppt": MarkitdownParser,
"xlsx": MarkitdownParser,
"xls": MarkitdownParser,
"csv": MarkitdownParser,
},
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
)
# NOTE: Engine listing is managed by Go-side engine registry
# (docparser.ListAllEngines). The Python list_engines method is kept for
# backward compatibility with the gRPC ListEngines RPC but the Go app
# no longer calls it. MinerU engines are handled natively by Go.
return reg
registry = _build_default_registry()
+98 -222
View File
@@ -5,96 +5,70 @@ import os
import traceback
import uuid
from abc import ABC, abstractmethod
from typing import Dict
from typing import Dict, Optional
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.config import CONFIG
from docreader.utils import endecode
logger = logging.getLogger(__name__)
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
"""Read a value from storage_config dict, falling back to env vars."""
if storage_config:
v = storage_config.get(key, "")
if v:
return str(v)
for ek in env_keys:
v = os.environ.get(ek, "")
if v:
return v
return default
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = (
self._init_cos_client()
)
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available,
# otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
secret_key = cos_config.get("secret_access_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = CONFIG.cos_secret_id
secret_key = CONFIG.cos_secret_key
region = CONFIG.cos_region
bucket_name = CONFIG.cos_bucket_name
appid = CONFIG.cos_app_id
prefix = CONFIG.cos_path_prefix
enable_old_domain = CONFIG.cos_enable_old_domain
sc = self.storage_config
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
region = _cfg(sc, "region", "COS_REGION")
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
appid = _cfg(sc, "app_id", "COS_APP_ID")
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, "
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
"Incomplete COS configuration: "
"secret_id=%s, region=%s, bucket=%s, appid=%s",
bool(secret_id), region, bucket_name, appid,
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
config = CosConfig(
Appid=appid,
Region=region,
@@ -102,93 +76,50 @@ class CosStorage(Storage):
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
logger.error("Failed to initialize COS client: %s", e)
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_ext = os.path.splitext(file_path)[1]
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
self.client.upload_file(
Bucket=self.bucket_name,
LocalFilePath=file_path,
Key=object_key,
)
# Get file URL
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
logger.info("COS upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
logger.error("COS upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated object key: {object_key}")
self.client.put_object(
Bucket=self.bucket_name, Body=content, Key=object_key
)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
logger.info("COS upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
logger.error("COS upload_bytes failed: %s", e)
traceback.print_exc()
return ""
@@ -196,67 +127,34 @@ class CosStorage(Storage):
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
self._init_minio_client()
)
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
If storage_config contains valid configuration, prefer those values
to override environment variables.
"""
try:
# Get configuration from storage_config with environment variables as fallback
# Each field can independently fall back to environment variables
access_key = (
self.storage_config.get("access_key_id")
if self.storage_config and self.storage_config.get("access_key_id")
else CONFIG.minio_access_key_id
)
secret_key = (
self.storage_config.get("secret_access_key")
if self.storage_config and self.storage_config.get("secret_access_key")
else CONFIG.minio_secret_access_key
)
bucket_name = (
self.storage_config.get("bucket_name")
if self.storage_config and self.storage_config.get("bucket_name")
else CONFIG.minio_bucket_name
)
path_prefix_raw = (
self.storage_config.get("path_prefix")
if self.storage_config and self.storage_config.get("path_prefix")
else CONFIG.minio_path_prefix
)
sc = self.storage_config
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
endpoint = CONFIG.minio_endpoint
use_ssl = CONFIG.minio_use_ssl
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error(
"Incomplete MinIO configuration, missing environment variables"
)
logger.error("Incomplete MinIO configuration")
return None, None, None, None, None
# Initialize client
client = Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
# Set public read policy for the bucket
policy = (
"{"
'"Version":"2012-10-17",'
@@ -273,51 +171,26 @@ class MinioStorage(Storage):
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
logger.error("Failed to initialize MinIO client: %s", e)
return None, None, None, None, None
def _get_download_url(self, object_key: str):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
# 1. Use public endpoint if provided
endpoint = CONFIG.minio_public_endpoint
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
# 2. Use SSL if enabled
if self.use_ssl:
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
# 3. Use HTTP default
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
if public_endpoint:
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
scheme = "https" if self.use_ssl else "http"
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to MinIO: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, "rb") as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
@@ -327,38 +200,22 @@ class MinioStorage(Storage):
length=file_size,
content_type="application/octet-stream",
)
# Get file URL
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
logger.info("MinIO upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
logger.error("MinIO upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name or "",
object_key,
@@ -367,52 +224,67 @@ class MinioStorage(Storage):
content_type="application/octet-stream",
)
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
logger.info("MinIO upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
logger.error("MinIO upload_bytes failed: %s", e)
traceback.print_exc()
return ""
class LocalStorage(Storage):
"""Local file system storage implementation"""
"""Local file system storage implementation.
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
self.image_dir = os.path.join(base_dir, "images")
Saves files under base_dir and returns web-accessible URL paths
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
"""
def __init__(self, storage_config: Optional[Dict] = None):
sc = storage_config or {}
self.base_dir = (
sc.get("base_dir")
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
)
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
if path_prefix:
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
else:
self.image_dir = os.path.join(self.base_dir, "images")
self.url_prefix = (
sc.get("url_prefix")
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
)
os.makedirs(self.image_dir, exist_ok=True)
def _to_url(self, fpath: str) -> str:
if self.url_prefix:
rel = os.path.relpath(fpath, self.base_dir)
return f"{self.url_prefix}/{rel}"
return fpath
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to local storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to local storage: {len(content)} bytes")
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fname, "wb") as f:
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fpath, "wb") as f:
f.write(content)
return fname
url = self._to_url(fpath)
logger.info("Local storage saved: %s -> %s", fpath, url)
return url
class Base64Storage(Storage):
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to base64 storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
file_ext = file_ext.lstrip(".")
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
class DummyStorage(Storage):
"""Dummy storage implementation.
It is used in tests or in environments where object storage is disabled.
All upload methods return empty string.
"""
"""Dummy storage — all uploads return empty string."""
def upload_file(self, file_path: str) -> str:
return ""
@@ -421,26 +293,30 @@ class DummyStorage(Storage):
return ""
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
"""Create a storage instance based on storage_config dict.
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
The ``provider`` key in storage_config determines the backend:
minio, cos, local, base64.
Falls back to STORAGE_TYPE env var, then ``local``.
"""
storage_type = CONFIG.storage_type
storage_type = ""
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
provider = str(storage_config.get("provider", "")).lower().strip()
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
storage_type = provider
if not storage_type:
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
logger.info("Creating %s storage instance", storage_type)
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
return CosStorage(storage_config)
elif storage_type == "local":
return LocalStorage(storage_config or {})
return LocalStorage(storage_config)
elif storage_type == "base64":
return Base64Storage()
return DummyStorage()
-55
View File
@@ -1,55 +0,0 @@
import logging
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class TextParser(BaseParser):
"""
Text document parser for processing plain text files.
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse text document content by decoding bytes to string.
This is a straightforward parser that simply converts the binary content
to text using appropriate character encoding.
Args:
content: Raw document content as bytes
Returns:
Parsed text content as string
"""
logger.info(f"Parsing text document, content size: {len(content)} bytes")
text = endecode.decode_bytes(content)
logger.info(
f"Successfully parsed text document, extracted {len(text)} characters"
)
return Document(content=text)
if __name__ == "__main__":
logger = logging.getLogger(__name__)
# Sample text for testing
text = """## 标题1
![alt text](image.png)
## 标题2
![alt text](image2.png)
## 标题3
![alt text](image3.png)"""
logger.info(f"Test text content: {text}")
# Define separators for text splitting
seperators = ["\n\n", "\n", "。"]
parser = TextParser(separators=seperators)
logger.info("Splitting text into units")
units = parser._split_into_units(text)
logger.info(f"Split text into {len(units)} units")
logger.info(f"Units: {units}")