Merge remote-tracking branch 'github_public/main'

This commit is contained in:
wizardchen
2026-01-15 14:39:35 +08:00
17 changed files with 301 additions and 63 deletions
+30 -33
View File
@@ -35,34 +35,37 @@ class BaseParser(ABC):
@staticmethod
def _is_safe_url(url: str) -> bool:
"""Validate URL to prevent SSRF attacks
Args:
url: URL to validate
Returns:
True if URL is safe to fetch, False otherwise
"""
try:
parsed_url = urlparse(url)
# Only allow http and https schemes
if parsed_url.scheme not in ['http', 'https']:
if parsed_url.scheme not in ["http", "https"]:
logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}")
return False
# Extract hostname
hostname = parsed_url.hostname
if not hostname:
logger.warning("No hostname found in URL")
return False
# Try to parse as IP address
try:
ip = ipaddress.ip_address(hostname)
# Reject private, loopback, link-local, multicast addresses
if (ip.is_private or ip.is_loopback or
ip.is_link_local or ip.is_multicast or
ip.is_reserved):
if (
ip.is_private
or ip.is_loopback
or ip.is_link_local
or ip.is_multicast
or ip.is_reserved
):
logger.warning(f"Rejected URL with restricted IP: {ip}")
return False
except ValueError:
@@ -70,17 +73,19 @@ class BaseParser(ABC):
# Reject localhost and common internal hostnames
hostname_lower = hostname.lower()
restricted_hostnames = [
'localhost',
'127.0.0.1',
'::1',
'metadata.google.internal',
'metadata.tencentyun.com',
'169.254.169.254', # AWS metadata endpoint
"localhost",
"127.0.0.1",
"::1",
"metadata.google.internal",
"metadata.tencentyun.com",
"169.254.169.254", # AWS metadata endpoint
]
if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'):
if hostname_lower in restricted_hostnames or hostname_lower.endswith(
".local"
):
logger.warning(f"Rejected URL with restricted hostname: {hostname}")
return False
return True
except Exception as e:
logger.warning(f"Error validating URL: {str(e)}")
@@ -121,7 +126,7 @@ class BaseParser(ABC):
chunk_size: int = 1000,
chunk_overlap: int = 200,
separators: list[str] = ["\n\n", "\n", "。"],
ocr_backend: str = "paddle",
ocr_backend: str = "no_ocr",
ocr_config: dict = {},
max_image_size: int = 1920, # Maximum image size
max_concurrent_tasks: int = 5, # Max concurrent tasks
@@ -151,7 +156,7 @@ class BaseParser(ABC):
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
self.separators = separators
self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend)
self.ocr_backend = ocr_backend
self.ocr_config = ocr_config
self.max_image_size = max_image_size
self.max_concurrent_tasks = max_concurrent_tasks
@@ -161,13 +166,14 @@ class BaseParser(ABC):
self.chunking_config.storage_config if self.chunking_config else None
)
logger.info(f"Initializing parser for file: {file_name}, type: {file_type}")
logger.info(
f"Parser config: chunk_size={chunk_size}, "
f"Initializing parser for file={file_name}, type={file_type}, "
f"parser config: chunk_size={chunk_size}, "
f"overlap={chunk_overlap}, "
f"multimodal={enable_multimodal}, "
f"max_chunks={max_chunks}, "
f"ocr_backend={ocr_backend}, "
f"max_chunks={max_chunks}"
f"max_concurrent_tasks={max_concurrent_tasks}"
)
# Only initialize Caption service if multimodal is enabled
vlm_config = self.chunking_config.vlm_config if self.chunking_config else None
@@ -305,13 +311,8 @@ class BaseParser(ABC):
logger.warning("No image data to process")
return []
# Set max concurrency, reduce concurrency to avoid resource contention
max_concurrency = min(
self.max_concurrent_tasks, 1
) # Reduce concurrency to prevent excessive memory usage
# Use semaphore to limit concurrency
semaphore = asyncio.Semaphore(max_concurrency)
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
# Store results to avoid overall failure due to task failure
results = []
@@ -741,7 +742,6 @@ class BaseParser(ABC):
if not self._is_safe_url(img_url):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Still need to get image object for OCR processing
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
@@ -792,7 +792,6 @@ class BaseParser(ABC):
if not self._is_safe_url(img_url):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
@@ -952,10 +951,8 @@ class BaseParser(ABC):
# Create and run all Chunk concurrent processing tasks
async def process_all_chunks():
# Set max concurrency, reduce concurrency to avoid resource contention
max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency
# Use semaphore to limit concurrency
semaphore = asyncio.Semaphore(max_concurrency)
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
async def process_with_limit(chunk, idx, total):
"""Use semaphore to control concurrent processing of Chunks"""
+5 -2
View File
@@ -1,4 +1,5 @@
import logging
import os
from typing import Dict, Type
from docreader.models.document import Document
@@ -109,8 +110,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
@@ -155,8 +157,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info("Starting to parse URL content")
+168 -7
View File
@@ -75,9 +75,9 @@ class TextSplitter(BaseModel, Generic[T]):
# link - Markdown link syntax [text](url)
r"\[.*?\]\(.*?\)",
# table header - Markdown table header with separator line
r"(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
# table body - Markdown table rows
r"(?:\|[^|\n]*)+\|[\r\n]+",
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+",
# code header - Code block start with language identifier
r"```(?:\w+)[\r\n]+[^\r\n]*",
],
@@ -137,6 +137,10 @@ class TextSplitter(BaseModel, Generic[T]):
# Step 4: Merge splits into final chunks with overlap
chunks = self._merge(splits)
# Step 5: Validate chunks and test restoration
# self._validate_chunks(chunks, text)
return chunks
def _split(self, text: str) -> List[str]:
@@ -250,10 +254,17 @@ class TextSplitter(BaseModel, Generic[T]):
cur_len > self.chunk_overlap
or cur_len + split_len + cur_headers_len > self.chunk_size
):
# Remove the first element to reduce overlap
# Remove the first element to reduce overlap.
# If the first element is a prepended header (start==end), also remove it.
first_chunk = cur_chunk.pop(0)
cur_len -= self.len_function(first_chunk[2])
# If we just popped a real content piece, there may be a header right after it
# (depending on previous iterations). Pop it only if it is actually a header.
if cur_chunk and first_chunk[0] == first_chunk[1]:
first_chunk = cur_chunk.pop(0)
cur_len -= self.len_function(first_chunk[2])
# Prepend headers to new chunk if:
# 1. Headers exist
# 2. Headers + split fit in chunk size
@@ -264,10 +275,8 @@ class TextSplitter(BaseModel, Generic[T]):
and cur_headers not in split
):
next_start = cur_chunk[0][0] if cur_chunk else cur_start
header_start = max(0, next_start - cur_headers_len)
header_end = cur_end
cur_chunk.insert(0, (header_start, header_end, cur_headers))
cur_chunk.insert(0, (next_start, next_start, cur_headers))
cur_len += cur_headers_len
# Add current split to the chunk
@@ -397,6 +406,154 @@ class TextSplitter(BaseModel, Generic[T]):
start = end
return res
def _validate_chunks(
self, chunks: List[Tuple[int, int, str]], original_text: str
) -> None:
"""Validate chunks order and test text restoration.
This method performs two validations:
1. Checks if chunk start positions are in ascending order
2. Tests if the original text can be restored from chunks
If validation fails, saves debug information to /tmp/chunk_error_<timestamp>.md
Args:
chunks: List of tuples (start_pos, end_pos, chunk_text) to validate
original_text: The original text that was split
"""
import datetime
errors = []
# Validation 1: Check if start positions are in ascending order
for i in range(1, len(chunks)):
prev_start = chunks[i - 1][0]
curr_start = chunks[i][0]
if curr_start < prev_start:
error_msg = (
f"Chunk order error: chunk[{i}] start position ({curr_start}) "
f"is less than chunk[{i - 1}] start position ({prev_start})"
)
errors.append(error_msg)
logger.error(error_msg)
# Validation 2: Test text restoration
try:
restored_text = self.restore_text(chunks)
if restored_text != original_text:
error_msg = (
f"Restoration failed: restored text differs from original. "
f"Original length: {len(original_text)}, "
f"Restored length: {len(restored_text)}"
)
errors.append(error_msg)
logger.error(error_msg)
# Find first difference position
min_len = min(len(original_text), len(restored_text))
diff_pos = -1
for i in range(min_len):
if original_text[i] != restored_text[i]:
diff_pos = i
break
if diff_pos >= 0:
context_start = max(0, diff_pos - 50)
context_end = min(len(original_text), diff_pos + 50)
errors.append(
f"First difference at position {diff_pos}:\n"
f"Original: {repr(original_text[context_start:context_end])}\n"
f"Restored: {repr(restored_text[context_start:context_end])}"
)
elif len(original_text) != len(restored_text):
errors.append(
f"Texts match up to position {min_len}, but lengths differ"
)
except Exception as e:
error_msg = f"Restoration exception: {str(e)}"
errors.append(error_msg)
logger.error(error_msg)
# If there are errors, save debug information to file
if errors:
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
error_file = f"/tmp/chunk_error_{timestamp}.md"
with open(error_file, "w", encoding="utf-8") as f:
f.write("# Chunk Validation Error Report\n\n")
f.write(f"Timestamp: {timestamp}\n\n")
f.write("## Errors\n\n")
for error in errors:
f.write(f"- {error}\n\n")
f.write("\n## Original Text\n\n")
f.write(f"Length: {len(original_text)}\n\n")
f.write("```\n")
f.write(original_text)
f.write("\n```\n\n")
f.write("\n## Chunks Information\n\n")
f.write(f"Total chunks: {len(chunks)}\n\n")
for i, (start, end, chunk_text) in enumerate(chunks):
f.write(f"### Chunk {i}\n\n")
f.write(f"- Position: [{start}:{end}]\n")
f.write(f"- Length: {len(chunk_text)}\n")
f.write(f"- Content:\n\n```\n{chunk_text}\n```\n\n")
try:
restored_text = self.restore_text(chunks)
f.write("\n## Restored Text\n\n")
f.write(f"Length: {len(restored_text)}\n\n")
f.write("```\n")
f.write(restored_text)
f.write("\n```\n")
except Exception as e:
f.write("\n## Restoration Failed\n\n")
f.write(f"Error: {str(e)}\n")
logger.error(f"Validation errors saved to: {error_file}")
def restore_text(self, chunks: List[Tuple[int, int, str]]) -> str:
"""Restore original text from chunks with overlap handling.
This method reconstructs the original text from chunks that may contain:
- Overlapping content between consecutive chunks
- Prepended headers that were added during merging (headers have start==end position)
The algorithm:
1. Sort chunks by their start position (and end position as tiebreaker)
2. Track the maximum end position seen so far
3. For each chunk, extract only the new content (after max_end_pos)
4. Concatenate all new content pieces
Args:
chunks: List of tuples (start_pos, end_pos, chunk_text) from split_text()
Returns:
The restored original text
Example:
>>> splitter = TextSplitter(chunk_size=10, chunk_overlap=3)
>>> chunks = splitter.split_text("Hello World!")
>>> restored = splitter.restore_text(chunks)
>>> assert restored == "Hello World!"
"""
if not chunks:
return ""
# Sort chunks by start position, then by end position
sorted_chunks = sorted(chunks, key=lambda x: (x[1], x[0]))
result_parts = []
last_end = 0
for start_pos, end_pos, chunk_text in sorted_chunks:
result_parts.append(chunk_text[last_end - end_pos :])
last_end = end_pos
return "".join(result_parts)
if __name__ == "__main__":
s = """
@@ -415,7 +572,11 @@ if __name__ == "__main__":
"""
sp = TextSplitter(chunk_size=200, chunk_overlap=2)
sp = TextSplitter(
chunk_size=200,
chunk_overlap=10,
separators=["\n\n", "\n", "。", "?", "!", ",", ";", ":"],
)
ck = sp.split_text(s)
for c in ck:
print("------", len(c))