Merge remote-tracking branch 'github_public/main'
This commit is contained in:
@@ -35,34 +35,37 @@ class BaseParser(ABC):
|
||||
@staticmethod
|
||||
def _is_safe_url(url: str) -> bool:
|
||||
"""Validate URL to prevent SSRF attacks
|
||||
|
||||
|
||||
Args:
|
||||
url: URL to validate
|
||||
|
||||
|
||||
Returns:
|
||||
True if URL is safe to fetch, False otherwise
|
||||
"""
|
||||
try:
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
|
||||
# Only allow http and https schemes
|
||||
if parsed_url.scheme not in ['http', 'https']:
|
||||
if parsed_url.scheme not in ["http", "https"]:
|
||||
logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}")
|
||||
return False
|
||||
|
||||
|
||||
# Extract hostname
|
||||
hostname = parsed_url.hostname
|
||||
if not hostname:
|
||||
logger.warning("No hostname found in URL")
|
||||
return False
|
||||
|
||||
# Try to parse as IP address
|
||||
try:
|
||||
ip = ipaddress.ip_address(hostname)
|
||||
# Reject private, loopback, link-local, multicast addresses
|
||||
if (ip.is_private or ip.is_loopback or
|
||||
ip.is_link_local or ip.is_multicast or
|
||||
ip.is_reserved):
|
||||
if (
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_multicast
|
||||
or ip.is_reserved
|
||||
):
|
||||
logger.warning(f"Rejected URL with restricted IP: {ip}")
|
||||
return False
|
||||
except ValueError:
|
||||
@@ -70,17 +73,19 @@ class BaseParser(ABC):
|
||||
# Reject localhost and common internal hostnames
|
||||
hostname_lower = hostname.lower()
|
||||
restricted_hostnames = [
|
||||
'localhost',
|
||||
'127.0.0.1',
|
||||
'::1',
|
||||
'metadata.google.internal',
|
||||
'metadata.tencentyun.com',
|
||||
'169.254.169.254', # AWS metadata endpoint
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"::1",
|
||||
"metadata.google.internal",
|
||||
"metadata.tencentyun.com",
|
||||
"169.254.169.254", # AWS metadata endpoint
|
||||
]
|
||||
if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'):
|
||||
if hostname_lower in restricted_hostnames or hostname_lower.endswith(
|
||||
".local"
|
||||
):
|
||||
logger.warning(f"Rejected URL with restricted hostname: {hostname}")
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Error validating URL: {str(e)}")
|
||||
@@ -121,7 +126,7 @@ class BaseParser(ABC):
|
||||
chunk_size: int = 1000,
|
||||
chunk_overlap: int = 200,
|
||||
separators: list[str] = ["\n\n", "\n", "。"],
|
||||
ocr_backend: str = "paddle",
|
||||
ocr_backend: str = "no_ocr",
|
||||
ocr_config: dict = {},
|
||||
max_image_size: int = 1920, # Maximum image size
|
||||
max_concurrent_tasks: int = 5, # Max concurrent tasks
|
||||
@@ -151,7 +156,7 @@ class BaseParser(ABC):
|
||||
self.chunk_size = chunk_size
|
||||
self.chunk_overlap = chunk_overlap
|
||||
self.separators = separators
|
||||
self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend)
|
||||
self.ocr_backend = ocr_backend
|
||||
self.ocr_config = ocr_config
|
||||
self.max_image_size = max_image_size
|
||||
self.max_concurrent_tasks = max_concurrent_tasks
|
||||
@@ -161,13 +166,14 @@ class BaseParser(ABC):
|
||||
self.chunking_config.storage_config if self.chunking_config else None
|
||||
)
|
||||
|
||||
logger.info(f"Initializing parser for file: {file_name}, type: {file_type}")
|
||||
logger.info(
|
||||
f"Parser config: chunk_size={chunk_size}, "
|
||||
f"Initializing parser for file={file_name}, type={file_type}, "
|
||||
f"parser config: chunk_size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, "
|
||||
f"multimodal={enable_multimodal}, "
|
||||
f"max_chunks={max_chunks}, "
|
||||
f"ocr_backend={ocr_backend}, "
|
||||
f"max_chunks={max_chunks}"
|
||||
f"max_concurrent_tasks={max_concurrent_tasks}"
|
||||
)
|
||||
# Only initialize Caption service if multimodal is enabled
|
||||
vlm_config = self.chunking_config.vlm_config if self.chunking_config else None
|
||||
@@ -305,13 +311,8 @@ class BaseParser(ABC):
|
||||
logger.warning("No image data to process")
|
||||
return []
|
||||
|
||||
# Set max concurrency, reduce concurrency to avoid resource contention
|
||||
max_concurrency = min(
|
||||
self.max_concurrent_tasks, 1
|
||||
) # Reduce concurrency to prevent excessive memory usage
|
||||
|
||||
# Use semaphore to limit concurrency
|
||||
semaphore = asyncio.Semaphore(max_concurrency)
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
|
||||
|
||||
# Store results to avoid overall failure due to task failure
|
||||
results = []
|
||||
@@ -741,7 +742,6 @@ class BaseParser(ABC):
|
||||
if not self._is_safe_url(img_url):
|
||||
logger.error(f"URL failed validation check: {img_url}")
|
||||
return img_url, img_url, None
|
||||
|
||||
# Still need to get image object for OCR processing
|
||||
# Get proxy settings from environment variables
|
||||
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
|
||||
@@ -792,7 +792,6 @@ class BaseParser(ABC):
|
||||
if not self._is_safe_url(img_url):
|
||||
logger.error(f"URL failed validation check: {img_url}")
|
||||
return img_url, img_url, None
|
||||
|
||||
# Get proxy settings from environment variables
|
||||
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
|
||||
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
|
||||
@@ -952,10 +951,8 @@ class BaseParser(ABC):
|
||||
|
||||
# Create and run all Chunk concurrent processing tasks
|
||||
async def process_all_chunks():
|
||||
# Set max concurrency, reduce concurrency to avoid resource contention
|
||||
max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency
|
||||
# Use semaphore to limit concurrency
|
||||
semaphore = asyncio.Semaphore(max_concurrency)
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
|
||||
|
||||
async def process_with_limit(chunk, idx, total):
|
||||
"""Use semaphore to control concurrent processing of Chunks"""
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Type
|
||||
|
||||
from docreader.models.document import Document
|
||||
@@ -109,8 +110,9 @@ class Parser:
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
@@ -155,8 +157,9 @@ class Parser:
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
)
|
||||
|
||||
logger.info("Starting to parse URL content")
|
||||
|
||||
@@ -75,9 +75,9 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
# link - Markdown link syntax [text](url)
|
||||
r"\[.*?\]\(.*?\)",
|
||||
# table header - Markdown table header with separator line
|
||||
r"(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
|
||||
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
|
||||
# table body - Markdown table rows
|
||||
r"(?:\|[^|\n]*)+\|[\r\n]+",
|
||||
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+",
|
||||
# code header - Code block start with language identifier
|
||||
r"```(?:\w+)[\r\n]+[^\r\n]*",
|
||||
],
|
||||
@@ -137,6 +137,10 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
|
||||
# Step 4: Merge splits into final chunks with overlap
|
||||
chunks = self._merge(splits)
|
||||
|
||||
# Step 5: Validate chunks and test restoration
|
||||
# self._validate_chunks(chunks, text)
|
||||
|
||||
return chunks
|
||||
|
||||
def _split(self, text: str) -> List[str]:
|
||||
@@ -250,10 +254,17 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
cur_len > self.chunk_overlap
|
||||
or cur_len + split_len + cur_headers_len > self.chunk_size
|
||||
):
|
||||
# Remove the first element to reduce overlap
|
||||
# Remove the first element to reduce overlap.
|
||||
# If the first element is a prepended header (start==end), also remove it.
|
||||
first_chunk = cur_chunk.pop(0)
|
||||
cur_len -= self.len_function(first_chunk[2])
|
||||
|
||||
# If we just popped a real content piece, there may be a header right after it
|
||||
# (depending on previous iterations). Pop it only if it is actually a header.
|
||||
if cur_chunk and first_chunk[0] == first_chunk[1]:
|
||||
first_chunk = cur_chunk.pop(0)
|
||||
cur_len -= self.len_function(first_chunk[2])
|
||||
|
||||
# Prepend headers to new chunk if:
|
||||
# 1. Headers exist
|
||||
# 2. Headers + split fit in chunk size
|
||||
@@ -264,10 +275,8 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
and cur_headers not in split
|
||||
):
|
||||
next_start = cur_chunk[0][0] if cur_chunk else cur_start
|
||||
header_start = max(0, next_start - cur_headers_len)
|
||||
header_end = cur_end
|
||||
|
||||
cur_chunk.insert(0, (header_start, header_end, cur_headers))
|
||||
cur_chunk.insert(0, (next_start, next_start, cur_headers))
|
||||
cur_len += cur_headers_len
|
||||
|
||||
# Add current split to the chunk
|
||||
@@ -397,6 +406,154 @@ class TextSplitter(BaseModel, Generic[T]):
|
||||
start = end
|
||||
return res
|
||||
|
||||
def _validate_chunks(
|
||||
self, chunks: List[Tuple[int, int, str]], original_text: str
|
||||
) -> None:
|
||||
"""Validate chunks order and test text restoration.
|
||||
|
||||
This method performs two validations:
|
||||
1. Checks if chunk start positions are in ascending order
|
||||
2. Tests if the original text can be restored from chunks
|
||||
|
||||
If validation fails, saves debug information to /tmp/chunk_error_<timestamp>.md
|
||||
|
||||
Args:
|
||||
chunks: List of tuples (start_pos, end_pos, chunk_text) to validate
|
||||
original_text: The original text that was split
|
||||
"""
|
||||
import datetime
|
||||
|
||||
errors = []
|
||||
|
||||
# Validation 1: Check if start positions are in ascending order
|
||||
for i in range(1, len(chunks)):
|
||||
prev_start = chunks[i - 1][0]
|
||||
curr_start = chunks[i][0]
|
||||
if curr_start < prev_start:
|
||||
error_msg = (
|
||||
f"Chunk order error: chunk[{i}] start position ({curr_start}) "
|
||||
f"is less than chunk[{i - 1}] start position ({prev_start})"
|
||||
)
|
||||
errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
# Validation 2: Test text restoration
|
||||
try:
|
||||
restored_text = self.restore_text(chunks)
|
||||
if restored_text != original_text:
|
||||
error_msg = (
|
||||
f"Restoration failed: restored text differs from original. "
|
||||
f"Original length: {len(original_text)}, "
|
||||
f"Restored length: {len(restored_text)}"
|
||||
)
|
||||
errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
# Find first difference position
|
||||
min_len = min(len(original_text), len(restored_text))
|
||||
diff_pos = -1
|
||||
for i in range(min_len):
|
||||
if original_text[i] != restored_text[i]:
|
||||
diff_pos = i
|
||||
break
|
||||
|
||||
if diff_pos >= 0:
|
||||
context_start = max(0, diff_pos - 50)
|
||||
context_end = min(len(original_text), diff_pos + 50)
|
||||
errors.append(
|
||||
f"First difference at position {diff_pos}:\n"
|
||||
f"Original: {repr(original_text[context_start:context_end])}\n"
|
||||
f"Restored: {repr(restored_text[context_start:context_end])}"
|
||||
)
|
||||
elif len(original_text) != len(restored_text):
|
||||
errors.append(
|
||||
f"Texts match up to position {min_len}, but lengths differ"
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = f"Restoration exception: {str(e)}"
|
||||
errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
# If there are errors, save debug information to file
|
||||
if errors:
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
error_file = f"/tmp/chunk_error_{timestamp}.md"
|
||||
|
||||
with open(error_file, "w", encoding="utf-8") as f:
|
||||
f.write("# Chunk Validation Error Report\n\n")
|
||||
f.write(f"Timestamp: {timestamp}\n\n")
|
||||
|
||||
f.write("## Errors\n\n")
|
||||
for error in errors:
|
||||
f.write(f"- {error}\n\n")
|
||||
|
||||
f.write("\n## Original Text\n\n")
|
||||
f.write(f"Length: {len(original_text)}\n\n")
|
||||
f.write("```\n")
|
||||
f.write(original_text)
|
||||
f.write("\n```\n\n")
|
||||
|
||||
f.write("\n## Chunks Information\n\n")
|
||||
f.write(f"Total chunks: {len(chunks)}\n\n")
|
||||
for i, (start, end, chunk_text) in enumerate(chunks):
|
||||
f.write(f"### Chunk {i}\n\n")
|
||||
f.write(f"- Position: [{start}:{end}]\n")
|
||||
f.write(f"- Length: {len(chunk_text)}\n")
|
||||
f.write(f"- Content:\n\n```\n{chunk_text}\n```\n\n")
|
||||
|
||||
try:
|
||||
restored_text = self.restore_text(chunks)
|
||||
f.write("\n## Restored Text\n\n")
|
||||
f.write(f"Length: {len(restored_text)}\n\n")
|
||||
f.write("```\n")
|
||||
f.write(restored_text)
|
||||
f.write("\n```\n")
|
||||
except Exception as e:
|
||||
f.write("\n## Restoration Failed\n\n")
|
||||
f.write(f"Error: {str(e)}\n")
|
||||
|
||||
logger.error(f"Validation errors saved to: {error_file}")
|
||||
|
||||
def restore_text(self, chunks: List[Tuple[int, int, str]]) -> str:
|
||||
"""Restore original text from chunks with overlap handling.
|
||||
|
||||
This method reconstructs the original text from chunks that may contain:
|
||||
- Overlapping content between consecutive chunks
|
||||
- Prepended headers that were added during merging (headers have start==end position)
|
||||
|
||||
The algorithm:
|
||||
1. Sort chunks by their start position (and end position as tiebreaker)
|
||||
2. Track the maximum end position seen so far
|
||||
3. For each chunk, extract only the new content (after max_end_pos)
|
||||
4. Concatenate all new content pieces
|
||||
|
||||
Args:
|
||||
chunks: List of tuples (start_pos, end_pos, chunk_text) from split_text()
|
||||
|
||||
Returns:
|
||||
The restored original text
|
||||
|
||||
Example:
|
||||
>>> splitter = TextSplitter(chunk_size=10, chunk_overlap=3)
|
||||
>>> chunks = splitter.split_text("Hello World!")
|
||||
>>> restored = splitter.restore_text(chunks)
|
||||
>>> assert restored == "Hello World!"
|
||||
"""
|
||||
if not chunks:
|
||||
return ""
|
||||
|
||||
# Sort chunks by start position, then by end position
|
||||
sorted_chunks = sorted(chunks, key=lambda x: (x[1], x[0]))
|
||||
|
||||
result_parts = []
|
||||
last_end = 0
|
||||
|
||||
for start_pos, end_pos, chunk_text in sorted_chunks:
|
||||
result_parts.append(chunk_text[last_end - end_pos :])
|
||||
last_end = end_pos
|
||||
|
||||
return "".join(result_parts)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
s = """
|
||||
@@ -415,7 +572,11 @@ if __name__ == "__main__":
|
||||
|
||||
"""
|
||||
|
||||
sp = TextSplitter(chunk_size=200, chunk_overlap=2)
|
||||
sp = TextSplitter(
|
||||
chunk_size=200,
|
||||
chunk_overlap=10,
|
||||
separators=["\n\n", "\n", "。", "?", "!", ",", ";", ":"],
|
||||
)
|
||||
ck = sp.split_text(s)
|
||||
for c in ck:
|
||||
print("------", len(c))
|
||||
|
||||
Reference in New Issue
Block a user