feat: Make OCR and task concurrency configurable
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Type
|
||||
|
||||
from docreader.models.document import Document
|
||||
@@ -109,8 +110,9 @@ class Parser:
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
@@ -155,8 +157,9 @@ class Parser:
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
)
|
||||
|
||||
logger.info("Starting to parse URL content")
|
||||
|
||||
Reference in New Issue
Block a user