feat: Make OCR and task concurrency configurable

This commit is contained in:
begoniezhao
2026-01-15 10:42:48 +08:00
parent 928d1c3d44
commit 1abdaa5d5c
5 changed files with 49 additions and 35 deletions
+5 -2
View File
@@ -1,4 +1,5 @@
import logging
import os
from typing import Dict, Type
from docreader.models.document import Document
@@ -109,8 +110,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
@@ -155,8 +157,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info("Starting to parse URL content")