diff --git a/docreader/splitter/splitter.py b/docreader/splitter/splitter.py index cac6ea2a..d0532707 100644 --- a/docreader/splitter/splitter.py +++ b/docreader/splitter/splitter.py @@ -20,8 +20,11 @@ from docreader.splitter.header_hook import ( from docreader.utils.split import split_by_char, split_by_sep # Default configuration for text chunking -DEFAULT_CHUNK_OVERLAP = 100 # Number of tokens to overlap between chunks -DEFAULT_CHUNK_SIZE = 512 # Maximum size of each chunk in tokens +# Aligned with internal/infrastructure/chunker/splitter.go (DefaultChunkOverlap = 80, +# DefaultChunkSize = 512). The Go splitter is now the production path; this +# Python splitter is kept for the docreader sidecar where it's still used. +DEFAULT_CHUNK_OVERLAP = 80 # Number of characters to overlap between chunks (~15% of chunk size) +DEFAULT_CHUNK_SIZE = 512 # Maximum size of each chunk in characters T = TypeVar("T")