chore(docreader): align Python splitter overlap default with Go
The Go production splitter now uses DefaultChunkOverlap = 80 (~15% of DefaultChunkSize). Bringing the Python sidecar splitter to the same number prevents subtle behavior drift if the docreader path ever falls back to it for chunking. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -20,8 +20,11 @@ from docreader.splitter.header_hook import (
|
||||
from docreader.utils.split import split_by_char, split_by_sep
|
||||
|
||||
# Default configuration for text chunking
|
||||
DEFAULT_CHUNK_OVERLAP = 100 # Number of tokens to overlap between chunks
|
||||
DEFAULT_CHUNK_SIZE = 512 # Maximum size of each chunk in tokens
|
||||
# Aligned with internal/infrastructure/chunker/splitter.go (DefaultChunkOverlap = 80,
|
||||
# DefaultChunkSize = 512). The Go splitter is now the production path; this
|
||||
# Python splitter is kept for the docreader sidecar where it's still used.
|
||||
DEFAULT_CHUNK_OVERLAP = 80 # Number of characters to overlap between chunks (~15% of chunk size)
|
||||
DEFAULT_CHUNK_SIZE = 512 # Maximum size of each chunk in characters
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user