fix: enforce 7500 char limit in chunker to prevent embedding API errors

Prevents 400 Bad Request errors when uploading documents with large
tables or code blocks by adding hard limits in buildUnitsWithProtection
and mergeUnits functions.
This commit is contained in:
ChenRussell
2026-03-13 20:24:45 +08:00
committed by lyingbug
parent 166f106b7f
commit 0fe5d67b6a
2 changed files with 128 additions and 7 deletions
+93 -5
View File
@@ -167,7 +167,11 @@ func SplitText(text string, cfg SplitterConfig) []Chunk {
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
// Start/End positions in the returned units are rune offsets (not byte offsets),
// because downstream merge logic indexes content via []rune slicing.
// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent
// creating chunks that are too large for downstream processing (e.g., embedding APIs).
func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit {
const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等)
var units []splitUnit
bytePos := 0
runePos := 0
@@ -191,11 +195,43 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
protText := text[p.start:p.end]
protRuneLen := runeLen(protText)
units = append(units, splitUnit{
text: protText,
start: runePos,
end: runePos + protRuneLen,
})
// If protected content is too large, forcibly split it
if protRuneLen > maxProtectedSize {
// Split into smaller chunks at line breaks or spaces
runes := []rune(protText)
offset := 0
for offset < len(runes) {
chunkEnd := offset + maxProtectedSize
if chunkEnd > len(runes) {
chunkEnd = len(runes)
} else {
// Try to break at a newline or space
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
if runes[i] == '\n' || runes[i] == ' ' {
chunkEnd = i + 1
break
}
}
}
chunkText := string(runes[offset:chunkEnd])
chunkLen := chunkEnd - offset
units = append(units, splitUnit{
text: chunkText,
start: runePos + offset,
end: runePos + offset + chunkLen,
})
offset = chunkEnd
}
} else {
// Normal case: keep protected content as a single unit
units = append(units, splitUnit{
text: protText,
start: runePos,
end: runePos + protRuneLen,
})
}
runePos += protRuneLen
bytePos = p.end
}
@@ -219,11 +255,15 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
}
// mergeUnits combines split units into chunks with overlap tracking.
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
if len(units) == 0 {
return nil
}
// Absolute maximum chunk size (留余量给标题等额外内容)
const absoluteMaxSize = 7500
var chunks []Chunk
var current []splitUnit
curLen := 0
@@ -231,6 +271,44 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
for _, u := range units {
uLen := runeLen(u.text)
// If this single unit exceeds absolute max, force split it further
if uLen > absoluteMaxSize {
// Flush current chunk if any
if len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
current = nil
curLen = 0
}
// Split this oversized unit into smaller chunks
runes := []rune(u.text)
offset := 0
for offset < len(runes) {
chunkEnd := offset + absoluteMaxSize
if chunkEnd > len(runes) {
chunkEnd = len(runes)
} else {
// Try to break at a newline or space
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
if runes[i] == '\n' || runes[i] == ' ' {
chunkEnd = i + 1
break
}
}
}
chunkText := string(runes[offset:chunkEnd])
chunks = append(chunks, Chunk{
Content: chunkText,
Seq: len(chunks),
Start: u.start + offset,
End: u.start + chunkEnd,
})
offset = chunkEnd
}
continue
}
// If adding this unit exceeds chunk size and we have content, flush
if curLen+uLen > chunkSize && len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
@@ -239,6 +317,16 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
}
// Check if adding this unit would exceed absolute max
if curLen+uLen > absoluteMaxSize {
// Flush current and start fresh
if len(current) > 0 {
chunks = append(chunks, buildChunk(current, len(chunks)))
current = nil
curLen = 0
}
}
current = append(current, u)
curLen += uLen
}
+35 -2
View File
@@ -149,6 +149,34 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
return nil, fmt.Errorf("marshal request: %w", err)
}
// Log request details for debugging
logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed: model=%s, input_count=%d, truncate_tokens=%d",
e.modelName, len(texts), e.truncatePromptTokens)
// Check for invalid input lengths and log details
hasInvalidLength := false
for i, text := range texts {
textLen := len(text)
textPreview := text
if len(textPreview) > 200 {
textPreview = textPreview[:200] + "..."
}
// Log warning if length is outside valid range [1, 8192]
if textLen == 0 || textLen > 8192 {
hasInvalidLength = true
logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed input[%d]: INVALID length=%d (must be [1, 8192]), preview=%s",
i, textLen, textPreview)
} else {
logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed input[%d]: length=%d, preview=%s",
i, textLen, textPreview)
}
}
if hasInvalidLength {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed: Found invalid input lengths, this will likely cause API error")
}
// Send request (passing jsonData instead of constructing http.Request)
resp, err := e.doRequestWithRetry(ctx, jsonData)
if err != nil {
@@ -167,8 +195,13 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status)
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
// Log detailed error response from OpenAI API
bodyStr := string(body)
if len(bodyStr) > 1000 {
bodyStr = bodyStr[:1000] + "... (truncated)"
}
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s, Response Body: %s", resp.Status, bodyStr)
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s, Response: %s", resp.Status, bodyStr)
}
// Parse response