fix: enforce 7500 char limit in chunker to prevent embedding API errors
Prevents 400 Bad Request errors when uploading documents with large tables or code blocks by adding hard limits in buildUnitsWithProtection and mergeUnits functions.
This commit is contained in:
@@ -167,7 +167,11 @@ func SplitText(text string, cfg SplitterConfig) []Chunk {
|
||||
// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
|
||||
// Start/End positions in the returned units are rune offsets (not byte offsets),
|
||||
// because downstream merge logic indexes content via []rune slicing.
|
||||
// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent
|
||||
// creating chunks that are too large for downstream processing (e.g., embedding APIs).
|
||||
func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit {
|
||||
const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等)
|
||||
|
||||
var units []splitUnit
|
||||
bytePos := 0
|
||||
runePos := 0
|
||||
@@ -191,11 +195,43 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
|
||||
|
||||
protText := text[p.start:p.end]
|
||||
protRuneLen := runeLen(protText)
|
||||
units = append(units, splitUnit{
|
||||
text: protText,
|
||||
start: runePos,
|
||||
end: runePos + protRuneLen,
|
||||
})
|
||||
|
||||
// If protected content is too large, forcibly split it
|
||||
if protRuneLen > maxProtectedSize {
|
||||
// Split into smaller chunks at line breaks or spaces
|
||||
runes := []rune(protText)
|
||||
offset := 0
|
||||
for offset < len(runes) {
|
||||
chunkEnd := offset + maxProtectedSize
|
||||
if chunkEnd > len(runes) {
|
||||
chunkEnd = len(runes)
|
||||
} else {
|
||||
// Try to break at a newline or space
|
||||
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
||||
if runes[i] == '\n' || runes[i] == ' ' {
|
||||
chunkEnd = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
chunkText := string(runes[offset:chunkEnd])
|
||||
chunkLen := chunkEnd - offset
|
||||
units = append(units, splitUnit{
|
||||
text: chunkText,
|
||||
start: runePos + offset,
|
||||
end: runePos + offset + chunkLen,
|
||||
})
|
||||
offset = chunkEnd
|
||||
}
|
||||
} else {
|
||||
// Normal case: keep protected content as a single unit
|
||||
units = append(units, splitUnit{
|
||||
text: protText,
|
||||
start: runePos,
|
||||
end: runePos + protRuneLen,
|
||||
})
|
||||
}
|
||||
runePos += protRuneLen
|
||||
bytePos = p.end
|
||||
}
|
||||
@@ -219,11 +255,15 @@ func buildUnitsWithProtection(text string, protected []span, separators []string
|
||||
}
|
||||
|
||||
// mergeUnits combines split units into chunks with overlap tracking.
|
||||
// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
|
||||
func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
if len(units) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Absolute maximum chunk size (留余量给标题等额外内容)
|
||||
const absoluteMaxSize = 7500
|
||||
|
||||
var chunks []Chunk
|
||||
var current []splitUnit
|
||||
curLen := 0
|
||||
@@ -231,6 +271,44 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
for _, u := range units {
|
||||
uLen := runeLen(u.text)
|
||||
|
||||
// If this single unit exceeds absolute max, force split it further
|
||||
if uLen > absoluteMaxSize {
|
||||
// Flush current chunk if any
|
||||
if len(current) > 0 {
|
||||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||||
current = nil
|
||||
curLen = 0
|
||||
}
|
||||
|
||||
// Split this oversized unit into smaller chunks
|
||||
runes := []rune(u.text)
|
||||
offset := 0
|
||||
for offset < len(runes) {
|
||||
chunkEnd := offset + absoluteMaxSize
|
||||
if chunkEnd > len(runes) {
|
||||
chunkEnd = len(runes)
|
||||
} else {
|
||||
// Try to break at a newline or space
|
||||
for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
|
||||
if runes[i] == '\n' || runes[i] == ' ' {
|
||||
chunkEnd = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
chunkText := string(runes[offset:chunkEnd])
|
||||
chunks = append(chunks, Chunk{
|
||||
Content: chunkText,
|
||||
Seq: len(chunks),
|
||||
Start: u.start + offset,
|
||||
End: u.start + chunkEnd,
|
||||
})
|
||||
offset = chunkEnd
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding this unit exceeds chunk size and we have content, flush
|
||||
if curLen+uLen > chunkSize && len(current) > 0 {
|
||||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||||
@@ -239,6 +317,16 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
|
||||
current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
|
||||
}
|
||||
|
||||
// Check if adding this unit would exceed absolute max
|
||||
if curLen+uLen > absoluteMaxSize {
|
||||
// Flush current and start fresh
|
||||
if len(current) > 0 {
|
||||
chunks = append(chunks, buildChunk(current, len(chunks)))
|
||||
current = nil
|
||||
curLen = 0
|
||||
}
|
||||
}
|
||||
|
||||
current = append(current, u)
|
||||
curLen += uLen
|
||||
}
|
||||
|
||||
@@ -149,6 +149,34 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Log request details for debugging
|
||||
logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed: model=%s, input_count=%d, truncate_tokens=%d",
|
||||
e.modelName, len(texts), e.truncatePromptTokens)
|
||||
|
||||
// Check for invalid input lengths and log details
|
||||
hasInvalidLength := false
|
||||
for i, text := range texts {
|
||||
textLen := len(text)
|
||||
textPreview := text
|
||||
if len(textPreview) > 200 {
|
||||
textPreview = textPreview[:200] + "..."
|
||||
}
|
||||
|
||||
// Log warning if length is outside valid range [1, 8192]
|
||||
if textLen == 0 || textLen > 8192 {
|
||||
hasInvalidLength = true
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed input[%d]: INVALID length=%d (must be [1, 8192]), preview=%s",
|
||||
i, textLen, textPreview)
|
||||
} else {
|
||||
logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed input[%d]: length=%d, preview=%s",
|
||||
i, textLen, textPreview)
|
||||
}
|
||||
}
|
||||
|
||||
if hasInvalidLength {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed: Found invalid input lengths, this will likely cause API error")
|
||||
}
|
||||
|
||||
// Send request (passing jsonData instead of constructing http.Request)
|
||||
resp, err := e.doRequestWithRetry(ctx, jsonData)
|
||||
if err != nil {
|
||||
@@ -167,8 +195,13 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status)
|
||||
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
|
||||
// Log detailed error response from OpenAI API
|
||||
bodyStr := string(body)
|
||||
if len(bodyStr) > 1000 {
|
||||
bodyStr = bodyStr[:1000] + "... (truncated)"
|
||||
}
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s, Response Body: %s", resp.Status, bodyStr)
|
||||
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s, Response: %s", resp.Status, bodyStr)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
|
||||
Reference in New Issue
Block a user