From 0fe5d67b6a53393d0bee104a4b9e18af8975d939 Mon Sep 17 00:00:00 2001 From: ChenRussell <792608290@qq.com> Date: Fri, 13 Mar 2026 20:24:45 +0800 Subject: [PATCH] fix: enforce 7500 char limit in chunker to prevent embedding API errors Prevents 400 Bad Request errors when uploading documents with large tables or code blocks by adding hard limits in buildUnitsWithProtection and mergeUnits functions. --- internal/infrastructure/chunker/splitter.go | 98 +++++++++++++++++++-- internal/models/embedding/openai.go | 37 +++++++- 2 files changed, 128 insertions(+), 7 deletions(-) diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 37dff63d..fefae05e 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -167,7 +167,11 @@ func SplitText(text string, cfg SplitterConfig) []Chunk { // buildUnitsWithProtection splits text into units, preserving protected spans as atomic. // Start/End positions in the returned units are rune offsets (not byte offsets), // because downstream merge logic indexes content via []rune slicing. +// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent +// creating chunks that are too large for downstream processing (e.g., embedding APIs). func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit { + const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等) + var units []splitUnit bytePos := 0 runePos := 0 @@ -191,11 +195,43 @@ func buildUnitsWithProtection(text string, protected []span, separators []string protText := text[p.start:p.end] protRuneLen := runeLen(protText) - units = append(units, splitUnit{ - text: protText, - start: runePos, - end: runePos + protRuneLen, - }) + + // If protected content is too large, forcibly split it + if protRuneLen > maxProtectedSize { + // Split into smaller chunks at line breaks or spaces + runes := []rune(protText) + offset := 0 + for offset < len(runes) { + chunkEnd := offset + maxProtectedSize + if chunkEnd > len(runes) { + chunkEnd = len(runes) + } else { + // Try to break at a newline or space + for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { + if runes[i] == '\n' || runes[i] == ' ' { + chunkEnd = i + 1 + break + } + } + } + + chunkText := string(runes[offset:chunkEnd]) + chunkLen := chunkEnd - offset + units = append(units, splitUnit{ + text: chunkText, + start: runePos + offset, + end: runePos + offset + chunkLen, + }) + offset = chunkEnd + } + } else { + // Normal case: keep protected content as a single unit + units = append(units, splitUnit{ + text: protText, + start: runePos, + end: runePos + protRuneLen, + }) + } runePos += protRuneLen bytePos = p.end } @@ -219,11 +255,15 @@ func buildUnitsWithProtection(text string, protected []span, separators []string } // mergeUnits combines split units into chunks with overlap tracking. +// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs). func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if len(units) == 0 { return nil } + // Absolute maximum chunk size (留余量给标题等额外内容) + const absoluteMaxSize = 7500 + var chunks []Chunk var current []splitUnit curLen := 0 @@ -231,6 +271,44 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { for _, u := range units { uLen := runeLen(u.text) + // If this single unit exceeds absolute max, force split it further + if uLen > absoluteMaxSize { + // Flush current chunk if any + if len(current) > 0 { + chunks = append(chunks, buildChunk(current, len(chunks))) + current = nil + curLen = 0 + } + + // Split this oversized unit into smaller chunks + runes := []rune(u.text) + offset := 0 + for offset < len(runes) { + chunkEnd := offset + absoluteMaxSize + if chunkEnd > len(runes) { + chunkEnd = len(runes) + } else { + // Try to break at a newline or space + for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { + if runes[i] == '\n' || runes[i] == ' ' { + chunkEnd = i + 1 + break + } + } + } + + chunkText := string(runes[offset:chunkEnd]) + chunks = append(chunks, Chunk{ + Content: chunkText, + Seq: len(chunks), + Start: u.start + offset, + End: u.start + chunkEnd, + }) + offset = chunkEnd + } + continue + } + // If adding this unit exceeds chunk size and we have content, flush if curLen+uLen > chunkSize && len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) @@ -239,6 +317,16 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen) } + // Check if adding this unit would exceed absolute max + if curLen+uLen > absoluteMaxSize { + // Flush current and start fresh + if len(current) > 0 { + chunks = append(chunks, buildChunk(current, len(chunks))) + current = nil + curLen = 0 + } + } + current = append(current, u) curLen += uLen } diff --git a/internal/models/embedding/openai.go b/internal/models/embedding/openai.go index 3c13fdb8..7e495e13 100644 --- a/internal/models/embedding/openai.go +++ b/internal/models/embedding/openai.go @@ -149,6 +149,34 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl return nil, fmt.Errorf("marshal request: %w", err) } + // Log request details for debugging + logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed: model=%s, input_count=%d, truncate_tokens=%d", + e.modelName, len(texts), e.truncatePromptTokens) + + // Check for invalid input lengths and log details + hasInvalidLength := false + for i, text := range texts { + textLen := len(text) + textPreview := text + if len(textPreview) > 200 { + textPreview = textPreview[:200] + "..." + } + + // Log warning if length is outside valid range [1, 8192] + if textLen == 0 || textLen > 8192 { + hasInvalidLength = true + logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed input[%d]: INVALID length=%d (must be [1, 8192]), preview=%s", + i, textLen, textPreview) + } else { + logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed input[%d]: length=%d, preview=%s", + i, textLen, textPreview) + } + } + + if hasInvalidLength { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed: Found invalid input lengths, this will likely cause API error") + } + // Send request (passing jsonData instead of constructing http.Request) resp, err := e.doRequestWithRetry(ctx, jsonData) if err != nil { @@ -167,8 +195,13 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl } if resp.StatusCode != http.StatusOK { - logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status) - return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status) + // Log detailed error response from OpenAI API + bodyStr := string(body) + if len(bodyStr) > 1000 { + bodyStr = bodyStr[:1000] + "... (truncated)" + } + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s, Response Body: %s", resp.Status, bodyStr) + return nil, fmt.Errorf("EmbedBatch API error: Http Status %s, Response: %s", resp.Status, bodyStr) } // Parse response