diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 37dff63d..fefae05e 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -167,7 +167,11 @@ func SplitText(text string, cfg SplitterConfig) []Chunk { // buildUnitsWithProtection splits text into units, preserving protected spans as atomic. // Start/End positions in the returned units are rune offsets (not byte offsets), // because downstream merge logic indexes content via []rune slicing. +// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent +// creating chunks that are too large for downstream processing (e.g., embedding APIs). func buildUnitsWithProtection(text string, protected []span, separators []string) []splitUnit { + const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等) + var units []splitUnit bytePos := 0 runePos := 0 @@ -191,11 +195,43 @@ func buildUnitsWithProtection(text string, protected []span, separators []string protText := text[p.start:p.end] protRuneLen := runeLen(protText) - units = append(units, splitUnit{ - text: protText, - start: runePos, - end: runePos + protRuneLen, - }) + + // If protected content is too large, forcibly split it + if protRuneLen > maxProtectedSize { + // Split into smaller chunks at line breaks or spaces + runes := []rune(protText) + offset := 0 + for offset < len(runes) { + chunkEnd := offset + maxProtectedSize + if chunkEnd > len(runes) { + chunkEnd = len(runes) + } else { + // Try to break at a newline or space + for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { + if runes[i] == '\n' || runes[i] == ' ' { + chunkEnd = i + 1 + break + } + } + } + + chunkText := string(runes[offset:chunkEnd]) + chunkLen := chunkEnd - offset + units = append(units, splitUnit{ + text: chunkText, + start: runePos + offset, + end: runePos + offset + chunkLen, + }) + offset = chunkEnd + } + } else { + // Normal case: keep protected content as a single unit + units = append(units, splitUnit{ + text: protText, + start: runePos, + end: runePos + protRuneLen, + }) + } runePos += protRuneLen bytePos = p.end } @@ -219,11 +255,15 @@ func buildUnitsWithProtection(text string, protected []span, separators []string } // mergeUnits combines split units into chunks with overlap tracking. +// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs). func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { if len(units) == 0 { return nil } + // Absolute maximum chunk size (留余量给标题等额外内容) + const absoluteMaxSize = 7500 + var chunks []Chunk var current []splitUnit curLen := 0 @@ -231,6 +271,44 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { for _, u := range units { uLen := runeLen(u.text) + // If this single unit exceeds absolute max, force split it further + if uLen > absoluteMaxSize { + // Flush current chunk if any + if len(current) > 0 { + chunks = append(chunks, buildChunk(current, len(chunks))) + current = nil + curLen = 0 + } + + // Split this oversized unit into smaller chunks + runes := []rune(u.text) + offset := 0 + for offset < len(runes) { + chunkEnd := offset + absoluteMaxSize + if chunkEnd > len(runes) { + chunkEnd = len(runes) + } else { + // Try to break at a newline or space + for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- { + if runes[i] == '\n' || runes[i] == ' ' { + chunkEnd = i + 1 + break + } + } + } + + chunkText := string(runes[offset:chunkEnd]) + chunks = append(chunks, Chunk{ + Content: chunkText, + Seq: len(chunks), + Start: u.start + offset, + End: u.start + chunkEnd, + }) + offset = chunkEnd + } + continue + } + // If adding this unit exceeds chunk size and we have content, flush if curLen+uLen > chunkSize && len(current) > 0 { chunks = append(chunks, buildChunk(current, len(chunks))) @@ -239,6 +317,16 @@ func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk { current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen) } + // Check if adding this unit would exceed absolute max + if curLen+uLen > absoluteMaxSize { + // Flush current and start fresh + if len(current) > 0 { + chunks = append(chunks, buildChunk(current, len(chunks))) + current = nil + curLen = 0 + } + } + current = append(current, u) curLen += uLen } diff --git a/internal/models/embedding/openai.go b/internal/models/embedding/openai.go index 3c13fdb8..7e495e13 100644 --- a/internal/models/embedding/openai.go +++ b/internal/models/embedding/openai.go @@ -149,6 +149,34 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl return nil, fmt.Errorf("marshal request: %w", err) } + // Log request details for debugging + logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed: model=%s, input_count=%d, truncate_tokens=%d", + e.modelName, len(texts), e.truncatePromptTokens) + + // Check for invalid input lengths and log details + hasInvalidLength := false + for i, text := range texts { + textLen := len(text) + textPreview := text + if len(textPreview) > 200 { + textPreview = textPreview[:200] + "..." + } + + // Log warning if length is outside valid range [1, 8192] + if textLen == 0 || textLen > 8192 { + hasInvalidLength = true + logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed input[%d]: INVALID length=%d (must be [1, 8192]), preview=%s", + i, textLen, textPreview) + } else { + logger.GetLogger(ctx).Debugf("OpenAIEmbedder BatchEmbed input[%d]: length=%d, preview=%s", + i, textLen, textPreview) + } + } + + if hasInvalidLength { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder BatchEmbed: Found invalid input lengths, this will likely cause API error") + } + // Send request (passing jsonData instead of constructing http.Request) resp, err := e.doRequestWithRetry(ctx, jsonData) if err != nil { @@ -167,8 +195,13 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl } if resp.StatusCode != http.StatusOK { - logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status) - return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status) + // Log detailed error response from OpenAI API + bodyStr := string(body) + if len(bodyStr) > 1000 { + bodyStr = bodyStr[:1000] + "... (truncated)" + } + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s, Response Body: %s", resp.Status, bodyStr) + return nil, fmt.Errorf("EmbedBatch API error: Http Status %s, Response: %s", resp.Status, bodyStr) } // Parse response