From 2241f9579d25a2ff4b0a249bd0a5ce1216ee1b51 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 29 May 2026 16:55:53 +0800 Subject: [PATCH] refactor(keywords_vector_hybrid_indexer): enhance image payload sanitization logic - Added a check to skip regex processing for non-image content, improving performance when no inline base64 payload is present. - Updated regex pattern to ensure it correctly matches base64 image data URIs. --- .../retriever/keywords_vector_hybrid_indexer.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index 7df44c3f..3e62ceb8 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -4,6 +4,7 @@ import ( "context" "regexp" "slices" + "strings" "time" "unicode/utf8" @@ -33,7 +34,7 @@ var embeddingImagePayloadPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?is)]*\bsrc=["']\s*data:image/[a-z0-9.+-]+;base64,[^"']+["'][^>]*>`), regexp.MustCompile(`(?is)!\[[^\]]*\]\(\s*data:image/[a-z0-9.+-]+;base64,[^)]+\)`), regexp.MustCompile(`(?i)data:image/[a-z0-9.+-]+;base64,[a-z0-9+/=]{200,}`), - regexp.MustCompile(`(?i)base64,[a-z0-9+/=]{200,}`), + regexp.MustCompile(`(?i)data:[a-z0-9.+/-]+;base64,[a-z0-9+/=]{200,}`), } // KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine @@ -157,8 +158,12 @@ func batchEmbedWithBackoff(ctx context.Context, embedder embedding.Embedder, con // realistic token limit. We log a warning whenever truncation kicks in. func sanitizeForEmbedding(ctx context.Context, content string) string { sanitized := content - for _, pattern := range embeddingImagePayloadPatterns { - sanitized = pattern.ReplaceAllString(sanitized, "[image]") + // Scrubbing only matters when an inline base64 payload is present; skip the + // regex passes otherwise so the common (no-image) path stays cheap. + if strings.Contains(content, "base64,") { + for _, pattern := range embeddingImagePayloadPatterns { + sanitized = pattern.ReplaceAllString(sanitized, "[image]") + } } if utf8.RuneCountInString(sanitized) <= safetyMaxChars {