From 6b3ad07543b14949bf0ed55c332d76f54fb4ce84 Mon Sep 17 00:00:00 2001 From: wolfkill <27876473+wolfkill@users.noreply.github.com> Date: Fri, 29 May 2026 10:59:31 +0800 Subject: [PATCH] fix embedding input image payload safety --- .../keywords_vector_hybrid_indexer.go | 21 +++- .../keywords_vector_hybrid_indexer_test.go | 118 ++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index a1b51de5..7df44c3f 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -2,6 +2,7 @@ package retriever import ( "context" + "regexp" "slices" "time" "unicode/utf8" @@ -28,6 +29,13 @@ const ( embedRetryBaseDelay = 200 * time.Millisecond ) +var embeddingImagePayloadPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?is)]*\bsrc=["']\s*data:image/[a-z0-9.+-]+;base64,[^"']+["'][^>]*>`), + regexp.MustCompile(`(?is)!\[[^\]]*\]\(\s*data:image/[a-z0-9.+-]+;base64,[^)]+\)`), + regexp.MustCompile(`(?i)data:image/[a-z0-9.+-]+;base64,[a-z0-9+/=]{200,}`), + regexp.MustCompile(`(?i)base64,[a-z0-9+/=]{200,}`), +} + // KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine // that supports both keyword-based and vector-based retrieval type KeywordsVectorHybridRetrieveEngineService struct { @@ -63,7 +71,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context, params := make(map[string]any) embeddingMap := make(map[string][]float32) if slices.Contains(retrieverTypes, types.VectorRetrieverType) { - embedding, err := embedder.Embed(ctx, indexInfo.Content) + embedding, err := embedder.Embed(ctx, sanitizeForEmbedding(ctx, indexInfo.Content)) if err != nil { return err } @@ -148,10 +156,15 @@ func batchEmbedWithBackoff(ctx context.Context, embedder embedding.Embedder, con // truncation point is char-based, not token-based, so it sits well above any // realistic token limit. We log a warning whenever truncation kicks in. func sanitizeForEmbedding(ctx context.Context, content string) string { - if utf8.RuneCountInString(content) <= safetyMaxChars { - return content + sanitized := content + for _, pattern := range embeddingImagePayloadPatterns { + sanitized = pattern.ReplaceAllString(sanitized, "[image]") } - runes := []rune(content) + + if utf8.RuneCountInString(sanitized) <= safetyMaxChars { + return sanitized + } + runes := []rune(sanitized) logger.Warnf(ctx, "embedding input truncated: %d runes -> %d", len(runes), safetyMaxChars) return string(runes[:safetyMaxChars]) } diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go new file mode 100644 index 00000000..3d374909 --- /dev/null +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go @@ -0,0 +1,118 @@ +package retriever + +import ( + "context" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +type capturingEmbedder struct { + embedding.Embedder + text string + batchTexts []string +} + +func (e *capturingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + e.text = text + return []float32{1}, nil +} + +func (e *capturingEmbedder) BatchEmbedWithPool( + ctx context.Context, + model embedding.Embedder, + texts []string, +) ([][]float32, error) { + e.batchTexts = append([]string(nil), texts...) + embeddings := make([][]float32, len(texts)) + for i := range texts { + embeddings[i] = []float32{1} + } + return embeddings, nil +} + +type saveOnlyRepository struct { + interfaces.RetrieveEngineRepository +} + +func (r *saveOnlyRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error { + return nil +} + +func (r *saveOnlyRepository) BatchSave( + ctx context.Context, + indexInfoList []*types.IndexInfo, + params map[string]any, +) error { + return nil +} + +func TestIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) { + ctx := context.Background() + embedder := &capturingEmbedder{} + service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}} + payload := strings.Repeat("A", 300) + content := "before after" + + err := service.Index(ctx, embedder, &types.IndexInfo{ + Content: content, + SourceID: "source-1", + }, []types.RetrieverType{types.VectorRetrieverType}) + if err != nil { + t.Fatalf("Index returned error: %v", err) + } + assertImagePayloadRemoved(t, embedder.text, payload) +} + +func TestBatchIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) { + ctx := context.Background() + embedder := &capturingEmbedder{} + service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}} + payload := strings.Repeat("A", 300) + content := "before ![chart](data:image/png;base64," + payload + ") after" + + err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{ + Content: content, + SourceID: "source-1", + }}, []types.RetrieverType{types.VectorRetrieverType}) + if err != nil { + t.Fatalf("BatchIndex returned error: %v", err) + } + if len(embedder.batchTexts) != 1 { + t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts)) + } + assertImagePayloadRemoved(t, embedder.batchTexts[0], payload) +} + +func TestBatchIndexTruncatesOversizedEmbeddingInput(t *testing.T) { + ctx := context.Background() + embedder := &capturingEmbedder{} + service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}} + + err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{ + Content: strings.Repeat("x", safetyMaxChars+10), + SourceID: "source-1", + }}, []types.RetrieverType{types.VectorRetrieverType}) + if err != nil { + t.Fatalf("BatchIndex returned error: %v", err) + } + if len(embedder.batchTexts) != 1 { + t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts)) + } + if got := len([]rune(embedder.batchTexts[0])); got > safetyMaxChars { + t.Fatalf("embedding input length = %d, want <= %d", got, safetyMaxChars) + } +} + +func assertImagePayloadRemoved(t *testing.T, content string, payload string) { + t.Helper() + if strings.Contains(content, "data:image/png;base64") || strings.Contains(content, payload) { + t.Fatalf("embedding input still contains inline image payload: %q", content) + } + if !strings.Contains(content, "before") || !strings.Contains(content, "after") { + t.Fatalf("embedding input should preserve surrounding text, got %q", content) + } +}