fix embedding input image payload safety

This commit is contained in:
wolfkill
2026-05-29 10:59:31 +08:00
committed by lyingbug
parent 2a54bb2d45
commit 6b3ad07543
2 changed files with 135 additions and 4 deletions
@@ -2,6 +2,7 @@ package retriever
import (
"context"
"regexp"
"slices"
"time"
"unicode/utf8"
@@ -28,6 +29,13 @@ const (
embedRetryBaseDelay = 200 * time.Millisecond
)
var embeddingImagePayloadPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?is)<img\b[^>]*\bsrc=["']\s*data:image/[a-z0-9.+-]+;base64,[^"']+["'][^>]*>`),
regexp.MustCompile(`(?is)!\[[^\]]*\]\(\s*data:image/[a-z0-9.+-]+;base64,[^)]+\)`),
regexp.MustCompile(`(?i)data:image/[a-z0-9.+-]+;base64,[a-z0-9+/=]{200,}`),
regexp.MustCompile(`(?i)base64,[a-z0-9+/=]{200,}`),
}
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
// that supports both keyword-based and vector-based retrieval
type KeywordsVectorHybridRetrieveEngineService struct {
@@ -63,7 +71,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
if slices.Contains(retrieverTypes, types.VectorRetrieverType) {
embedding, err := embedder.Embed(ctx, indexInfo.Content)
embedding, err := embedder.Embed(ctx, sanitizeForEmbedding(ctx, indexInfo.Content))
if err != nil {
return err
}
@@ -148,10 +156,15 @@ func batchEmbedWithBackoff(ctx context.Context, embedder embedding.Embedder, con
// truncation point is char-based, not token-based, so it sits well above any
// realistic token limit. We log a warning whenever truncation kicks in.
func sanitizeForEmbedding(ctx context.Context, content string) string {
if utf8.RuneCountInString(content) <= safetyMaxChars {
return content
sanitized := content
for _, pattern := range embeddingImagePayloadPatterns {
sanitized = pattern.ReplaceAllString(sanitized, "[image]")
}
runes := []rune(content)
if utf8.RuneCountInString(sanitized) <= safetyMaxChars {
return sanitized
}
runes := []rune(sanitized)
logger.Warnf(ctx, "embedding input truncated: %d runes -> %d", len(runes), safetyMaxChars)
return string(runes[:safetyMaxChars])
}
@@ -0,0 +1,118 @@
package retriever
import (
"context"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
type capturingEmbedder struct {
embedding.Embedder
text string
batchTexts []string
}
func (e *capturingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
e.text = text
return []float32{1}, nil
}
func (e *capturingEmbedder) BatchEmbedWithPool(
ctx context.Context,
model embedding.Embedder,
texts []string,
) ([][]float32, error) {
e.batchTexts = append([]string(nil), texts...)
embeddings := make([][]float32, len(texts))
for i := range texts {
embeddings[i] = []float32{1}
}
return embeddings, nil
}
type saveOnlyRepository struct {
interfaces.RetrieveEngineRepository
}
func (r *saveOnlyRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
return nil
}
func (r *saveOnlyRepository) BatchSave(
ctx context.Context,
indexInfoList []*types.IndexInfo,
params map[string]any,
) error {
return nil
}
func TestIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
ctx := context.Background()
embedder := &capturingEmbedder{}
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
payload := strings.Repeat("A", 300)
content := "before <img src=\"data:image/png;base64," + payload + "\"> after"
err := service.Index(ctx, embedder, &types.IndexInfo{
Content: content,
SourceID: "source-1",
}, []types.RetrieverType{types.VectorRetrieverType})
if err != nil {
t.Fatalf("Index returned error: %v", err)
}
assertImagePayloadRemoved(t, embedder.text, payload)
}
func TestBatchIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
ctx := context.Background()
embedder := &capturingEmbedder{}
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
payload := strings.Repeat("A", 300)
content := "before ![chart](data:image/png;base64," + payload + ") after"
err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
Content: content,
SourceID: "source-1",
}}, []types.RetrieverType{types.VectorRetrieverType})
if err != nil {
t.Fatalf("BatchIndex returned error: %v", err)
}
if len(embedder.batchTexts) != 1 {
t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
}
assertImagePayloadRemoved(t, embedder.batchTexts[0], payload)
}
func TestBatchIndexTruncatesOversizedEmbeddingInput(t *testing.T) {
ctx := context.Background()
embedder := &capturingEmbedder{}
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
Content: strings.Repeat("x", safetyMaxChars+10),
SourceID: "source-1",
}}, []types.RetrieverType{types.VectorRetrieverType})
if err != nil {
t.Fatalf("BatchIndex returned error: %v", err)
}
if len(embedder.batchTexts) != 1 {
t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
}
if got := len([]rune(embedder.batchTexts[0])); got > safetyMaxChars {
t.Fatalf("embedding input length = %d, want <= %d", got, safetyMaxChars)
}
}
func assertImagePayloadRemoved(t *testing.T, content string, payload string) {
t.Helper()
if strings.Contains(content, "data:image/png;base64") || strings.Contains(content, payload) {
t.Fatalf("embedding input still contains inline image payload: %q", content)
}
if !strings.Contains(content, "before") || !strings.Contains(content, "after") {
t.Fatalf("embedding input should preserve surrounding text, got %q", content)
}
}