fix embedding input image payload safety
This commit is contained in:
@@ -2,6 +2,7 @@ package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"slices"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
@@ -28,6 +29,13 @@ const (
|
||||
embedRetryBaseDelay = 200 * time.Millisecond
|
||||
)
|
||||
|
||||
var embeddingImagePayloadPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?is)<img\b[^>]*\bsrc=["']\s*data:image/[a-z0-9.+-]+;base64,[^"']+["'][^>]*>`),
|
||||
regexp.MustCompile(`(?is)!\[[^\]]*\]\(\s*data:image/[a-z0-9.+-]+;base64,[^)]+\)`),
|
||||
regexp.MustCompile(`(?i)data:image/[a-z0-9.+-]+;base64,[a-z0-9+/=]{200,}`),
|
||||
regexp.MustCompile(`(?i)base64,[a-z0-9+/=]{200,}`),
|
||||
}
|
||||
|
||||
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
|
||||
// that supports both keyword-based and vector-based retrieval
|
||||
type KeywordsVectorHybridRetrieveEngineService struct {
|
||||
@@ -63,7 +71,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
|
||||
params := make(map[string]any)
|
||||
embeddingMap := make(map[string][]float32)
|
||||
if slices.Contains(retrieverTypes, types.VectorRetrieverType) {
|
||||
embedding, err := embedder.Embed(ctx, indexInfo.Content)
|
||||
embedding, err := embedder.Embed(ctx, sanitizeForEmbedding(ctx, indexInfo.Content))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -148,10 +156,15 @@ func batchEmbedWithBackoff(ctx context.Context, embedder embedding.Embedder, con
|
||||
// truncation point is char-based, not token-based, so it sits well above any
|
||||
// realistic token limit. We log a warning whenever truncation kicks in.
|
||||
func sanitizeForEmbedding(ctx context.Context, content string) string {
|
||||
if utf8.RuneCountInString(content) <= safetyMaxChars {
|
||||
return content
|
||||
sanitized := content
|
||||
for _, pattern := range embeddingImagePayloadPatterns {
|
||||
sanitized = pattern.ReplaceAllString(sanitized, "[image]")
|
||||
}
|
||||
runes := []rune(content)
|
||||
|
||||
if utf8.RuneCountInString(sanitized) <= safetyMaxChars {
|
||||
return sanitized
|
||||
}
|
||||
runes := []rune(sanitized)
|
||||
logger.Warnf(ctx, "embedding input truncated: %d runes -> %d", len(runes), safetyMaxChars)
|
||||
return string(runes[:safetyMaxChars])
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
type capturingEmbedder struct {
|
||||
embedding.Embedder
|
||||
text string
|
||||
batchTexts []string
|
||||
}
|
||||
|
||||
func (e *capturingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
e.text = text
|
||||
return []float32{1}, nil
|
||||
}
|
||||
|
||||
func (e *capturingEmbedder) BatchEmbedWithPool(
|
||||
ctx context.Context,
|
||||
model embedding.Embedder,
|
||||
texts []string,
|
||||
) ([][]float32, error) {
|
||||
e.batchTexts = append([]string(nil), texts...)
|
||||
embeddings := make([][]float32, len(texts))
|
||||
for i := range texts {
|
||||
embeddings[i] = []float32{1}
|
||||
}
|
||||
return embeddings, nil
|
||||
}
|
||||
|
||||
type saveOnlyRepository struct {
|
||||
interfaces.RetrieveEngineRepository
|
||||
}
|
||||
|
||||
func (r *saveOnlyRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *saveOnlyRepository) BatchSave(
|
||||
ctx context.Context,
|
||||
indexInfoList []*types.IndexInfo,
|
||||
params map[string]any,
|
||||
) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
embedder := &capturingEmbedder{}
|
||||
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
|
||||
payload := strings.Repeat("A", 300)
|
||||
content := "before <img src=\"data:image/png;base64," + payload + "\"> after"
|
||||
|
||||
err := service.Index(ctx, embedder, &types.IndexInfo{
|
||||
Content: content,
|
||||
SourceID: "source-1",
|
||||
}, []types.RetrieverType{types.VectorRetrieverType})
|
||||
if err != nil {
|
||||
t.Fatalf("Index returned error: %v", err)
|
||||
}
|
||||
assertImagePayloadRemoved(t, embedder.text, payload)
|
||||
}
|
||||
|
||||
func TestBatchIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
embedder := &capturingEmbedder{}
|
||||
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
|
||||
payload := strings.Repeat("A", 300)
|
||||
content := "before  after"
|
||||
|
||||
err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
|
||||
Content: content,
|
||||
SourceID: "source-1",
|
||||
}}, []types.RetrieverType{types.VectorRetrieverType})
|
||||
if err != nil {
|
||||
t.Fatalf("BatchIndex returned error: %v", err)
|
||||
}
|
||||
if len(embedder.batchTexts) != 1 {
|
||||
t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
|
||||
}
|
||||
assertImagePayloadRemoved(t, embedder.batchTexts[0], payload)
|
||||
}
|
||||
|
||||
func TestBatchIndexTruncatesOversizedEmbeddingInput(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
embedder := &capturingEmbedder{}
|
||||
service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
|
||||
|
||||
err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
|
||||
Content: strings.Repeat("x", safetyMaxChars+10),
|
||||
SourceID: "source-1",
|
||||
}}, []types.RetrieverType{types.VectorRetrieverType})
|
||||
if err != nil {
|
||||
t.Fatalf("BatchIndex returned error: %v", err)
|
||||
}
|
||||
if len(embedder.batchTexts) != 1 {
|
||||
t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
|
||||
}
|
||||
if got := len([]rune(embedder.batchTexts[0])); got > safetyMaxChars {
|
||||
t.Fatalf("embedding input length = %d, want <= %d", got, safetyMaxChars)
|
||||
}
|
||||
}
|
||||
|
||||
func assertImagePayloadRemoved(t *testing.T, content string, payload string) {
|
||||
t.Helper()
|
||||
if strings.Contains(content, "data:image/png;base64") || strings.Contains(content, payload) {
|
||||
t.Fatalf("embedding input still contains inline image payload: %q", content)
|
||||
}
|
||||
if !strings.Contains(content, "before") || !strings.Contains(content, "after") {
|
||||
t.Fatalf("embedding input should preserve surrounding text, got %q", content)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user