diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go
index a1b51de5..7df44c3f 100644
--- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go
+++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go
@@ -2,6 +2,7 @@ package retriever
import (
"context"
+ "regexp"
"slices"
"time"
"unicode/utf8"
@@ -28,6 +29,13 @@ const (
embedRetryBaseDelay = 200 * time.Millisecond
)
+var embeddingImagePayloadPatterns = []*regexp.Regexp{
+ regexp.MustCompile(`(?is)
]*\bsrc=["']\s*data:image/[a-z0-9.+-]+;base64,[^"']+["'][^>]*>`),
+ regexp.MustCompile(`(?is)!\[[^\]]*\]\(\s*data:image/[a-z0-9.+-]+;base64,[^)]+\)`),
+ regexp.MustCompile(`(?i)data:image/[a-z0-9.+-]+;base64,[a-z0-9+/=]{200,}`),
+ regexp.MustCompile(`(?i)base64,[a-z0-9+/=]{200,}`),
+}
+
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
// that supports both keyword-based and vector-based retrieval
type KeywordsVectorHybridRetrieveEngineService struct {
@@ -63,7 +71,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
if slices.Contains(retrieverTypes, types.VectorRetrieverType) {
- embedding, err := embedder.Embed(ctx, indexInfo.Content)
+ embedding, err := embedder.Embed(ctx, sanitizeForEmbedding(ctx, indexInfo.Content))
if err != nil {
return err
}
@@ -148,10 +156,15 @@ func batchEmbedWithBackoff(ctx context.Context, embedder embedding.Embedder, con
// truncation point is char-based, not token-based, so it sits well above any
// realistic token limit. We log a warning whenever truncation kicks in.
func sanitizeForEmbedding(ctx context.Context, content string) string {
- if utf8.RuneCountInString(content) <= safetyMaxChars {
- return content
+ sanitized := content
+ for _, pattern := range embeddingImagePayloadPatterns {
+ sanitized = pattern.ReplaceAllString(sanitized, "[image]")
}
- runes := []rune(content)
+
+ if utf8.RuneCountInString(sanitized) <= safetyMaxChars {
+ return sanitized
+ }
+ runes := []rune(sanitized)
logger.Warnf(ctx, "embedding input truncated: %d runes -> %d", len(runes), safetyMaxChars)
return string(runes[:safetyMaxChars])
}
diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go
new file mode 100644
index 00000000..3d374909
--- /dev/null
+++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer_test.go
@@ -0,0 +1,118 @@
+package retriever
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/embedding"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+type capturingEmbedder struct {
+ embedding.Embedder
+ text string
+ batchTexts []string
+}
+
+func (e *capturingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
+ e.text = text
+ return []float32{1}, nil
+}
+
+func (e *capturingEmbedder) BatchEmbedWithPool(
+ ctx context.Context,
+ model embedding.Embedder,
+ texts []string,
+) ([][]float32, error) {
+ e.batchTexts = append([]string(nil), texts...)
+ embeddings := make([][]float32, len(texts))
+ for i := range texts {
+ embeddings[i] = []float32{1}
+ }
+ return embeddings, nil
+}
+
+type saveOnlyRepository struct {
+ interfaces.RetrieveEngineRepository
+}
+
+func (r *saveOnlyRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
+ return nil
+}
+
+func (r *saveOnlyRepository) BatchSave(
+ ctx context.Context,
+ indexInfoList []*types.IndexInfo,
+ params map[string]any,
+) error {
+ return nil
+}
+
+func TestIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
+ ctx := context.Background()
+ embedder := &capturingEmbedder{}
+ service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
+ payload := strings.Repeat("A", 300)
+ content := "before
after"
+
+ err := service.Index(ctx, embedder, &types.IndexInfo{
+ Content: content,
+ SourceID: "source-1",
+ }, []types.RetrieverType{types.VectorRetrieverType})
+ if err != nil {
+ t.Fatalf("Index returned error: %v", err)
+ }
+ assertImagePayloadRemoved(t, embedder.text, payload)
+}
+
+func TestBatchIndexRemovesInlineImagePayloadBeforeEmbedding(t *testing.T) {
+ ctx := context.Background()
+ embedder := &capturingEmbedder{}
+ service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
+ payload := strings.Repeat("A", 300)
+ content := "before  after"
+
+ err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
+ Content: content,
+ SourceID: "source-1",
+ }}, []types.RetrieverType{types.VectorRetrieverType})
+ if err != nil {
+ t.Fatalf("BatchIndex returned error: %v", err)
+ }
+ if len(embedder.batchTexts) != 1 {
+ t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
+ }
+ assertImagePayloadRemoved(t, embedder.batchTexts[0], payload)
+}
+
+func TestBatchIndexTruncatesOversizedEmbeddingInput(t *testing.T) {
+ ctx := context.Background()
+ embedder := &capturingEmbedder{}
+ service := &KeywordsVectorHybridRetrieveEngineService{indexRepository: &saveOnlyRepository{}}
+
+ err := service.BatchIndex(ctx, embedder, []*types.IndexInfo{{
+ Content: strings.Repeat("x", safetyMaxChars+10),
+ SourceID: "source-1",
+ }}, []types.RetrieverType{types.VectorRetrieverType})
+ if err != nil {
+ t.Fatalf("BatchIndex returned error: %v", err)
+ }
+ if len(embedder.batchTexts) != 1 {
+ t.Fatalf("expected one embedding input, got %d", len(embedder.batchTexts))
+ }
+ if got := len([]rune(embedder.batchTexts[0])); got > safetyMaxChars {
+ t.Fatalf("embedding input length = %d, want <= %d", got, safetyMaxChars)
+ }
+}
+
+func assertImagePayloadRemoved(t *testing.T, content string, payload string) {
+ t.Helper()
+ if strings.Contains(content, "data:image/png;base64") || strings.Contains(content, payload) {
+ t.Fatalf("embedding input still contains inline image payload: %q", content)
+ }
+ if !strings.Contains(content, "before") || !strings.Contains(content, "after") {
+ t.Fatalf("embedding input should preserve surrounding text, got %q", content)
+ }
+}