Add embedding dimension override support
This commit is contained in:
@@ -21,16 +21,17 @@ const (
|
||||
|
||||
// AliyunEmbedder implements text vectorization using Aliyun DashScope multimodal embedding API
|
||||
type AliyunEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -39,6 +40,10 @@ func (e *AliyunEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *AliyunEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// AliyunEmbedParameters represents the parameters for Aliyun DashScope multimodal embedding
|
||||
type AliyunEmbedParameters struct {
|
||||
Dimension int `json:"dimension,omitempty"`
|
||||
@@ -46,8 +51,8 @@ type AliyunEmbedParameters struct {
|
||||
|
||||
// AliyunEmbedRequest represents an Aliyun DashScope multimodal embedding request
|
||||
type AliyunEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input AliyunEmbedInput `json:"input"`
|
||||
Model string `json:"model"`
|
||||
Input AliyunEmbedInput `json:"input"`
|
||||
Parameters *AliyunEmbedParameters `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
@@ -194,7 +199,7 @@ func (e *AliyunEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
Contents: contents,
|
||||
},
|
||||
}
|
||||
if e.dimensions > 0 {
|
||||
if e.supportsDimensionsParam() {
|
||||
reqBody.Parameters = &AliyunEmbedParameters{Dimension: e.dimensions}
|
||||
}
|
||||
|
||||
@@ -253,6 +258,10 @@ func (e *AliyunEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
func (e *AliyunEmbedder) supportsDimensionsParam() bool {
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *AliyunEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -16,16 +15,17 @@ import (
|
||||
|
||||
// AzureOpenAIEmbedder implements text vectorization using Azure OpenAI API
|
||||
type AzureOpenAIEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
apiVersion string
|
||||
httpClient *http.Client
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
apiVersion string
|
||||
httpClient *http.Client
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -34,6 +34,10 @@ func (e *AzureOpenAIEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *AzureOpenAIEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
type azureOpenAIEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
@@ -175,27 +179,7 @@ func (e *AzureOpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData [
|
||||
}
|
||||
|
||||
func (e *AzureOpenAIEmbedder) supportsDimensionsParam() bool {
|
||||
if e.dimensions <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Azure only supports the dimensions parameter on newer embeddings APIs.
|
||||
if strings.TrimSpace(e.apiVersion) < "2024-10-21" {
|
||||
return false
|
||||
}
|
||||
|
||||
modelRef := strings.ToLower(strings.TrimSpace(e.modelID))
|
||||
if modelRef == "" {
|
||||
modelRef = strings.ToLower(strings.TrimSpace(e.modelName))
|
||||
}
|
||||
|
||||
// Fixed-dimension legacy models reject the field entirely.
|
||||
if strings.Contains(modelRef, "ada-002") {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(modelRef, "text-embedding-3-small") ||
|
||||
strings.Contains(modelRef, "text-embedding-3-large")
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
func (e *AzureOpenAIEmbedder) GetModelName() string { return e.modelName }
|
||||
|
||||
@@ -47,6 +47,7 @@ func TestAzureOpenAIEmbedderBatchEmbedSendsConfiguredDimensions(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create embedder: %v", err)
|
||||
}
|
||||
embedder.SetSupportsDimensionOverride(true)
|
||||
embedder.httpClient = &http.Client{Transport: transport}
|
||||
|
||||
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
|
||||
@@ -63,7 +64,7 @@ func TestAzureOpenAIEmbedderBatchEmbedSendsConfiguredDimensions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testing.T) {
|
||||
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsByDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var requestBody map[string]any
|
||||
@@ -103,7 +104,7 @@ func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForOlderAPIVersion(t *testing.T) {
|
||||
func TestAzureOpenAIEmbedderBatchEmbedSendsDimensionsWhenOverrideEnabledRegardlessOfAPIVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var requestBody map[string]any
|
||||
@@ -132,14 +133,19 @@ func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForOlderAPIVersion(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("create embedder: %v", err)
|
||||
}
|
||||
embedder.SetSupportsDimensionOverride(true)
|
||||
embedder.httpClient = &http.Client{Transport: transport}
|
||||
|
||||
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
|
||||
t.Fatalf("BatchEmbed returned error: %v", err)
|
||||
}
|
||||
|
||||
if _, ok := requestBody["dimensions"]; ok {
|
||||
t.Fatalf("expected request body to omit dimensions for older API versions, got %v", requestBody)
|
||||
got, ok := requestBody["dimensions"]
|
||||
if !ok {
|
||||
t.Fatalf("expected request body to include dimensions when override is enabled, got %v", requestBody)
|
||||
}
|
||||
if got != float64(256) {
|
||||
t.Fatalf("unexpected dimensions value: got %v want 256", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,8 +16,9 @@ func TestConfigFromModel(t *testing.T) {
|
||||
APIKey: "sk-xxx",
|
||||
Provider: "openai",
|
||||
EmbeddingParameters: types.EmbeddingParameters{
|
||||
Dimension: 1536,
|
||||
TruncatePromptTokens: 512,
|
||||
Dimension: 1536,
|
||||
TruncatePromptTokens: 512,
|
||||
SupportsDimensionOverride: true,
|
||||
},
|
||||
ExtraConfig: map[string]string{"region": "us-east"},
|
||||
CustomHeaders: map[string]string{"X-Gateway": "g1"},
|
||||
@@ -31,6 +32,9 @@ func TestConfigFromModel(t *testing.T) {
|
||||
if cfg.Dimensions != 1536 || cfg.TruncatePromptTokens != 512 {
|
||||
t.Errorf("embedding params mismatch: %+v", cfg)
|
||||
}
|
||||
if !cfg.SupportsDimensionOverride {
|
||||
t.Errorf("SupportsDimensionOverride not propagated: %+v", cfg)
|
||||
}
|
||||
if cfg.CustomHeaders["X-Gateway"] != "g1" {
|
||||
t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package embedding
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEmbeddingDimensionsParamRequiresExplicitOverride(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
supports func(bool) bool
|
||||
}{
|
||||
{
|
||||
name: "aliyun",
|
||||
supports: func(enabled bool) bool {
|
||||
e := &AliyunEmbedder{dimensions: 1024}
|
||||
e.SetSupportsDimensionOverride(enabled)
|
||||
return e.supportsDimensionsParam()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nvidia",
|
||||
supports: func(enabled bool) bool {
|
||||
e := &NvidiaEmbedder{dimensions: 1024}
|
||||
e.SetSupportsDimensionOverride(enabled)
|
||||
return e.supportsDimensionsParam()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "volcengine",
|
||||
supports: func(enabled bool) bool {
|
||||
e := &VolcengineEmbedder{dimensions: 1024}
|
||||
e.SetSupportsDimensionOverride(enabled)
|
||||
return e.supportsDimensionsParam()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "zhipu",
|
||||
supports: func(enabled bool) bool {
|
||||
e := &ZhipuEmbedder{dimensions: 1024}
|
||||
e.SetSupportsDimensionOverride(enabled)
|
||||
return e.supportsDimensionsParam()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "azure_openai",
|
||||
supports: func(enabled bool) bool {
|
||||
e := &AzureOpenAIEmbedder{dimensions: 1024}
|
||||
e.SetSupportsDimensionOverride(enabled)
|
||||
return e.supportsDimensionsParam()
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.supports(false); got {
|
||||
t.Fatalf("supportsDimensionsParam() = true with override disabled")
|
||||
}
|
||||
if got := tt.supports(true); !got {
|
||||
t.Fatalf("supportsDimensionsParam() = false with override enabled")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddingDimensionsParamRequiresPositiveDimension(t *testing.T) {
|
||||
e := &OpenAIEmbedder{dimensions: 0}
|
||||
e.SetSupportsDimensionOverride(true)
|
||||
if e.supportsDimensionsParam() {
|
||||
t.Fatal("expected dimensions param to be omitted when configured dimension is zero")
|
||||
}
|
||||
}
|
||||
@@ -41,15 +41,16 @@ type EmbedderType string
|
||||
|
||||
// Config represents the embedder configuration
|
||||
type Config struct {
|
||||
Source types.ModelSource `json:"source"`
|
||||
BaseURL string `json:"base_url"`
|
||||
ModelName string `json:"model_name"`
|
||||
APIKey string `json:"api_key"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
|
||||
Dimensions int `json:"dimensions"`
|
||||
ModelID string `json:"model_id"`
|
||||
Provider string `json:"provider"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
Source types.ModelSource `json:"source"`
|
||||
BaseURL string `json:"base_url"`
|
||||
ModelName string `json:"model_name"`
|
||||
APIKey string `json:"api_key"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
|
||||
Dimensions int `json:"dimensions"`
|
||||
SupportsDimensionOverride bool `json:"supports_dimension_override"`
|
||||
ModelID string `json:"model_id"`
|
||||
Provider string `json:"provider"`
|
||||
ExtraConfig map[string]string `json:"extra_config"`
|
||||
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string `json:"custom_headers"`
|
||||
AppID string
|
||||
@@ -64,18 +65,19 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) Config {
|
||||
return Config{}
|
||||
}
|
||||
return Config{
|
||||
Source: m.Source,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
ModelID: m.ID,
|
||||
ModelName: m.Name,
|
||||
Dimensions: m.Parameters.EmbeddingParameters.Dimension,
|
||||
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
|
||||
Provider: m.Parameters.Provider,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
Source: m.Source,
|
||||
BaseURL: m.Parameters.BaseURL,
|
||||
APIKey: m.Parameters.APIKey,
|
||||
ModelID: m.ID,
|
||||
ModelName: m.Name,
|
||||
Dimensions: m.Parameters.EmbeddingParameters.Dimension,
|
||||
SupportsDimensionOverride: m.Parameters.EmbeddingParameters.SupportsDimensionOverride,
|
||||
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
|
||||
Provider: m.Parameters.Provider,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
AppID: appID,
|
||||
AppSecret: appSecret,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +87,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
|
||||
if err != nil {
|
||||
return e, err
|
||||
}
|
||||
if setter, ok := e.(interface{ SetSupportsDimensionOverride(bool) }); ok {
|
||||
setter.SetSupportsDimensionOverride(config.SupportsDimensionOverride)
|
||||
}
|
||||
if logger.LLMDebugEnabled() {
|
||||
e = &debugEmbedder{inner: e}
|
||||
}
|
||||
|
||||
@@ -19,16 +19,17 @@ const geminiEmbeddingBaseURL = "https://generativelanguage.googleapis.com/v1beta
|
||||
// GeminiEmbedder implements text vectorization using the native Gemini
|
||||
// embedContent / batchEmbedContents REST API.
|
||||
type GeminiEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -96,6 +97,10 @@ func (e *GeminiEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *GeminiEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
func (e *GeminiEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
embeddings, err := e.BatchEmbed(ctx, []string{text})
|
||||
if err != nil {
|
||||
@@ -114,13 +119,16 @@ func (e *GeminiEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
|
||||
requests := make([]geminiEmbedRequest, 0, len(texts))
|
||||
for _, text := range texts {
|
||||
requests = append(requests, geminiEmbedRequest{
|
||||
req := geminiEmbedRequest{
|
||||
Model: "models/" + e.modelName,
|
||||
Content: geminiContent{Parts: []geminiPart{
|
||||
{Text: text},
|
||||
}},
|
||||
OutputDimensionality: e.dimensions,
|
||||
})
|
||||
}
|
||||
if e.supportsDimensionOverride && e.dimensions > 0 {
|
||||
req.OutputDimensionality = e.dimensions
|
||||
}
|
||||
requests = append(requests, req)
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(geminiBatchEmbedRequest{Requests: requests})
|
||||
|
||||
@@ -54,8 +54,8 @@ func TestGeminiEmbedderBatchEmbedUsesNativeAPI(t *testing.T) {
|
||||
if gotReq.Requests[0].Model != "models/gemini-embedding-2" {
|
||||
t.Fatalf("request model = %q", gotReq.Requests[0].Model)
|
||||
}
|
||||
if gotReq.Requests[0].OutputDimensionality != 768 {
|
||||
t.Fatalf("output_dimensionality = %d", gotReq.Requests[0].OutputDimensionality)
|
||||
if gotReq.Requests[0].OutputDimensionality != 0 {
|
||||
t.Fatalf("output_dimensionality = %d, want omitted by default", gotReq.Requests[0].OutputDimensionality)
|
||||
}
|
||||
if gotReq.Requests[0].Content.Parts[0].Text != "hello" {
|
||||
t.Fatalf("first text = %q", gotReq.Requests[0].Content.Parts[0].Text)
|
||||
@@ -65,6 +65,32 @@ func TestGeminiEmbedderBatchEmbedUsesNativeAPI(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiEmbedderBatchEmbedSendsOutputDimensionalityWhenOverrideEnabled(t *testing.T) {
|
||||
var gotReq geminiBatchEmbedRequest
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"embeddings":[{"values":[0.1,0.2]}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
embedder, err := NewGeminiEmbedder("test-key", server.URL, "gemini-embedding-2",
|
||||
0, 768, "model-id", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("NewGeminiEmbedder: %v", err)
|
||||
}
|
||||
embedder.SetSupportsDimensionOverride(true)
|
||||
|
||||
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
|
||||
t.Fatalf("BatchEmbed: %v", err)
|
||||
}
|
||||
if gotReq.Requests[0].OutputDimensionality != 768 {
|
||||
t.Fatalf("output_dimensionality = %d, want 768", gotReq.Requests[0].OutputDimensionality)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiEmbedderReturnsAPIErrorBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
|
||||
|
||||
@@ -16,15 +16,16 @@ import (
|
||||
// JinaEmbedder implements text vectorization functionality using Jina AI API
|
||||
// Jina API is mostly OpenAI-compatible but does NOT support truncate_prompt_tokens
|
||||
type JinaEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -33,6 +34,10 @@ func (e *JinaEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *JinaEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// JinaEmbedRequest represents a Jina embedding request
|
||||
// Note: Jina uses 'truncate' (boolean) instead of 'truncate_prompt_tokens' (integer)
|
||||
type JinaEmbedRequest struct {
|
||||
@@ -146,8 +151,7 @@ func (e *JinaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]floa
|
||||
Truncate: true, // Enable truncation for long texts
|
||||
}
|
||||
|
||||
// Only include dimensions if specified and greater than 0
|
||||
if e.dimensions > 0 {
|
||||
if e.supportsDimensionOverride && e.dimensions > 0 {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
|
||||
|
||||
@@ -32,9 +32,9 @@ func (d *debugEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder,
|
||||
return d.inner.BatchEmbedWithPool(ctx, d, texts)
|
||||
}
|
||||
|
||||
func (d *debugEmbedder) GetModelName() string { return d.inner.GetModelName() }
|
||||
func (d *debugEmbedder) GetDimensions() int { return d.inner.GetDimensions() }
|
||||
func (d *debugEmbedder) GetModelID() string { return d.inner.GetModelID() }
|
||||
func (d *debugEmbedder) GetModelName() string { return d.inner.GetModelName() }
|
||||
func (d *debugEmbedder) GetDimensions() int { return d.inner.GetDimensions() }
|
||||
func (d *debugEmbedder) GetModelID() string { return d.inner.GetModelID() }
|
||||
|
||||
func singleToDouble(v []float32) [][]float32 {
|
||||
if v == nil {
|
||||
|
||||
@@ -16,15 +16,16 @@ import (
|
||||
|
||||
// NvidiaEmbedder implements text vectorization functionality using NVIDIA API
|
||||
type NvidiaEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -33,6 +34,10 @@ func (e *NvidiaEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *NvidiaEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// NvidiaEmbedRequest represents an NVIDIA embedding request
|
||||
type NvidiaEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
@@ -145,9 +150,11 @@ func (e *NvidiaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
EncodingFormat: "float",
|
||||
Dimensions: e.dimensions,
|
||||
InputType: "passage",
|
||||
}
|
||||
if e.supportsDimensionsParam() {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
isQuery, _ := ctx.Value(types.EmbedQueryContextKey).(bool)
|
||||
if isQuery {
|
||||
reqBody.InputType = "query"
|
||||
@@ -202,6 +209,10 @@ func (e *NvidiaEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
func (e *NvidiaEmbedder) supportsDimensionsParam() bool {
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *NvidiaEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
|
||||
@@ -12,11 +12,12 @@ import (
|
||||
|
||||
// OllamaEmbedder implements text vectorization functionality using Ollama
|
||||
type OllamaEmbedder struct {
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
ollamaService *ollama.OllamaService
|
||||
dimensions int
|
||||
modelID string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
ollamaService *ollama.OllamaService
|
||||
dimensions int
|
||||
modelID string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -65,6 +66,10 @@ func (e *OllamaEmbedder) ensureModelAvailable(ctx context.Context) error {
|
||||
return e.ollamaService.EnsureModelAvailable(ctx, e.modelName)
|
||||
}
|
||||
|
||||
func (e *OllamaEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *OllamaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
embedding, err := e.BatchEmbed(ctx, []string{text})
|
||||
@@ -88,10 +93,12 @@ func (e *OllamaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
|
||||
// Create request
|
||||
req := &ollamaapi.EmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
Dimensions: e.dimensions,
|
||||
Options: make(map[string]interface{}),
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
Options: make(map[string]interface{}),
|
||||
}
|
||||
if e.supportsDimensionOverride && e.dimensions > 0 {
|
||||
req.Dimensions = e.dimensions
|
||||
}
|
||||
|
||||
// Set truncation parameters
|
||||
|
||||
@@ -15,16 +15,17 @@ import (
|
||||
|
||||
// OpenAIEmbedder implements text vectorization functionality using OpenAI API
|
||||
type OpenAIEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -88,6 +89,10 @@ func (e *OpenAIEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *OpenAIEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *OpenAIEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
for range 3 {
|
||||
@@ -162,9 +167,11 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
EncodingFormat: "float",
|
||||
Dimensions: e.dimensions,
|
||||
TruncatePromptTokens: e.truncatePromptTokens,
|
||||
}
|
||||
if e.supportsDimensionsParam() {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
@@ -248,6 +255,10 @@ func (e *OpenAIEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
func (e *OpenAIEmbedder) supportsDimensionsParam() bool {
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *OpenAIEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsByDefault(t *testing.T) {
|
||||
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-3-small", 256, false)
|
||||
|
||||
if _, ok := requestBody["dimensions"]; ok {
|
||||
t.Fatalf("expected request body to omit dimensions by default, got %v", requestBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIEmbedderBatchEmbedSendsDimensionsWhenOverrideEnabled(t *testing.T) {
|
||||
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-3-small", 256, true)
|
||||
|
||||
got, ok := requestBody["dimensions"]
|
||||
if !ok {
|
||||
t.Fatalf("expected request body to include dimensions, got %v", requestBody)
|
||||
}
|
||||
if got != float64(256) {
|
||||
t.Fatalf("unexpected dimensions value: got %v want 256", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsForOpenAICompatibleModels(t *testing.T) {
|
||||
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-v3", 1024, false)
|
||||
|
||||
if _, ok := requestBody["dimensions"]; ok {
|
||||
t.Fatalf("expected request body to omit dimensions for OpenAI-compatible model, got %v", requestBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testing.T) {
|
||||
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-ada-002", 1536, false)
|
||||
|
||||
if _, ok := requestBody["dimensions"]; ok {
|
||||
t.Fatalf("expected request body to omit dimensions for fixed-size model, got %v", requestBody)
|
||||
}
|
||||
}
|
||||
|
||||
func captureOpenAIEmbeddingRequest(t *testing.T, modelName string, dimensions int, supportsDimensionOverride bool) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
requestBody := map[string]any{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/embeddings" {
|
||||
t.Fatalf("unexpected request path: %s", r.URL.Path)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
|
||||
t.Fatalf("decode request body: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2],"index":0}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
embedder, err := NewOpenAIEmbedder(
|
||||
"test-key",
|
||||
server.URL,
|
||||
modelName,
|
||||
511,
|
||||
dimensions,
|
||||
"8f7d6082-5a15-4f84-ae55-88b2bdac4ba0",
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAIEmbedder: %v", err)
|
||||
}
|
||||
embedder.SetSupportsDimensionOverride(supportsDimensionOverride)
|
||||
|
||||
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
|
||||
t.Fatalf("BatchEmbed: %v", err)
|
||||
}
|
||||
|
||||
return requestBody
|
||||
}
|
||||
@@ -21,16 +21,17 @@ const (
|
||||
|
||||
// VolcengineEmbedder implements text vectorization using Volcengine Ark multimodal embedding API
|
||||
type VolcengineEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -39,6 +40,10 @@ func (e *VolcengineEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *VolcengineEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// VolcengineEmbedRequest represents a Volcengine Ark multimodal embedding request
|
||||
type VolcengineEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
@@ -200,9 +205,11 @@ func (e *VolcengineEmbedder) BatchEmbed(ctx context.Context, texts []string) ([]
|
||||
}
|
||||
|
||||
reqBody := VolcengineEmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: input,
|
||||
Dimensions: e.dimensions,
|
||||
Model: e.modelName,
|
||||
Input: input,
|
||||
}
|
||||
if e.supportsDimensionsParam() {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
@@ -252,6 +259,10 @@ func (e *VolcengineEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
func (e *VolcengineEmbedder) supportsDimensionsParam() bool {
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *VolcengineEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
|
||||
@@ -18,14 +18,15 @@ const weKnoraCloudEmbedPath = "/api/v1/embeddings"
|
||||
|
||||
// WeKnoraCloudEmbedder 实现 embedding.Embedder 接口,对接 WeKnoraCloud /api/v1/embeddings
|
||||
type WeKnoraCloudEmbedder struct {
|
||||
modelName string
|
||||
remoteModelName string
|
||||
modelID string
|
||||
appID string
|
||||
apiKey string
|
||||
baseURL string
|
||||
dimensions int
|
||||
client *http.Client
|
||||
modelName string
|
||||
remoteModelName string
|
||||
modelID string
|
||||
appID string
|
||||
apiKey string
|
||||
baseURL string
|
||||
dimensions int
|
||||
supportsDimensionOverride bool
|
||||
client *http.Client
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -42,14 +43,15 @@ func NewWeKnoraCloudEmbedder(config Config) (*WeKnoraCloudEmbedder, error) {
|
||||
remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"])
|
||||
}
|
||||
return &WeKnoraCloudEmbedder{
|
||||
modelName: config.ModelName,
|
||||
remoteModelName: remoteModelName,
|
||||
modelID: config.ModelID,
|
||||
appID: config.AppID,
|
||||
apiKey: config.AppSecret,
|
||||
baseURL: strings.TrimRight(config.BaseURL, "/"),
|
||||
dimensions: config.Dimensions,
|
||||
client: &http.Client{Timeout: 60 * time.Second},
|
||||
modelName: config.ModelName,
|
||||
remoteModelName: remoteModelName,
|
||||
modelID: config.ModelID,
|
||||
appID: config.AppID,
|
||||
apiKey: config.AppSecret,
|
||||
baseURL: strings.TrimRight(config.BaseURL, "/"),
|
||||
dimensions: config.Dimensions,
|
||||
supportsDimensionOverride: config.SupportsDimensionOverride,
|
||||
client: &http.Client{Timeout: 60 * time.Second},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -79,7 +81,10 @@ func (e *WeKnoraCloudEmbedder) Embed(ctx context.Context, text string) ([]float3
|
||||
}
|
||||
|
||||
func (e *WeKnoraCloudEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts, Dimensions: e.dimensions}
|
||||
reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts}
|
||||
if e.supportsDimensionOverride && e.dimensions > 0 {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud embedder: marshal: %w", err)
|
||||
@@ -129,6 +134,10 @@ func (e *WeKnoraCloudEmbedder) BatchEmbedWithPool(ctx context.Context, model Emb
|
||||
return e.BatchEmbed(ctx, texts)
|
||||
}
|
||||
|
||||
func (e *WeKnoraCloudEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
func (e *WeKnoraCloudEmbedder) effectiveModelName() string {
|
||||
if e.remoteModelName != "" {
|
||||
return e.remoteModelName
|
||||
|
||||
@@ -16,16 +16,17 @@ import (
|
||||
|
||||
// ZhipuEmbedder implements text vectorization functionality using Zhipu AI API
|
||||
type ZhipuEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
customHeaders map[string]string
|
||||
supportsDimensionOverride bool
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
@@ -88,6 +89,10 @@ func (e *ZhipuEmbedder) SetCustomHeaders(headers map[string]string) {
|
||||
e.customHeaders = headers
|
||||
}
|
||||
|
||||
func (e *ZhipuEmbedder) SetSupportsDimensionOverride(supported bool) {
|
||||
e.supportsDimensionOverride = supported
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *ZhipuEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
for range 3 {
|
||||
@@ -149,9 +154,11 @@ func (e *ZhipuEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]flo
|
||||
reqBody := ZhipuEmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
Dimensions: e.dimensions,
|
||||
TruncatePromptTokens: e.truncatePromptTokens,
|
||||
}
|
||||
if e.supportsDimensionsParam() {
|
||||
reqBody.Dimensions = e.dimensions
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
@@ -235,6 +242,10 @@ func (e *ZhipuEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
func (e *ZhipuEmbedder) supportsDimensionsParam() bool {
|
||||
return e.supportsDimensionOverride && e.dimensions > 0
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *ZhipuEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
|
||||
Reference in New Issue
Block a user