Add embedding dimension override support

This commit is contained in:
wizardchen
2026-06-15 19:16:31 +08:00
committed by lyingbug
parent 90f8e8b390
commit 53e0333a53
27 changed files with 476 additions and 189 deletions
+22 -13
View File
@@ -21,16 +21,17 @@ const (
// AliyunEmbedder implements text vectorization using Aliyun DashScope multimodal embedding API
type AliyunEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -39,6 +40,10 @@ func (e *AliyunEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *AliyunEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// AliyunEmbedParameters represents the parameters for Aliyun DashScope multimodal embedding
type AliyunEmbedParameters struct {
Dimension int `json:"dimension,omitempty"`
@@ -46,8 +51,8 @@ type AliyunEmbedParameters struct {
// AliyunEmbedRequest represents an Aliyun DashScope multimodal embedding request
type AliyunEmbedRequest struct {
Model string `json:"model"`
Input AliyunEmbedInput `json:"input"`
Model string `json:"model"`
Input AliyunEmbedInput `json:"input"`
Parameters *AliyunEmbedParameters `json:"parameters,omitempty"`
}
@@ -194,7 +199,7 @@ func (e *AliyunEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
Contents: contents,
},
}
if e.dimensions > 0 {
if e.supportsDimensionsParam() {
reqBody.Parameters = &AliyunEmbedParameters{Dimension: e.dimensions}
}
@@ -253,6 +258,10 @@ func (e *AliyunEmbedder) GetModelName() string {
return e.modelName
}
func (e *AliyunEmbedder) supportsDimensionsParam() bool {
return e.supportsDimensionOverride && e.dimensions > 0
}
// GetDimensions returns the vector dimensions
func (e *AliyunEmbedder) GetDimensions() int {
return e.dimensions
+16 -32
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
@@ -16,16 +15,17 @@ import (
// AzureOpenAIEmbedder implements text vectorization using Azure OpenAI API
type AzureOpenAIEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
apiVersion string
httpClient *http.Client
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
apiVersion string
httpClient *http.Client
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -34,6 +34,10 @@ func (e *AzureOpenAIEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *AzureOpenAIEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
type azureOpenAIEmbedRequest struct {
Model string `json:"model"`
Input []string `json:"input"`
@@ -175,27 +179,7 @@ func (e *AzureOpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData [
}
func (e *AzureOpenAIEmbedder) supportsDimensionsParam() bool {
if e.dimensions <= 0 {
return false
}
// Azure only supports the dimensions parameter on newer embeddings APIs.
if strings.TrimSpace(e.apiVersion) < "2024-10-21" {
return false
}
modelRef := strings.ToLower(strings.TrimSpace(e.modelID))
if modelRef == "" {
modelRef = strings.ToLower(strings.TrimSpace(e.modelName))
}
// Fixed-dimension legacy models reject the field entirely.
if strings.Contains(modelRef, "ada-002") {
return false
}
return strings.Contains(modelRef, "text-embedding-3-small") ||
strings.Contains(modelRef, "text-embedding-3-large")
return e.supportsDimensionOverride && e.dimensions > 0
}
func (e *AzureOpenAIEmbedder) GetModelName() string { return e.modelName }
+10 -4
View File
@@ -47,6 +47,7 @@ func TestAzureOpenAIEmbedderBatchEmbedSendsConfiguredDimensions(t *testing.T) {
if err != nil {
t.Fatalf("create embedder: %v", err)
}
embedder.SetSupportsDimensionOverride(true)
embedder.httpClient = &http.Client{Transport: transport}
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
@@ -63,7 +64,7 @@ func TestAzureOpenAIEmbedderBatchEmbedSendsConfiguredDimensions(t *testing.T) {
}
}
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testing.T) {
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsByDefault(t *testing.T) {
t.Parallel()
var requestBody map[string]any
@@ -103,7 +104,7 @@ func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testi
}
}
func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForOlderAPIVersion(t *testing.T) {
func TestAzureOpenAIEmbedderBatchEmbedSendsDimensionsWhenOverrideEnabledRegardlessOfAPIVersion(t *testing.T) {
t.Parallel()
var requestBody map[string]any
@@ -132,14 +133,19 @@ func TestAzureOpenAIEmbedderBatchEmbedOmitsDimensionsForOlderAPIVersion(t *testi
if err != nil {
t.Fatalf("create embedder: %v", err)
}
embedder.SetSupportsDimensionOverride(true)
embedder.httpClient = &http.Client{Transport: transport}
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
t.Fatalf("BatchEmbed returned error: %v", err)
}
if _, ok := requestBody["dimensions"]; ok {
t.Fatalf("expected request body to omit dimensions for older API versions, got %v", requestBody)
got, ok := requestBody["dimensions"]
if !ok {
t.Fatalf("expected request body to include dimensions when override is enabled, got %v", requestBody)
}
if got != float64(256) {
t.Fatalf("unexpected dimensions value: got %v want 256", got)
}
}
@@ -16,8 +16,9 @@ func TestConfigFromModel(t *testing.T) {
APIKey: "sk-xxx",
Provider: "openai",
EmbeddingParameters: types.EmbeddingParameters{
Dimension: 1536,
TruncatePromptTokens: 512,
Dimension: 1536,
TruncatePromptTokens: 512,
SupportsDimensionOverride: true,
},
ExtraConfig: map[string]string{"region": "us-east"},
CustomHeaders: map[string]string{"X-Gateway": "g1"},
@@ -31,6 +32,9 @@ func TestConfigFromModel(t *testing.T) {
if cfg.Dimensions != 1536 || cfg.TruncatePromptTokens != 512 {
t.Errorf("embedding params mismatch: %+v", cfg)
}
if !cfg.SupportsDimensionOverride {
t.Errorf("SupportsDimensionOverride not propagated: %+v", cfg)
}
if cfg.CustomHeaders["X-Gateway"] != "g1" {
t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
}
@@ -0,0 +1,70 @@
package embedding
import "testing"
func TestEmbeddingDimensionsParamRequiresExplicitOverride(t *testing.T) {
tests := []struct {
name string
supports func(bool) bool
}{
{
name: "aliyun",
supports: func(enabled bool) bool {
e := &AliyunEmbedder{dimensions: 1024}
e.SetSupportsDimensionOverride(enabled)
return e.supportsDimensionsParam()
},
},
{
name: "nvidia",
supports: func(enabled bool) bool {
e := &NvidiaEmbedder{dimensions: 1024}
e.SetSupportsDimensionOverride(enabled)
return e.supportsDimensionsParam()
},
},
{
name: "volcengine",
supports: func(enabled bool) bool {
e := &VolcengineEmbedder{dimensions: 1024}
e.SetSupportsDimensionOverride(enabled)
return e.supportsDimensionsParam()
},
},
{
name: "zhipu",
supports: func(enabled bool) bool {
e := &ZhipuEmbedder{dimensions: 1024}
e.SetSupportsDimensionOverride(enabled)
return e.supportsDimensionsParam()
},
},
{
name: "azure_openai",
supports: func(enabled bool) bool {
e := &AzureOpenAIEmbedder{dimensions: 1024}
e.SetSupportsDimensionOverride(enabled)
return e.supportsDimensionsParam()
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.supports(false); got {
t.Fatalf("supportsDimensionsParam() = true with override disabled")
}
if got := tt.supports(true); !got {
t.Fatalf("supportsDimensionsParam() = false with override enabled")
}
})
}
}
func TestEmbeddingDimensionsParamRequiresPositiveDimension(t *testing.T) {
e := &OpenAIEmbedder{dimensions: 0}
e.SetSupportsDimensionOverride(true)
if e.supportsDimensionsParam() {
t.Fatal("expected dimensions param to be omitted when configured dimension is zero")
}
}
+26 -21
View File
@@ -41,15 +41,16 @@ type EmbedderType string
// Config represents the embedder configuration
type Config struct {
Source types.ModelSource `json:"source"`
BaseURL string `json:"base_url"`
ModelName string `json:"model_name"`
APIKey string `json:"api_key"`
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
Dimensions int `json:"dimensions"`
ModelID string `json:"model_id"`
Provider string `json:"provider"`
ExtraConfig map[string]string `json:"extra_config"`
Source types.ModelSource `json:"source"`
BaseURL string `json:"base_url"`
ModelName string `json:"model_name"`
APIKey string `json:"api_key"`
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
Dimensions int `json:"dimensions"`
SupportsDimensionOverride bool `json:"supports_dimension_override"`
ModelID string `json:"model_id"`
Provider string `json:"provider"`
ExtraConfig map[string]string `json:"extra_config"`
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
CustomHeaders map[string]string `json:"custom_headers"`
AppID string
@@ -64,18 +65,19 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) Config {
return Config{}
}
return Config{
Source: m.Source,
BaseURL: m.Parameters.BaseURL,
APIKey: m.Parameters.APIKey,
ModelID: m.ID,
ModelName: m.Name,
Dimensions: m.Parameters.EmbeddingParameters.Dimension,
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
Provider: m.Parameters.Provider,
ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
AppID: appID,
AppSecret: appSecret,
Source: m.Source,
BaseURL: m.Parameters.BaseURL,
APIKey: m.Parameters.APIKey,
ModelID: m.ID,
ModelName: m.Name,
Dimensions: m.Parameters.EmbeddingParameters.Dimension,
SupportsDimensionOverride: m.Parameters.EmbeddingParameters.SupportsDimensionOverride,
TruncatePromptTokens: m.Parameters.EmbeddingParameters.TruncatePromptTokens,
Provider: m.Parameters.Provider,
ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
AppID: appID,
AppSecret: appSecret,
}
}
@@ -85,6 +87,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
if err != nil {
return e, err
}
if setter, ok := e.(interface{ SetSupportsDimensionOverride(bool) }); ok {
setter.SetSupportsDimensionOverride(config.SupportsDimensionOverride)
}
if logger.LLMDebugEnabled() {
e = &debugEmbedder{inner: e}
}
+21 -13
View File
@@ -19,16 +19,17 @@ const geminiEmbeddingBaseURL = "https://generativelanguage.googleapis.com/v1beta
// GeminiEmbedder implements text vectorization using the native Gemini
// embedContent / batchEmbedContents REST API.
type GeminiEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -96,6 +97,10 @@ func (e *GeminiEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *GeminiEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
func (e *GeminiEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
embeddings, err := e.BatchEmbed(ctx, []string{text})
if err != nil {
@@ -114,13 +119,16 @@ func (e *GeminiEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
requests := make([]geminiEmbedRequest, 0, len(texts))
for _, text := range texts {
requests = append(requests, geminiEmbedRequest{
req := geminiEmbedRequest{
Model: "models/" + e.modelName,
Content: geminiContent{Parts: []geminiPart{
{Text: text},
}},
OutputDimensionality: e.dimensions,
})
}
if e.supportsDimensionOverride && e.dimensions > 0 {
req.OutputDimensionality = e.dimensions
}
requests = append(requests, req)
}
jsonData, err := json.Marshal(geminiBatchEmbedRequest{Requests: requests})
+28 -2
View File
@@ -54,8 +54,8 @@ func TestGeminiEmbedderBatchEmbedUsesNativeAPI(t *testing.T) {
if gotReq.Requests[0].Model != "models/gemini-embedding-2" {
t.Fatalf("request model = %q", gotReq.Requests[0].Model)
}
if gotReq.Requests[0].OutputDimensionality != 768 {
t.Fatalf("output_dimensionality = %d", gotReq.Requests[0].OutputDimensionality)
if gotReq.Requests[0].OutputDimensionality != 0 {
t.Fatalf("output_dimensionality = %d, want omitted by default", gotReq.Requests[0].OutputDimensionality)
}
if gotReq.Requests[0].Content.Parts[0].Text != "hello" {
t.Fatalf("first text = %q", gotReq.Requests[0].Content.Parts[0].Text)
@@ -65,6 +65,32 @@ func TestGeminiEmbedderBatchEmbedUsesNativeAPI(t *testing.T) {
}
}
func TestGeminiEmbedderBatchEmbedSendsOutputDimensionalityWhenOverrideEnabled(t *testing.T) {
var gotReq geminiBatchEmbedRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil {
t.Fatalf("decode request: %v", err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"embeddings":[{"values":[0.1,0.2]}]}`))
}))
defer server.Close()
embedder, err := NewGeminiEmbedder("test-key", server.URL, "gemini-embedding-2",
0, 768, "model-id", nil)
if err != nil {
t.Fatalf("NewGeminiEmbedder: %v", err)
}
embedder.SetSupportsDimensionOverride(true)
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
t.Fatalf("BatchEmbed: %v", err)
}
if gotReq.Requests[0].OutputDimensionality != 768 {
t.Fatalf("output_dimensionality = %d, want 768", gotReq.Requests[0].OutputDimensionality)
}
}
func TestGeminiEmbedderReturnsAPIErrorBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
+15 -11
View File
@@ -16,15 +16,16 @@ import (
// JinaEmbedder implements text vectorization functionality using Jina AI API
// Jina API is mostly OpenAI-compatible but does NOT support truncate_prompt_tokens
type JinaEmbedder struct {
apiKey string
baseURL string
modelName string
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -33,6 +34,10 @@ func (e *JinaEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *JinaEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// JinaEmbedRequest represents a Jina embedding request
// Note: Jina uses 'truncate' (boolean) instead of 'truncate_prompt_tokens' (integer)
type JinaEmbedRequest struct {
@@ -146,8 +151,7 @@ func (e *JinaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]floa
Truncate: true, // Enable truncation for long texts
}
// Only include dimensions if specified and greater than 0
if e.dimensions > 0 {
if e.supportsDimensionOverride && e.dimensions > 0 {
reqBody.Dimensions = e.dimensions
}
+3 -3
View File
@@ -32,9 +32,9 @@ func (d *debugEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder,
return d.inner.BatchEmbedWithPool(ctx, d, texts)
}
func (d *debugEmbedder) GetModelName() string { return d.inner.GetModelName() }
func (d *debugEmbedder) GetDimensions() int { return d.inner.GetDimensions() }
func (d *debugEmbedder) GetModelID() string { return d.inner.GetModelID() }
func (d *debugEmbedder) GetModelName() string { return d.inner.GetModelName() }
func (d *debugEmbedder) GetDimensions() int { return d.inner.GetDimensions() }
func (d *debugEmbedder) GetModelID() string { return d.inner.GetModelID() }
func singleToDouble(v []float32) [][]float32 {
if v == nil {
+21 -10
View File
@@ -16,15 +16,16 @@ import (
// NvidiaEmbedder implements text vectorization functionality using NVIDIA API
type NvidiaEmbedder struct {
apiKey string
baseURL string
modelName string
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -33,6 +34,10 @@ func (e *NvidiaEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *NvidiaEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// NvidiaEmbedRequest represents an NVIDIA embedding request
type NvidiaEmbedRequest struct {
Model string `json:"model"`
@@ -145,9 +150,11 @@ func (e *NvidiaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
Model: e.modelName,
Input: texts,
EncodingFormat: "float",
Dimensions: e.dimensions,
InputType: "passage",
}
if e.supportsDimensionsParam() {
reqBody.Dimensions = e.dimensions
}
isQuery, _ := ctx.Value(types.EmbedQueryContextKey).(bool)
if isQuery {
reqBody.InputType = "query"
@@ -202,6 +209,10 @@ func (e *NvidiaEmbedder) GetModelName() string {
return e.modelName
}
func (e *NvidiaEmbedder) supportsDimensionsParam() bool {
return e.supportsDimensionOverride && e.dimensions > 0
}
// GetDimensions returns the vector dimensions
func (e *NvidiaEmbedder) GetDimensions() int {
return e.dimensions
+16 -9
View File
@@ -12,11 +12,12 @@ import (
// OllamaEmbedder implements text vectorization functionality using Ollama
type OllamaEmbedder struct {
modelName string
truncatePromptTokens int
ollamaService *ollama.OllamaService
dimensions int
modelID string
modelName string
truncatePromptTokens int
ollamaService *ollama.OllamaService
dimensions int
modelID string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -65,6 +66,10 @@ func (e *OllamaEmbedder) ensureModelAvailable(ctx context.Context) error {
return e.ollamaService.EnsureModelAvailable(ctx, e.modelName)
}
func (e *OllamaEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// Embed converts text to vector
func (e *OllamaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
embedding, err := e.BatchEmbed(ctx, []string{text})
@@ -88,10 +93,12 @@ func (e *OllamaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
// Create request
req := &ollamaapi.EmbedRequest{
Model: e.modelName,
Input: texts,
Dimensions: e.dimensions,
Options: make(map[string]interface{}),
Model: e.modelName,
Input: texts,
Options: make(map[string]interface{}),
}
if e.supportsDimensionOverride && e.dimensions > 0 {
req.Dimensions = e.dimensions
}
// Set truncation parameters
+22 -11
View File
@@ -15,16 +15,17 @@ import (
// OpenAIEmbedder implements text vectorization functionality using OpenAI API
type OpenAIEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -88,6 +89,10 @@ func (e *OpenAIEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *OpenAIEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// Embed converts text to vector
func (e *OpenAIEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
for range 3 {
@@ -162,9 +167,11 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl
Model: e.modelName,
Input: texts,
EncodingFormat: "float",
Dimensions: e.dimensions,
TruncatePromptTokens: e.truncatePromptTokens,
}
if e.supportsDimensionsParam() {
reqBody.Dimensions = e.dimensions
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
@@ -248,6 +255,10 @@ func (e *OpenAIEmbedder) GetModelName() string {
return e.modelName
}
func (e *OpenAIEmbedder) supportsDimensionsParam() bool {
return e.supportsDimensionOverride && e.dimensions > 0
}
// GetDimensions returns the vector dimensions
func (e *OpenAIEmbedder) GetDimensions() int {
return e.dimensions
+82
View File
@@ -0,0 +1,82 @@
package embedding
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsByDefault(t *testing.T) {
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-3-small", 256, false)
if _, ok := requestBody["dimensions"]; ok {
t.Fatalf("expected request body to omit dimensions by default, got %v", requestBody)
}
}
func TestOpenAIEmbedderBatchEmbedSendsDimensionsWhenOverrideEnabled(t *testing.T) {
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-3-small", 256, true)
got, ok := requestBody["dimensions"]
if !ok {
t.Fatalf("expected request body to include dimensions, got %v", requestBody)
}
if got != float64(256) {
t.Fatalf("unexpected dimensions value: got %v want 256", got)
}
}
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsForOpenAICompatibleModels(t *testing.T) {
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-v3", 1024, false)
if _, ok := requestBody["dimensions"]; ok {
t.Fatalf("expected request body to omit dimensions for OpenAI-compatible model, got %v", requestBody)
}
}
func TestOpenAIEmbedderBatchEmbedOmitsDimensionsForFixedSizeModels(t *testing.T) {
requestBody := captureOpenAIEmbeddingRequest(t, "text-embedding-ada-002", 1536, false)
if _, ok := requestBody["dimensions"]; ok {
t.Fatalf("expected request body to omit dimensions for fixed-size model, got %v", requestBody)
}
}
func captureOpenAIEmbeddingRequest(t *testing.T, modelName string, dimensions int, supportsDimensionOverride bool) map[string]any {
t.Helper()
requestBody := map[string]any{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/embeddings" {
t.Fatalf("unexpected request path: %s", r.URL.Path)
}
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
t.Fatalf("decode request body: %v", err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2],"index":0}]}`))
}))
defer server.Close()
embedder, err := NewOpenAIEmbedder(
"test-key",
server.URL,
modelName,
511,
dimensions,
"8f7d6082-5a15-4f84-ae55-88b2bdac4ba0",
nil,
)
if err != nil {
t.Fatalf("NewOpenAIEmbedder: %v", err)
}
embedder.SetSupportsDimensionOverride(supportsDimensionOverride)
if _, err := embedder.BatchEmbed(context.Background(), []string{"hello"}); err != nil {
t.Fatalf("BatchEmbed: %v", err)
}
return requestBody
}
+24 -13
View File
@@ -21,16 +21,17 @@ const (
// VolcengineEmbedder implements text vectorization using Volcengine Ark multimodal embedding API
type VolcengineEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -39,6 +40,10 @@ func (e *VolcengineEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *VolcengineEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// VolcengineEmbedRequest represents a Volcengine Ark multimodal embedding request
type VolcengineEmbedRequest struct {
Model string `json:"model"`
@@ -200,9 +205,11 @@ func (e *VolcengineEmbedder) BatchEmbed(ctx context.Context, texts []string) ([]
}
reqBody := VolcengineEmbedRequest{
Model: e.modelName,
Input: input,
Dimensions: e.dimensions,
Model: e.modelName,
Input: input,
}
if e.supportsDimensionsParam() {
reqBody.Dimensions = e.dimensions
}
jsonData, err := json.Marshal(reqBody)
@@ -252,6 +259,10 @@ func (e *VolcengineEmbedder) GetModelName() string {
return e.modelName
}
func (e *VolcengineEmbedder) supportsDimensionsParam() bool {
return e.supportsDimensionOverride && e.dimensions > 0
}
// GetDimensions returns the vector dimensions
func (e *VolcengineEmbedder) GetDimensions() int {
return e.dimensions
+26 -17
View File
@@ -18,14 +18,15 @@ const weKnoraCloudEmbedPath = "/api/v1/embeddings"
// WeKnoraCloudEmbedder 实现 embedding.Embedder 接口,对接 WeKnoraCloud /api/v1/embeddings
type WeKnoraCloudEmbedder struct {
modelName string
remoteModelName string
modelID string
appID string
apiKey string
baseURL string
dimensions int
client *http.Client
modelName string
remoteModelName string
modelID string
appID string
apiKey string
baseURL string
dimensions int
supportsDimensionOverride bool
client *http.Client
EmbedderPooler
}
@@ -42,14 +43,15 @@ func NewWeKnoraCloudEmbedder(config Config) (*WeKnoraCloudEmbedder, error) {
remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"])
}
return &WeKnoraCloudEmbedder{
modelName: config.ModelName,
remoteModelName: remoteModelName,
modelID: config.ModelID,
appID: config.AppID,
apiKey: config.AppSecret,
baseURL: strings.TrimRight(config.BaseURL, "/"),
dimensions: config.Dimensions,
client: &http.Client{Timeout: 60 * time.Second},
modelName: config.ModelName,
remoteModelName: remoteModelName,
modelID: config.ModelID,
appID: config.AppID,
apiKey: config.AppSecret,
baseURL: strings.TrimRight(config.BaseURL, "/"),
dimensions: config.Dimensions,
supportsDimensionOverride: config.SupportsDimensionOverride,
client: &http.Client{Timeout: 60 * time.Second},
}, nil
}
@@ -79,7 +81,10 @@ func (e *WeKnoraCloudEmbedder) Embed(ctx context.Context, text string) ([]float3
}
func (e *WeKnoraCloudEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts, Dimensions: e.dimensions}
reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts}
if e.supportsDimensionOverride && e.dimensions > 0 {
reqBody.Dimensions = e.dimensions
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("weknoracloud embedder: marshal: %w", err)
@@ -129,6 +134,10 @@ func (e *WeKnoraCloudEmbedder) BatchEmbedWithPool(ctx context.Context, model Emb
return e.BatchEmbed(ctx, texts)
}
func (e *WeKnoraCloudEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
func (e *WeKnoraCloudEmbedder) effectiveModelName() string {
if e.remoteModelName != "" {
return e.remoteModelName
+22 -11
View File
@@ -16,16 +16,17 @@ import (
// ZhipuEmbedder implements text vectorization functionality using Zhipu AI API
type ZhipuEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
customHeaders map[string]string
supportsDimensionOverride bool
EmbedderPooler
}
@@ -88,6 +89,10 @@ func (e *ZhipuEmbedder) SetCustomHeaders(headers map[string]string) {
e.customHeaders = headers
}
func (e *ZhipuEmbedder) SetSupportsDimensionOverride(supported bool) {
e.supportsDimensionOverride = supported
}
// Embed converts text to vector
func (e *ZhipuEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
for range 3 {
@@ -149,9 +154,11 @@ func (e *ZhipuEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]flo
reqBody := ZhipuEmbedRequest{
Model: e.modelName,
Input: texts,
Dimensions: e.dimensions,
TruncatePromptTokens: e.truncatePromptTokens,
}
if e.supportsDimensionsParam() {
reqBody.Dimensions = e.dimensions
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
@@ -235,6 +242,10 @@ func (e *ZhipuEmbedder) GetModelName() string {
return e.modelName
}
func (e *ZhipuEmbedder) supportsDimensionsParam() bool {
return e.supportsDimensionOverride && e.dimensions > 0
}
// GetDimensions returns the vector dimensions
func (e *ZhipuEmbedder) GetDimensions() int {
return e.dimensions