feat: support NVIDIA model API
This commit is contained in:
@@ -356,6 +356,18 @@ const fallbackProviderOptions = computed(() => [
|
||||
description: t('model.editor.providers.jina.description'),
|
||||
modelTypes: ['embedding', 'rerank']
|
||||
},
|
||||
{
|
||||
value: 'nvidia',
|
||||
label: t('model.editor.providers.nvidia.label'),
|
||||
defaultUrls: {
|
||||
chat: 'https://integrate.api.nvidia.com/v1',
|
||||
embedding: 'https://integrate.api.nvidia.com/v1',
|
||||
rerank: 'https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking',
|
||||
vllm: 'https://integrate.api.nvidia.com/v1',
|
||||
},
|
||||
description: t('model.editor.providers.nvidia.description'),
|
||||
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
|
||||
},
|
||||
{
|
||||
value: 'generic',
|
||||
label: t('model.editor.providers.generic.label'),
|
||||
|
||||
@@ -2129,6 +2129,10 @@ export default {
|
||||
label: 'Tencent Cloud LKEAP',
|
||||
description: 'DeepSeek-R1, DeepSeek-V3 series with chain-of-thought',
|
||||
},
|
||||
nvidia: {
|
||||
label: "NVIDIA",
|
||||
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
|
||||
},
|
||||
},
|
||||
},
|
||||
builtinTag: 'Built-in',
|
||||
|
||||
@@ -1611,6 +1611,10 @@ export default {
|
||||
label: "텐센트 클라우드 LKEAP",
|
||||
description: "DeepSeek-R1, DeepSeek-V3 시리즈, 사고 체인 지원",
|
||||
},
|
||||
nvidia: {
|
||||
label: "NVIDIA",
|
||||
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
|
||||
},
|
||||
},
|
||||
},
|
||||
builtinTag: '내장',
|
||||
|
||||
@@ -1457,7 +1457,11 @@ export default {
|
||||
lkeap: {
|
||||
label: 'Tencent Cloud LKEAP',
|
||||
description: 'DeepSeek-R1, DeepSeek-V3 с поддержкой цепочки рассуждений'
|
||||
}
|
||||
},
|
||||
nvidia: {
|
||||
label: "NVIDIA",
|
||||
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
|
||||
},
|
||||
}
|
||||
},
|
||||
builtinTag: 'Built-in'
|
||||
|
||||
@@ -1588,6 +1588,10 @@ export default {
|
||||
label: "腾讯云 LKEAP",
|
||||
description: "DeepSeek-R1, DeepSeek-V3 系列模型,支持思维链",
|
||||
},
|
||||
nvidia: {
|
||||
label: "NVIDIA",
|
||||
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
|
||||
},
|
||||
},
|
||||
},
|
||||
builtinTag: "内置",
|
||||
|
||||
@@ -660,7 +660,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
// Generate embedding vector for the query text
|
||||
logger.Info(ctx, "Starting to generate query embedding")
|
||||
queryEmbedding, err := embeddingModel.Embed(ctx, params.QueryText)
|
||||
queryEmbedding, err := embeddingModel.Embed(context.WithValue(ctx, types.EmbedQueryContextKey, true), params.QueryText)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to embed query text, query text: %s, error: %v", params.QueryText, err)
|
||||
return nil, err
|
||||
|
||||
@@ -126,6 +126,14 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
|
||||
config.ModelID,
|
||||
pooler)
|
||||
return embedder, err
|
||||
case provider.ProviderNvidia:
|
||||
embedder, err = NewNvidiaEmbedder(config.APIKey,
|
||||
config.BaseURL,
|
||||
config.ModelName,
|
||||
config.Dimensions,
|
||||
config.ModelID,
|
||||
pooler)
|
||||
return embedder, err
|
||||
default:
|
||||
// Use OpenAI-compatible embedder for other providers
|
||||
embedder, err = NewOpenAIEmbedder(config.APIKey,
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// NvidiaEmbedder implements text vectorization functionality using NVIDIA API
|
||||
type NvidiaEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
// NvidiaEmbedRequest represents an NVIDIA embedding request
|
||||
type NvidiaEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
EncodingFormat string `json:"encoding_format,omitempty"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"`
|
||||
InputType string `json:"input_type,omitempty"`
|
||||
}
|
||||
|
||||
// NvidiaEmbedResponse represents an NVIDIA embedding response
|
||||
type NvidiaEmbedResponse struct {
|
||||
Data []struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
Index int `json:"index"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// NewNvidiaEmbedder creates a new NVIDIA embedder
|
||||
func NewNvidiaEmbedder(apiKey, baseURL, modelName string,
|
||||
dimensions int, modelID string, pooler EmbedderPooler,
|
||||
) (*NvidiaEmbedder, error) {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://integrate.api.nvidia.com/v1"
|
||||
}
|
||||
|
||||
if modelName == "" {
|
||||
return nil, fmt.Errorf("model name is required")
|
||||
}
|
||||
|
||||
timeout := 60 * time.Second
|
||||
|
||||
// Create HTTP client
|
||||
client := &http.Client{
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
return &NvidiaEmbedder{
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
modelName: modelName,
|
||||
httpClient: client,
|
||||
EmbedderPooler: pooler,
|
||||
dimensions: dimensions,
|
||||
modelID: modelID,
|
||||
timeout: timeout,
|
||||
maxRetries: 3, // Maximum retry count
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *NvidiaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
for range 3 {
|
||||
embeddings, err := e.BatchEmbed(ctx, []string{text})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(embeddings) > 0 {
|
||||
return embeddings[0], nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no embedding returned")
|
||||
}
|
||||
|
||||
func (e *NvidiaEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) {
|
||||
var resp *http.Response
|
||||
var err error
|
||||
url := e.baseURL + "/embeddings"
|
||||
|
||||
for i := 0; i <= e.maxRetries; i++ {
|
||||
if i > 0 {
|
||||
backoffTime := time.Duration(1<<uint(i-1)) * time.Second
|
||||
if backoffTime > 10*time.Second {
|
||||
backoffTime = 10 * time.Second
|
||||
}
|
||||
logger.GetLogger(ctx).
|
||||
Infof("NvidiaEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
|
||||
|
||||
select {
|
||||
case <-time.After(backoffTime):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild request each time to ensure Body is valid
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData))
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder failed to create request: %v", err)
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+e.apiKey)
|
||||
|
||||
resp, err = e.httpClient.Do(req)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err)
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (e *NvidiaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
// Create request body
|
||||
reqBody := NvidiaEmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
EncodingFormat: "float",
|
||||
InputType: "passage",
|
||||
}
|
||||
isQuery, _ := ctx.Value(types.EmbedQueryContextKey).(bool)
|
||||
if isQuery {
|
||||
reqBody.InputType = "query"
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch marshal request error: %v", err)
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Send request (passing jsonData instead of constructing http.Request)
|
||||
resp, err := e.doRequestWithRetry(ctx, jsonData)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch send request error: %v", err)
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
}
|
||||
if resp.Body != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
|
||||
// Read response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch read response error: %v", err)
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch API error: Http Status %s", resp.Status)
|
||||
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response NvidiaEmbedResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch unmarshal response error: %v", err)
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
// Extract embedding vectors
|
||||
embeddings := make([][]float32, 0, len(response.Data))
|
||||
for _, data := range response.Data {
|
||||
embeddings = append(embeddings, data.Embedding)
|
||||
}
|
||||
|
||||
return embeddings, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the model name
|
||||
func (e *NvidiaEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *NvidiaEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
}
|
||||
|
||||
// GetModelID returns the model ID
|
||||
func (e *NvidiaEmbedder) GetModelID() string {
|
||||
return e.modelID
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
// NvidiaChatBaseURL NVIDIA Chat 的默认 BaseURL
|
||||
NvidiaChatBaseURL = "https://integrate.api.nvidia.com/v1"
|
||||
// NvidiaRerankBaseURL NVIDIA Rerank 的默认 BaseURL
|
||||
NvidiaRerankBaseURL = "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
|
||||
)
|
||||
|
||||
// NvidiaProvider 实现NVIDIA AI 的 Provider 接口
|
||||
type NvidiaProvider struct{}
|
||||
|
||||
func init() {
|
||||
Register(&NvidiaProvider{})
|
||||
}
|
||||
|
||||
// Info 返回NVIDIA provider 的元数据
|
||||
func (p *NvidiaProvider) Info() ProviderInfo {
|
||||
return ProviderInfo{
|
||||
Name: ProviderNvidia,
|
||||
DisplayName: "NVIDIA",
|
||||
Description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
|
||||
DefaultURLs: map[types.ModelType]string{
|
||||
types.ModelTypeKnowledgeQA: NvidiaChatBaseURL,
|
||||
types.ModelTypeEmbedding: NvidiaChatBaseURL,
|
||||
types.ModelTypeRerank: NvidiaRerankBaseURL,
|
||||
types.ModelTypeVLLM: NvidiaChatBaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
RequiresAuth: true,
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateConfig 验证NVIDIA provider 配置
|
||||
func (p *NvidiaProvider) ValidateConfig(config *Config) error {
|
||||
if config.APIKey == "" {
|
||||
return fmt.Errorf("API key is required for NVIDIA")
|
||||
}
|
||||
if config.ModelName == "" {
|
||||
return fmt.Errorf("model name is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -53,6 +53,8 @@ const (
|
||||
ProviderLongCat ProviderName = "longcat"
|
||||
// 腾讯云 LKEAP (知识引擎原子能力)
|
||||
ProviderLKEAP ProviderName = "lkeap"
|
||||
// NVIDIA
|
||||
ProviderNvidia ProviderName = "nvidia"
|
||||
)
|
||||
|
||||
// AllProviders 返回所有注册的提供者名称
|
||||
@@ -78,6 +80,7 @@ func AllProviders() []ProviderName {
|
||||
ProviderLongCat,
|
||||
ProviderLKEAP,
|
||||
ProviderGPUStack,
|
||||
ProviderNvidia,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,6 +246,8 @@ func DetectProvider(baseURL string) ProviderName {
|
||||
return ProviderLongCat
|
||||
case containsAny(baseURL, "lkeap.cloud.tencent.com", "api.lkeap"):
|
||||
return ProviderLKEAP
|
||||
case containsAny(baseURL, "nvidia.com"):
|
||||
return ProviderNvidia
|
||||
default:
|
||||
return ProviderGeneric
|
||||
}
|
||||
|
||||
@@ -48,6 +48,8 @@ func TestDetectProvider(t *testing.T) {
|
||||
{"https://api.xiaomimimo.com/v1", ProviderMimo},
|
||||
{"https://custom-endpoint.example.com/v1", ProviderGeneric},
|
||||
{"http://localhost:11434/v1", ProviderGeneric},
|
||||
{"https://integrate.api.nvidia.com/v1", ProviderNvidia},
|
||||
{"https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", ProviderNvidia},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// NvidiaReranker implements a reranking system using Jina AI API
|
||||
// Jina API uses different parameters than standard OpenAI-compatible APIs
|
||||
type NvidiaReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
}
|
||||
type NvidiaRerankDocument struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// NvidiaRerankRequest represents a Jina rerank request
|
||||
// Note: Jina does NOT support truncate_prompt_tokens parameter
|
||||
type NvidiaRerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against
|
||||
Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank
|
||||
}
|
||||
|
||||
type NvidiaRankResult struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"logit"`
|
||||
}
|
||||
|
||||
// NvidiaRerankResponse represents the response from a Jina reranking request
|
||||
type NvidiaRerankResponse struct {
|
||||
Model string `json:"model"` // Model used for reranking
|
||||
Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores
|
||||
}
|
||||
|
||||
// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration
|
||||
func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
|
||||
return &NvidiaReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query
|
||||
func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body - Jina does NOT use truncate_prompt_tokens
|
||||
requestBody := &NvidiaRerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: NvidiaRerankDocument{Text: query},
|
||||
Documents: make([]NvidiaRerankDocument, len(documents)),
|
||||
}
|
||||
for i := range requestBody.Documents {
|
||||
requestBody.Documents[i].Text = documents[i]
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
|
||||
// Log the curl equivalent for debugging (API key masked for security)
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
|
||||
r.baseURL, string(jsonData),
|
||||
)
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
|
||||
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
|
||||
}
|
||||
|
||||
var response NvidiaRerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
ret := make([]RankResult, len(response.Results))
|
||||
for i, result := range response.Results {
|
||||
ret[i] = RankResult{
|
||||
Index: result.Index,
|
||||
Document: DocumentInfo{Text: documents[result.Index]},
|
||||
RelevanceScore: result.RelevanceScore,
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *NvidiaReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *NvidiaReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
@@ -101,6 +101,8 @@ func NewReranker(config *RerankerConfig) (Reranker, error) {
|
||||
return NewZhipuReranker(config)
|
||||
case provider.ProviderJina:
|
||||
return NewJinaReranker(config)
|
||||
case provider.ProviderNvidia:
|
||||
return NewNvidiaReranker(config)
|
||||
default:
|
||||
return NewOpenAIReranker(config)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,8 @@ const (
|
||||
// SessionTenantIDContextKey is the context key for session owner's tenant ID.
|
||||
// When set (e.g. in pipeline with shared agent), session/message lookups use this instead of TenantIDContextKey.
|
||||
SessionTenantIDContextKey ContextKey = "SessionTenantID"
|
||||
// EmbedQueryContextKey is the context key for embedding query text
|
||||
EmbedQueryContextKey ContextKey = "EmbedQuery"
|
||||
)
|
||||
|
||||
// String returns the string representation of the context key
|
||||
|
||||
Reference in New Issue
Block a user