feat: support NVIDIA model API

This commit is contained in:
Manx98
2026-03-10 12:32:48 +08:00
committed by lyingbug
parent b7ed54f8de
commit 5afc81d00b
14 changed files with 444 additions and 2 deletions
@@ -356,6 +356,18 @@ const fallbackProviderOptions = computed(() => [
description: t('model.editor.providers.jina.description'),
modelTypes: ['embedding', 'rerank']
},
{
value: 'nvidia',
label: t('model.editor.providers.nvidia.label'),
defaultUrls: {
chat: 'https://integrate.api.nvidia.com/v1',
embedding: 'https://integrate.api.nvidia.com/v1',
rerank: 'https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking',
vllm: 'https://integrate.api.nvidia.com/v1',
},
description: t('model.editor.providers.nvidia.description'),
modelTypes: ['chat', 'embedding', 'rerank', 'vllm']
},
{
value: 'generic',
label: t('model.editor.providers.generic.label'),
+4
View File
@@ -2129,6 +2129,10 @@ export default {
label: 'Tencent Cloud LKEAP',
description: 'DeepSeek-R1, DeepSeek-V3 series with chain-of-thought',
},
nvidia: {
label: "NVIDIA",
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
},
},
},
builtinTag: 'Built-in',
+4
View File
@@ -1611,6 +1611,10 @@ export default {
label: "텐센트 클라우드 LKEAP",
description: "DeepSeek-R1, DeepSeek-V3 시리즈, 사고 체인 지원",
},
nvidia: {
label: "NVIDIA",
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
},
},
},
builtinTag: '내장',
+5 -1
View File
@@ -1457,7 +1457,11 @@ export default {
lkeap: {
label: 'Tencent Cloud LKEAP',
description: 'DeepSeek-R1, DeepSeek-V3 с поддержкой цепочки рассуждений'
}
},
nvidia: {
label: "NVIDIA",
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
},
}
},
builtinTag: 'Built-in'
+4
View File
@@ -1588,6 +1588,10 @@ export default {
label: "腾讯云 LKEAP",
description: "DeepSeek-R1, DeepSeek-V3 系列模型,支持思维链",
},
nvidia: {
label: "NVIDIA",
description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
},
},
},
builtinTag: "内置",
@@ -660,7 +660,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
// Generate embedding vector for the query text
logger.Info(ctx, "Starting to generate query embedding")
queryEmbedding, err := embeddingModel.Embed(ctx, params.QueryText)
queryEmbedding, err := embeddingModel.Embed(context.WithValue(ctx, types.EmbedQueryContextKey, true), params.QueryText)
if err != nil {
logger.Errorf(ctx, "Failed to embed query text, query text: %s, error: %v", params.QueryText, err)
return nil, err
+8
View File
@@ -126,6 +126,14 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll
config.ModelID,
pooler)
return embedder, err
case provider.ProviderNvidia:
embedder, err = NewNvidiaEmbedder(config.APIKey,
config.BaseURL,
config.ModelName,
config.Dimensions,
config.ModelID,
pooler)
return embedder, err
default:
// Use OpenAI-compatible embedder for other providers
embedder, err = NewOpenAIEmbedder(config.APIKey,
+203
View File
@@ -0,0 +1,203 @@
package embedding
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
// NvidiaEmbedder implements text vectorization functionality using NVIDIA API
type NvidiaEmbedder struct {
apiKey string
baseURL string
modelName string
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
EmbedderPooler
}
// NvidiaEmbedRequest represents an NVIDIA embedding request
type NvidiaEmbedRequest struct {
Model string `json:"model"`
Input []string `json:"input"`
EncodingFormat string `json:"encoding_format,omitempty"`
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"`
InputType string `json:"input_type,omitempty"`
}
// NvidiaEmbedResponse represents an NVIDIA embedding response
type NvidiaEmbedResponse struct {
Data []struct {
Embedding []float32 `json:"embedding"`
Index int `json:"index"`
} `json:"data"`
}
// NewNvidiaEmbedder creates a new NVIDIA embedder
func NewNvidiaEmbedder(apiKey, baseURL, modelName string,
dimensions int, modelID string, pooler EmbedderPooler,
) (*NvidiaEmbedder, error) {
if baseURL == "" {
baseURL = "https://integrate.api.nvidia.com/v1"
}
if modelName == "" {
return nil, fmt.Errorf("model name is required")
}
timeout := 60 * time.Second
// Create HTTP client
client := &http.Client{
Timeout: timeout,
}
return &NvidiaEmbedder{
apiKey: apiKey,
baseURL: baseURL,
modelName: modelName,
httpClient: client,
EmbedderPooler: pooler,
dimensions: dimensions,
modelID: modelID,
timeout: timeout,
maxRetries: 3, // Maximum retry count
}, nil
}
// Embed converts text to vector
func (e *NvidiaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
for range 3 {
embeddings, err := e.BatchEmbed(ctx, []string{text})
if err != nil {
return nil, err
}
if len(embeddings) > 0 {
return embeddings[0], nil
}
}
return nil, fmt.Errorf("no embedding returned")
}
func (e *NvidiaEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) {
var resp *http.Response
var err error
url := e.baseURL + "/embeddings"
for i := 0; i <= e.maxRetries; i++ {
if i > 0 {
backoffTime := time.Duration(1<<uint(i-1)) * time.Second
if backoffTime > 10*time.Second {
backoffTime = 10 * time.Second
}
logger.GetLogger(ctx).
Infof("NvidiaEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
select {
case <-time.After(backoffTime):
case <-ctx.Done():
return nil, ctx.Err()
}
}
// Rebuild request each time to ensure Body is valid
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData))
if err != nil {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder failed to create request: %v", err)
continue
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+e.apiKey)
resp, err = e.httpClient.Do(req)
if err == nil {
return resp, nil
}
logger.GetLogger(ctx).Errorf("NvidiaEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err)
}
return nil, err
}
func (e *NvidiaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
// Create request body
reqBody := NvidiaEmbedRequest{
Model: e.modelName,
Input: texts,
EncodingFormat: "float",
InputType: "passage",
}
isQuery, _ := ctx.Value(types.EmbedQueryContextKey).(bool)
if isQuery {
reqBody.InputType = "query"
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch marshal request error: %v", err)
return nil, fmt.Errorf("marshal request: %w", err)
}
// Send request (passing jsonData instead of constructing http.Request)
resp, err := e.doRequestWithRetry(ctx, jsonData)
if err != nil {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch send request error: %v", err)
return nil, fmt.Errorf("send request: %w", err)
}
if resp.Body != nil {
defer resp.Body.Close()
}
// Read response
body, err := io.ReadAll(resp.Body)
if err != nil {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch read response error: %v", err)
return nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch API error: Http Status %s", resp.Status)
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
}
// Parse response
var response NvidiaEmbedResponse
if err := json.Unmarshal(body, &response); err != nil {
logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch unmarshal response error: %v", err)
return nil, fmt.Errorf("unmarshal response: %w", err)
}
// Extract embedding vectors
embeddings := make([][]float32, 0, len(response.Data))
for _, data := range response.Data {
embeddings = append(embeddings, data.Embedding)
}
return embeddings, nil
}
// GetModelName returns the model name
func (e *NvidiaEmbedder) GetModelName() string {
return e.modelName
}
// GetDimensions returns the vector dimensions
func (e *NvidiaEmbedder) GetDimensions() int {
return e.dimensions
}
// GetModelID returns the model ID
func (e *NvidiaEmbedder) GetModelID() string {
return e.modelID
}
+54
View File
@@ -0,0 +1,54 @@
package provider
import (
"fmt"
"github.com/Tencent/WeKnora/internal/types"
)
const (
// NvidiaChatBaseURL NVIDIA Chat 的默认 BaseURL
NvidiaChatBaseURL = "https://integrate.api.nvidia.com/v1"
// NvidiaRerankBaseURL NVIDIA Rerank 的默认 BaseURL
NvidiaRerankBaseURL = "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
)
// NvidiaProvider 实现NVIDIA AI 的 Provider 接口
type NvidiaProvider struct{}
func init() {
Register(&NvidiaProvider{})
}
// Info 返回NVIDIA provider 的元数据
func (p *NvidiaProvider) Info() ProviderInfo {
return ProviderInfo{
Name: ProviderNvidia,
DisplayName: "NVIDIA",
Description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.",
DefaultURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: NvidiaChatBaseURL,
types.ModelTypeEmbedding: NvidiaChatBaseURL,
types.ModelTypeRerank: NvidiaRerankBaseURL,
types.ModelTypeVLLM: NvidiaChatBaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
RequiresAuth: true,
}
}
// ValidateConfig 验证NVIDIA provider 配置
func (p *NvidiaProvider) ValidateConfig(config *Config) error {
if config.APIKey == "" {
return fmt.Errorf("API key is required for NVIDIA")
}
if config.ModelName == "" {
return fmt.Errorf("model name is required")
}
return nil
}
+5
View File
@@ -53,6 +53,8 @@ const (
ProviderLongCat ProviderName = "longcat"
// 腾讯云 LKEAP (知识引擎原子能力)
ProviderLKEAP ProviderName = "lkeap"
// NVIDIA
ProviderNvidia ProviderName = "nvidia"
)
// AllProviders 返回所有注册的提供者名称
@@ -78,6 +80,7 @@ func AllProviders() []ProviderName {
ProviderLongCat,
ProviderLKEAP,
ProviderGPUStack,
ProviderNvidia,
}
}
@@ -243,6 +246,8 @@ func DetectProvider(baseURL string) ProviderName {
return ProviderLongCat
case containsAny(baseURL, "lkeap.cloud.tencent.com", "api.lkeap"):
return ProviderLKEAP
case containsAny(baseURL, "nvidia.com"):
return ProviderNvidia
default:
return ProviderGeneric
}
@@ -48,6 +48,8 @@ func TestDetectProvider(t *testing.T) {
{"https://api.xiaomimimo.com/v1", ProviderMimo},
{"https://custom-endpoint.example.com/v1", ProviderGeneric},
{"http://localhost:11434/v1", ProviderGeneric},
{"https://integrate.api.nvidia.com/v1", ProviderNvidia},
{"https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", ProviderNvidia},
}
for _, tt := range tests {
+138
View File
@@ -0,0 +1,138 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/Tencent/WeKnora/internal/logger"
)
// NvidiaReranker implements a reranking system using Jina AI API
// Jina API uses different parameters than standard OpenAI-compatible APIs
type NvidiaReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
}
type NvidiaRerankDocument struct {
Text string `json:"text"`
}
// NvidiaRerankRequest represents a Jina rerank request
// Note: Jina does NOT support truncate_prompt_tokens parameter
type NvidiaRerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against
Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank
}
type NvidiaRankResult struct {
Index int `json:"index"`
RelevanceScore float64 `json:"logit"`
}
// NvidiaRerankResponse represents the response from a Jina reranking request
type NvidiaRerankResponse struct {
Model string `json:"model"` // Model used for reranking
Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores
}
// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration
func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) {
apiKey := config.APIKey
baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
if url := config.BaseURL; url != "" {
baseURL = url
}
return &NvidiaReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
},
},
}, nil
}
// Rerank performs document reranking based on relevance to the query
func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body - Jina does NOT use truncate_prompt_tokens
requestBody := &NvidiaRerankRequest{
Model: r.modelName,
Query: NvidiaRerankDocument{Text: query},
Documents: make([]NvidiaRerankDocument, len(documents)),
}
for i := range requestBody.Documents {
requestBody.Documents[i].Text = documents[i]
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
// Log the curl equivalent for debugging (API key masked for security)
logger.GetLogger(ctx).Infof(
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
r.baseURL, string(jsonData),
)
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
}
var response NvidiaRerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
ret := make([]RankResult, len(response.Results))
for i, result := range response.Results {
ret[i] = RankResult{
Index: result.Index,
Document: DocumentInfo{Text: documents[result.Index]},
RelevanceScore: result.RelevanceScore,
}
}
return ret, nil
}
// GetModelName returns the name of the reranking model
func (r *NvidiaReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *NvidiaReranker) GetModelID() string {
return r.modelID
}
+2
View File
@@ -101,6 +101,8 @@ func NewReranker(config *RerankerConfig) (Reranker, error) {
return NewZhipuReranker(config)
case provider.ProviderJina:
return NewJinaReranker(config)
case provider.ProviderNvidia:
return NewNvidiaReranker(config)
default:
return NewOpenAIReranker(config)
}
+2
View File
@@ -19,6 +19,8 @@ const (
// SessionTenantIDContextKey is the context key for session owner's tenant ID.
// When set (e.g. in pipeline with shared agent), session/message lookups use this instead of TenantIDContextKey.
SessionTenantIDContextKey ContextKey = "SessionTenantID"
// EmbedQueryContextKey is the context key for embedding query text
EmbedQueryContextKey ContextKey = "EmbedQuery"
)
// String returns the string representation of the context key