diff --git a/frontend/src/components/ModelEditorDialog.vue b/frontend/src/components/ModelEditorDialog.vue index 4012aae7..b1918d30 100644 --- a/frontend/src/components/ModelEditorDialog.vue +++ b/frontend/src/components/ModelEditorDialog.vue @@ -356,6 +356,18 @@ const fallbackProviderOptions = computed(() => [ description: t('model.editor.providers.jina.description'), modelTypes: ['embedding', 'rerank'] }, + { + value: 'nvidia', + label: t('model.editor.providers.nvidia.label'), + defaultUrls: { + chat: 'https://integrate.api.nvidia.com/v1', + embedding: 'https://integrate.api.nvidia.com/v1', + rerank: 'https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking', + vllm: 'https://integrate.api.nvidia.com/v1', + }, + description: t('model.editor.providers.nvidia.description'), + modelTypes: ['chat', 'embedding', 'rerank', 'vllm'] + }, { value: 'generic', label: t('model.editor.providers.generic.label'), diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 8ce2b4c7..cde8b640 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -2129,6 +2129,10 @@ export default { label: 'Tencent Cloud LKEAP', description: 'DeepSeek-R1, DeepSeek-V3 series with chain-of-thought', }, + nvidia: { + label: "NVIDIA", + description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.", + }, }, }, builtinTag: 'Built-in', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index d1948dfa..3397adcc 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1611,6 +1611,10 @@ export default { label: "텐센트 클라우드 LKEAP", description: "DeepSeek-R1, DeepSeek-V3 시리즈, 사고 체인 지원", }, + nvidia: { + label: "NVIDIA", + description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.", + }, }, }, builtinTag: '내장', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 0ac82bf8..c5e02379 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1457,7 +1457,11 @@ export default { lkeap: { label: 'Tencent Cloud LKEAP', description: 'DeepSeek-R1, DeepSeek-V3 с поддержкой цепочки рассуждений' - } + }, + nvidia: { + label: "NVIDIA", + description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.", + }, } }, builtinTag: 'Built-in' diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index a084635e..ba42f3cc 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1588,6 +1588,10 @@ export default { label: "腾讯云 LKEAP", description: "DeepSeek-R1, DeepSeek-V3 系列模型,支持思维链", }, + nvidia: { + label: "NVIDIA", + description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.", + }, }, }, builtinTag: "内置", diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index c5c9314b..41f4c029 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -660,7 +660,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, // Generate embedding vector for the query text logger.Info(ctx, "Starting to generate query embedding") - queryEmbedding, err := embeddingModel.Embed(ctx, params.QueryText) + queryEmbedding, err := embeddingModel.Embed(context.WithValue(ctx, types.EmbedQueryContextKey, true), params.QueryText) if err != nil { logger.Errorf(ctx, "Failed to embed query text, query text: %s, error: %v", params.QueryText, err) return nil, err diff --git a/internal/models/embedding/embedder.go b/internal/models/embedding/embedder.go index 470edae9..014c4d96 100644 --- a/internal/models/embedding/embedder.go +++ b/internal/models/embedding/embedder.go @@ -126,6 +126,14 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll config.ModelID, pooler) return embedder, err + case provider.ProviderNvidia: + embedder, err = NewNvidiaEmbedder(config.APIKey, + config.BaseURL, + config.ModelName, + config.Dimensions, + config.ModelID, + pooler) + return embedder, err default: // Use OpenAI-compatible embedder for other providers embedder, err = NewOpenAIEmbedder(config.APIKey, diff --git a/internal/models/embedding/nvidia.go b/internal/models/embedding/nvidia.go new file mode 100644 index 00000000..2c37dbbd --- /dev/null +++ b/internal/models/embedding/nvidia.go @@ -0,0 +1,203 @@ +package embedding + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// NvidiaEmbedder implements text vectorization functionality using NVIDIA API +type NvidiaEmbedder struct { + apiKey string + baseURL string + modelName string + dimensions int + modelID string + httpClient *http.Client + timeout time.Duration + maxRetries int + EmbedderPooler +} + +// NvidiaEmbedRequest represents an NVIDIA embedding request +type NvidiaEmbedRequest struct { + Model string `json:"model"` + Input []string `json:"input"` + EncodingFormat string `json:"encoding_format,omitempty"` + TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` + InputType string `json:"input_type,omitempty"` +} + +// NvidiaEmbedResponse represents an NVIDIA embedding response +type NvidiaEmbedResponse struct { + Data []struct { + Embedding []float32 `json:"embedding"` + Index int `json:"index"` + } `json:"data"` +} + +// NewNvidiaEmbedder creates a new NVIDIA embedder +func NewNvidiaEmbedder(apiKey, baseURL, modelName string, + dimensions int, modelID string, pooler EmbedderPooler, +) (*NvidiaEmbedder, error) { + if baseURL == "" { + baseURL = "https://integrate.api.nvidia.com/v1" + } + + if modelName == "" { + return nil, fmt.Errorf("model name is required") + } + + timeout := 60 * time.Second + + // Create HTTP client + client := &http.Client{ + Timeout: timeout, + } + + return &NvidiaEmbedder{ + apiKey: apiKey, + baseURL: baseURL, + modelName: modelName, + httpClient: client, + EmbedderPooler: pooler, + dimensions: dimensions, + modelID: modelID, + timeout: timeout, + maxRetries: 3, // Maximum retry count + }, nil +} + +// Embed converts text to vector +func (e *NvidiaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + for range 3 { + embeddings, err := e.BatchEmbed(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(embeddings) > 0 { + return embeddings[0], nil + } + } + return nil, fmt.Errorf("no embedding returned") +} + +func (e *NvidiaEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) { + var resp *http.Response + var err error + url := e.baseURL + "/embeddings" + + for i := 0; i <= e.maxRetries; i++ { + if i > 0 { + backoffTime := time.Duration(1< 10*time.Second { + backoffTime = 10 * time.Second + } + logger.GetLogger(ctx). + Infof("NvidiaEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime) + + select { + case <-time.After(backoffTime): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + // Rebuild request each time to ensure Body is valid + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData)) + if err != nil { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder failed to create request: %v", err) + continue + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+e.apiKey) + + resp, err = e.httpClient.Do(req) + if err == nil { + return resp, nil + } + + logger.GetLogger(ctx).Errorf("NvidiaEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err) + } + + return nil, err +} + +func (e *NvidiaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + // Create request body + reqBody := NvidiaEmbedRequest{ + Model: e.modelName, + Input: texts, + EncodingFormat: "float", + InputType: "passage", + } + isQuery, _ := ctx.Value(types.EmbedQueryContextKey).(bool) + if isQuery { + reqBody.InputType = "query" + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch marshal request error: %v", err) + return nil, fmt.Errorf("marshal request: %w", err) + } + + // Send request (passing jsonData instead of constructing http.Request) + resp, err := e.doRequestWithRetry(ctx, jsonData) + if err != nil { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch send request error: %v", err) + return nil, fmt.Errorf("send request: %w", err) + } + if resp.Body != nil { + defer resp.Body.Close() + } + + // Read response + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch read response error: %v", err) + return nil, fmt.Errorf("read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch API error: Http Status %s", resp.Status) + return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status) + } + + // Parse response + var response NvidiaEmbedResponse + if err := json.Unmarshal(body, &response); err != nil { + logger.GetLogger(ctx).Errorf("NvidiaEmbedder EmbedBatch unmarshal response error: %v", err) + return nil, fmt.Errorf("unmarshal response: %w", err) + } + + // Extract embedding vectors + embeddings := make([][]float32, 0, len(response.Data)) + for _, data := range response.Data { + embeddings = append(embeddings, data.Embedding) + } + + return embeddings, nil +} + +// GetModelName returns the model name +func (e *NvidiaEmbedder) GetModelName() string { + return e.modelName +} + +// GetDimensions returns the vector dimensions +func (e *NvidiaEmbedder) GetDimensions() int { + return e.dimensions +} + +// GetModelID returns the model ID +func (e *NvidiaEmbedder) GetModelID() string { + return e.modelID +} diff --git a/internal/models/provider/nvidia.go b/internal/models/provider/nvidia.go new file mode 100644 index 00000000..9fb87c0d --- /dev/null +++ b/internal/models/provider/nvidia.go @@ -0,0 +1,54 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + // NvidiaChatBaseURL NVIDIA Chat 的默认 BaseURL + NvidiaChatBaseURL = "https://integrate.api.nvidia.com/v1" + // NvidiaRerankBaseURL NVIDIA Rerank 的默认 BaseURL + NvidiaRerankBaseURL = "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking" +) + +// NvidiaProvider 实现NVIDIA AI 的 Provider 接口 +type NvidiaProvider struct{} + +func init() { + Register(&NvidiaProvider{}) +} + +// Info 返回NVIDIA provider 的元数据 +func (p *NvidiaProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderNvidia, + DisplayName: "NVIDIA", + Description: "deepseek-ai-deepseek-v3_1, nv-embed-v1, rerank-qa-mistral-4b, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: NvidiaChatBaseURL, + types.ModelTypeEmbedding: NvidiaChatBaseURL, + types.ModelTypeRerank: NvidiaRerankBaseURL, + types.ModelTypeVLLM: NvidiaChatBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeRerank, + types.ModelTypeVLLM, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证NVIDIA provider 配置 +func (p *NvidiaProvider) ValidateConfig(config *Config) error { + if config.APIKey == "" { + return fmt.Errorf("API key is required for NVIDIA") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/provider.go b/internal/models/provider/provider.go index aabfb8f6..6accdf1d 100644 --- a/internal/models/provider/provider.go +++ b/internal/models/provider/provider.go @@ -53,6 +53,8 @@ const ( ProviderLongCat ProviderName = "longcat" // 腾讯云 LKEAP (知识引擎原子能力) ProviderLKEAP ProviderName = "lkeap" + // NVIDIA + ProviderNvidia ProviderName = "nvidia" ) // AllProviders 返回所有注册的提供者名称 @@ -78,6 +80,7 @@ func AllProviders() []ProviderName { ProviderLongCat, ProviderLKEAP, ProviderGPUStack, + ProviderNvidia, } } @@ -243,6 +246,8 @@ func DetectProvider(baseURL string) ProviderName { return ProviderLongCat case containsAny(baseURL, "lkeap.cloud.tencent.com", "api.lkeap"): return ProviderLKEAP + case containsAny(baseURL, "nvidia.com"): + return ProviderNvidia default: return ProviderGeneric } diff --git a/internal/models/provider/provider_test.go b/internal/models/provider/provider_test.go index fde9cbce..e0b4a014 100644 --- a/internal/models/provider/provider_test.go +++ b/internal/models/provider/provider_test.go @@ -48,6 +48,8 @@ func TestDetectProvider(t *testing.T) { {"https://api.xiaomimimo.com/v1", ProviderMimo}, {"https://custom-endpoint.example.com/v1", ProviderGeneric}, {"http://localhost:11434/v1", ProviderGeneric}, + {"https://integrate.api.nvidia.com/v1", ProviderNvidia}, + {"https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", ProviderNvidia}, } for _, tt := range tests { diff --git a/internal/models/rerank/nvidia_reranker.go b/internal/models/rerank/nvidia_reranker.go new file mode 100644 index 00000000..13097ed8 --- /dev/null +++ b/internal/models/rerank/nvidia_reranker.go @@ -0,0 +1,138 @@ +package rerank + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/Tencent/WeKnora/internal/logger" +) + +// NvidiaReranker implements a reranking system using Jina AI API +// Jina API uses different parameters than standard OpenAI-compatible APIs +type NvidiaReranker struct { + modelName string // Name of the model used for reranking + modelID string // Unique identifier of the model + apiKey string // API key for authentication + baseURL string // Base URL for API requests + client *http.Client // HTTP client for making API requests +} +type NvidiaRerankDocument struct { + Text string `json:"text"` +} + +// NvidiaRerankRequest represents a Jina rerank request +// Note: Jina does NOT support truncate_prompt_tokens parameter +type NvidiaRerankRequest struct { + Model string `json:"model"` // Model to use for reranking + Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against + Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank +} + +type NvidiaRankResult struct { + Index int `json:"index"` + RelevanceScore float64 `json:"logit"` +} + +// NvidiaRerankResponse represents the response from a Jina reranking request +type NvidiaRerankResponse struct { + Model string `json:"model"` // Model used for reranking + Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores +} + +// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration +func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) { + apiKey := config.APIKey + baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking" + if url := config.BaseURL; url != "" { + baseURL = url + } + + return &NvidiaReranker{ + modelName: config.ModelName, + modelID: config.ModelID, + apiKey: apiKey, + baseURL: baseURL, + client: &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + }, + }, nil +} + +// Rerank performs document reranking based on relevance to the query +func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { + // Build the request body - Jina does NOT use truncate_prompt_tokens + requestBody := &NvidiaRerankRequest{ + Model: r.modelName, + Query: NvidiaRerankDocument{Text: query}, + Documents: make([]NvidiaRerankDocument, len(documents)), + } + for i := range requestBody.Documents { + requestBody.Documents[i].Text = documents[i] + } + + jsonData, err := json.Marshal(requestBody) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + + // Send the request + req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) + + // Log the curl equivalent for debugging (API key masked for security) + logger.GetLogger(ctx).Infof( + "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", + r.baseURL, string(jsonData), + ) + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + // Read the response + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body)) + return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status) + } + + var response NvidiaRerankResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + ret := make([]RankResult, len(response.Results)) + for i, result := range response.Results { + ret[i] = RankResult{ + Index: result.Index, + Document: DocumentInfo{Text: documents[result.Index]}, + RelevanceScore: result.RelevanceScore, + } + } + return ret, nil +} + +// GetModelName returns the name of the reranking model +func (r *NvidiaReranker) GetModelName() string { + return r.modelName +} + +// GetModelID returns the unique identifier of the reranking model +func (r *NvidiaReranker) GetModelID() string { + return r.modelID +} diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go index d9e7d321..5ed45dba 100644 --- a/internal/models/rerank/reranker.go +++ b/internal/models/rerank/reranker.go @@ -101,6 +101,8 @@ func NewReranker(config *RerankerConfig) (Reranker, error) { return NewZhipuReranker(config) case provider.ProviderJina: return NewJinaReranker(config) + case provider.ProviderNvidia: + return NewNvidiaReranker(config) default: return NewOpenAIReranker(config) } diff --git a/internal/types/const.go b/internal/types/const.go index db586915..dcbe343e 100644 --- a/internal/types/const.go +++ b/internal/types/const.go @@ -19,6 +19,8 @@ const ( // SessionTenantIDContextKey is the context key for session owner's tenant ID. // When set (e.g. in pipeline with shared agent), session/message lookups use this instead of TenantIDContextKey. SessionTenantIDContextKey ContextKey = "SessionTenantID" + // EmbedQueryContextKey is the context key for embedding query text + EmbedQueryContextKey ContextKey = "EmbedQuery" ) // String returns the string representation of the context key