init commit
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
)
|
||||
|
||||
type batchEmbedder struct {
|
||||
pool *ants.Pool
|
||||
}
|
||||
|
||||
func NewBatchEmbedder(pool *ants.Pool) EmbedderPooler {
|
||||
return &batchEmbedder{pool: pool}
|
||||
}
|
||||
|
||||
type textEmbedding struct {
|
||||
text string
|
||||
results []float32
|
||||
}
|
||||
|
||||
func (e *batchEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) {
|
||||
// Create goroutine pool for concurrent processing of document chunks
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex // For synchronizing access to error
|
||||
var firstErr error // Record the first error that occurs
|
||||
batchSize := 5
|
||||
textEmbeddings := utils.MapSlice(texts, func(text string) *textEmbedding {
|
||||
return &textEmbedding{text: text}
|
||||
})
|
||||
|
||||
// Function to process each document chunk
|
||||
processChunk := func(texts []*textEmbedding) func() {
|
||||
return func() {
|
||||
defer wg.Done()
|
||||
// If an error has already occurred, don't continue processing
|
||||
if firstErr != nil {
|
||||
return
|
||||
}
|
||||
// Embed text
|
||||
embedding, err := model.BatchEmbed(ctx, utils.MapSlice(texts, func(text *textEmbedding) string {
|
||||
return text.text
|
||||
}))
|
||||
if err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
for i, text := range texts {
|
||||
text.results = embedding[i]
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Submit all tasks to the goroutine pool
|
||||
for _, texts := range utils.ChunkSlice(textEmbeddings, batchSize) {
|
||||
wg.Add(1)
|
||||
err := e.pool.Submit(processChunk(texts))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for all tasks to complete
|
||||
wg.Wait()
|
||||
|
||||
// Check if any errors occurred
|
||||
if firstErr != nil {
|
||||
return nil, firstErr
|
||||
}
|
||||
|
||||
results := utils.MapSlice(textEmbeddings, func(text *textEmbedding) []float32 {
|
||||
return text.results
|
||||
})
|
||||
return results, nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Embedder defines the interface for text vectorization
|
||||
type Embedder interface {
|
||||
// Embed converts text to vector
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
|
||||
// BatchEmbed converts multiple texts to vectors in batch
|
||||
BatchEmbed(ctx context.Context, texts []string) ([][]float32, error)
|
||||
|
||||
// GetModelName returns the model name
|
||||
GetModelName() string
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
GetDimensions() int
|
||||
|
||||
// GetModelID returns the model ID
|
||||
GetModelID() string
|
||||
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
type EmbedderPooler interface {
|
||||
BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error)
|
||||
}
|
||||
|
||||
// EmbedderType represents the embedder type
|
||||
type EmbedderType string
|
||||
|
||||
// Config represents the embedder configuration
|
||||
type Config struct {
|
||||
Source types.ModelSource `json:"source"`
|
||||
BaseURL string `json:"base_url"`
|
||||
ModelName string `json:"model_name"`
|
||||
APIKey string `json:"api_key"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
|
||||
Dimensions int `json:"dimensions"`
|
||||
ModelID string `json:"model_id"`
|
||||
}
|
||||
|
||||
// NewEmbedder creates an embedder based on the configuration
|
||||
func NewEmbedder(config Config) (Embedder, error) {
|
||||
var embedder Embedder
|
||||
var err error
|
||||
switch strings.ToLower(string(config.Source)) {
|
||||
case string(types.ModelSourceLocal):
|
||||
runtime.GetContainer().Invoke(func(pooler EmbedderPooler, ollamaService *ollama.OllamaService) {
|
||||
embedder, err = NewOllamaEmbedder(config.BaseURL,
|
||||
config.ModelName, config.TruncatePromptTokens, config.Dimensions, config.ModelID, pooler, ollamaService)
|
||||
})
|
||||
return embedder, err
|
||||
case string(types.ModelSourceRemote):
|
||||
runtime.GetContainer().Invoke(func(pooler EmbedderPooler) {
|
||||
embedder, err = NewOpenAIEmbedder(config.APIKey,
|
||||
config.BaseURL,
|
||||
config.ModelName,
|
||||
config.TruncatePromptTokens,
|
||||
config.Dimensions,
|
||||
config.ModelID,
|
||||
pooler)
|
||||
})
|
||||
return embedder, err
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported embedder source: %s", config.Source)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
ollamaapi "github.com/ollama/ollama/api"
|
||||
)
|
||||
|
||||
// OllamaEmbedder implements text vectorization functionality using Ollama
|
||||
type OllamaEmbedder struct {
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
ollamaService *ollama.OllamaService
|
||||
dimensions int
|
||||
modelID string
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
// OllamaEmbedRequest represents an Ollama embedding request
|
||||
type OllamaEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Prompt string `json:"prompt"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
|
||||
}
|
||||
|
||||
// OllamaEmbedResponse represents an Ollama embedding response
|
||||
type OllamaEmbedResponse struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
}
|
||||
|
||||
// NewOllamaEmbedder creates a new Ollama embedder
|
||||
func NewOllamaEmbedder(baseURL,
|
||||
modelName string,
|
||||
truncatePromptTokens int,
|
||||
dimensions int,
|
||||
modelID string,
|
||||
pooler EmbedderPooler,
|
||||
ollamaService *ollama.OllamaService,
|
||||
) (*OllamaEmbedder, error) {
|
||||
if modelName == "" {
|
||||
modelName = "nomic-embed-text"
|
||||
}
|
||||
|
||||
if truncatePromptTokens == 0 {
|
||||
truncatePromptTokens = 511
|
||||
}
|
||||
|
||||
return &OllamaEmbedder{
|
||||
modelName: modelName,
|
||||
truncatePromptTokens: truncatePromptTokens,
|
||||
ollamaService: ollamaService,
|
||||
EmbedderPooler: pooler,
|
||||
dimensions: dimensions,
|
||||
modelID: modelID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureModelAvailable ensures that the model is available
|
||||
func (e *OllamaEmbedder) ensureModelAvailable(ctx context.Context) error {
|
||||
logger.GetLogger(ctx).Infof("Ensuring model %s is available", e.modelName)
|
||||
return e.ollamaService.EnsureModelAvailable(ctx, e.modelName)
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *OllamaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
embedding, err := e.BatchEmbed(ctx, []string{text})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to embed text: %w", err)
|
||||
}
|
||||
|
||||
if len(embedding) == 0 {
|
||||
return nil, fmt.Errorf("failed to embed text: %w", err)
|
||||
}
|
||||
|
||||
return embedding[0], nil
|
||||
}
|
||||
|
||||
// BatchEmbed converts multiple texts to vectors in batch
|
||||
func (e *OllamaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
// Ensure model is available
|
||||
if err := e.ensureModelAvailable(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create request
|
||||
req := &ollamaapi.EmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
Options: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Set truncation parameters
|
||||
if e.truncatePromptTokens > 0 {
|
||||
req.Options["truncate"] = e.truncatePromptTokens
|
||||
}
|
||||
|
||||
// Send request
|
||||
startTime := time.Now()
|
||||
resp, err := e.ollamaService.Embeddings(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get embedding vectors: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Debugf("Embedding vector retrieval took: %v", time.Since(startTime))
|
||||
return resp.Embeddings, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the model name
|
||||
func (e *OllamaEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *OllamaEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
}
|
||||
|
||||
// GetModelID returns the model ID
|
||||
func (e *OllamaEmbedder) GetModelID() string {
|
||||
return e.modelID
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// OpenAIEmbedder implements text vectorization functionality using OpenAI API
|
||||
type OpenAIEmbedder struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
modelName string
|
||||
truncatePromptTokens int
|
||||
dimensions int
|
||||
modelID string
|
||||
httpClient *http.Client
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
EmbedderPooler
|
||||
}
|
||||
|
||||
// OpenAIEmbedRequest represents an OpenAI embedding request
|
||||
type OpenAIEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
|
||||
}
|
||||
|
||||
// OpenAIEmbedResponse represents an OpenAI embedding response
|
||||
type OpenAIEmbedResponse struct {
|
||||
Data []struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
Index int `json:"index"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// NewOpenAIEmbedder creates a new OpenAI embedder
|
||||
func NewOpenAIEmbedder(apiKey, baseURL, modelName string,
|
||||
truncatePromptTokens int, dimensions int, modelID string, pooler EmbedderPooler,
|
||||
) (*OpenAIEmbedder, error) {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://api.openai.com/v1"
|
||||
}
|
||||
|
||||
if modelName == "" {
|
||||
return nil, fmt.Errorf("model name is required")
|
||||
}
|
||||
|
||||
if truncatePromptTokens == 0 {
|
||||
truncatePromptTokens = 511
|
||||
}
|
||||
|
||||
timeout := 60 * time.Second
|
||||
|
||||
// Create HTTP client
|
||||
client := &http.Client{
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
return &OpenAIEmbedder{
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
modelName: modelName,
|
||||
httpClient: client,
|
||||
truncatePromptTokens: truncatePromptTokens,
|
||||
EmbedderPooler: pooler,
|
||||
dimensions: dimensions,
|
||||
modelID: modelID,
|
||||
timeout: timeout,
|
||||
maxRetries: 3, // Maximum retry count
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Embed converts text to vector
|
||||
func (e *OpenAIEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
for range 3 {
|
||||
embeddings, err := e.BatchEmbed(ctx, []string{text})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(embeddings) > 0 {
|
||||
return embeddings[0], nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no embedding returned")
|
||||
}
|
||||
|
||||
func (e *OpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) {
|
||||
var resp *http.Response
|
||||
var err error
|
||||
url := e.baseURL + "/embeddings"
|
||||
|
||||
for i := 0; i <= e.maxRetries; i++ {
|
||||
if i > 0 {
|
||||
backoffTime := time.Duration(1<<uint(i-1)) * time.Second
|
||||
if backoffTime > 10*time.Second {
|
||||
backoffTime = 10 * time.Second
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("OpenAIEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
|
||||
|
||||
select {
|
||||
case <-time.After(backoffTime):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild request each time to ensure Body is valid
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData))
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder failed to create request: %v", err)
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+e.apiKey)
|
||||
|
||||
resp, err = e.httpClient.Do(req)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err)
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
// Create request body
|
||||
reqBody := OpenAIEmbedRequest{
|
||||
Model: e.modelName,
|
||||
Input: texts,
|
||||
TruncatePromptTokens: e.truncatePromptTokens,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch marshal request error: %v", err)
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Send request (passing jsonData instead of constructing http.Request)
|
||||
resp, err := e.doRequestWithRetry(ctx, jsonData)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch send request error: %v", err)
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch read response error: %v", err)
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status)
|
||||
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response OpenAIEmbedResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch unmarshal response error: %v", err)
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
// Extract embedding vectors
|
||||
embeddings := make([][]float32, 0, len(response.Data))
|
||||
for _, data := range response.Data {
|
||||
embeddings = append(embeddings, data.Embedding)
|
||||
}
|
||||
|
||||
return embeddings, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the model name
|
||||
func (e *OpenAIEmbedder) GetModelName() string {
|
||||
return e.modelName
|
||||
}
|
||||
|
||||
// GetDimensions returns the vector dimensions
|
||||
func (e *OpenAIEmbedder) GetDimensions() int {
|
||||
return e.dimensions
|
||||
}
|
||||
|
||||
// GetModelID returns the model ID
|
||||
func (e *OpenAIEmbedder) GetModelID() string {
|
||||
return e.modelID
|
||||
}
|
||||
Reference in New Issue
Block a user