init commit

This commit is contained in:
wizardchen
2025-08-05 15:08:07 +08:00
commit 56eb2bce33
282 changed files with 49077 additions and 0 deletions
+83
View File
@@ -0,0 +1,83 @@
package embedding
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/panjf2000/ants/v2"
)
type batchEmbedder struct {
pool *ants.Pool
}
func NewBatchEmbedder(pool *ants.Pool) EmbedderPooler {
return &batchEmbedder{pool: pool}
}
type textEmbedding struct {
text string
results []float32
}
func (e *batchEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) {
// Create goroutine pool for concurrent processing of document chunks
var wg sync.WaitGroup
var mu sync.Mutex // For synchronizing access to error
var firstErr error // Record the first error that occurs
batchSize := 5
textEmbeddings := utils.MapSlice(texts, func(text string) *textEmbedding {
return &textEmbedding{text: text}
})
// Function to process each document chunk
processChunk := func(texts []*textEmbedding) func() {
return func() {
defer wg.Done()
// If an error has already occurred, don't continue processing
if firstErr != nil {
return
}
// Embed text
embedding, err := model.BatchEmbed(ctx, utils.MapSlice(texts, func(text *textEmbedding) string {
return text.text
}))
if err != nil {
mu.Lock()
if firstErr == nil {
firstErr = err
}
mu.Unlock()
return
}
mu.Lock()
for i, text := range texts {
text.results = embedding[i]
}
mu.Unlock()
}
}
// Submit all tasks to the goroutine pool
for _, texts := range utils.ChunkSlice(textEmbeddings, batchSize) {
wg.Add(1)
err := e.pool.Submit(processChunk(texts))
if err != nil {
return nil, err
}
}
// Wait for all tasks to complete
wg.Wait()
// Check if any errors occurred
if firstErr != nil {
return nil, firstErr
}
results := utils.MapSlice(textEmbeddings, func(text *textEmbedding) []float32 {
return text.results
})
return results, nil
}
+76
View File
@@ -0,0 +1,76 @@
package embedding
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/types"
)
// Embedder defines the interface for text vectorization
type Embedder interface {
// Embed converts text to vector
Embed(ctx context.Context, text string) ([]float32, error)
// BatchEmbed converts multiple texts to vectors in batch
BatchEmbed(ctx context.Context, texts []string) ([][]float32, error)
// GetModelName returns the model name
GetModelName() string
// GetDimensions returns the vector dimensions
GetDimensions() int
// GetModelID returns the model ID
GetModelID() string
EmbedderPooler
}
type EmbedderPooler interface {
BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error)
}
// EmbedderType represents the embedder type
type EmbedderType string
// Config represents the embedder configuration
type Config struct {
Source types.ModelSource `json:"source"`
BaseURL string `json:"base_url"`
ModelName string `json:"model_name"`
APIKey string `json:"api_key"`
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
Dimensions int `json:"dimensions"`
ModelID string `json:"model_id"`
}
// NewEmbedder creates an embedder based on the configuration
func NewEmbedder(config Config) (Embedder, error) {
var embedder Embedder
var err error
switch strings.ToLower(string(config.Source)) {
case string(types.ModelSourceLocal):
runtime.GetContainer().Invoke(func(pooler EmbedderPooler, ollamaService *ollama.OllamaService) {
embedder, err = NewOllamaEmbedder(config.BaseURL,
config.ModelName, config.TruncatePromptTokens, config.Dimensions, config.ModelID, pooler, ollamaService)
})
return embedder, err
case string(types.ModelSourceRemote):
runtime.GetContainer().Invoke(func(pooler EmbedderPooler) {
embedder, err = NewOpenAIEmbedder(config.APIKey,
config.BaseURL,
config.ModelName,
config.TruncatePromptTokens,
config.Dimensions,
config.ModelID,
pooler)
})
return embedder, err
default:
return nil, fmt.Errorf("unsupported embedder source: %s", config.Source)
}
}
+125
View File
@@ -0,0 +1,125 @@
package embedding
import (
"context"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
ollamaapi "github.com/ollama/ollama/api"
)
// OllamaEmbedder implements text vectorization functionality using Ollama
type OllamaEmbedder struct {
modelName string
truncatePromptTokens int
ollamaService *ollama.OllamaService
dimensions int
modelID string
EmbedderPooler
}
// OllamaEmbedRequest represents an Ollama embedding request
type OllamaEmbedRequest struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
}
// OllamaEmbedResponse represents an Ollama embedding response
type OllamaEmbedResponse struct {
Embedding []float32 `json:"embedding"`
}
// NewOllamaEmbedder creates a new Ollama embedder
func NewOllamaEmbedder(baseURL,
modelName string,
truncatePromptTokens int,
dimensions int,
modelID string,
pooler EmbedderPooler,
ollamaService *ollama.OllamaService,
) (*OllamaEmbedder, error) {
if modelName == "" {
modelName = "nomic-embed-text"
}
if truncatePromptTokens == 0 {
truncatePromptTokens = 511
}
return &OllamaEmbedder{
modelName: modelName,
truncatePromptTokens: truncatePromptTokens,
ollamaService: ollamaService,
EmbedderPooler: pooler,
dimensions: dimensions,
modelID: modelID,
}, nil
}
// ensureModelAvailable ensures that the model is available
func (e *OllamaEmbedder) ensureModelAvailable(ctx context.Context) error {
logger.GetLogger(ctx).Infof("Ensuring model %s is available", e.modelName)
return e.ollamaService.EnsureModelAvailable(ctx, e.modelName)
}
// Embed converts text to vector
func (e *OllamaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
embedding, err := e.BatchEmbed(ctx, []string{text})
if err != nil {
return nil, fmt.Errorf("failed to embed text: %w", err)
}
if len(embedding) == 0 {
return nil, fmt.Errorf("failed to embed text: %w", err)
}
return embedding[0], nil
}
// BatchEmbed converts multiple texts to vectors in batch
func (e *OllamaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
// Ensure model is available
if err := e.ensureModelAvailable(ctx); err != nil {
return nil, err
}
// Create request
req := &ollamaapi.EmbedRequest{
Model: e.modelName,
Input: texts,
Options: make(map[string]interface{}),
}
// Set truncation parameters
if e.truncatePromptTokens > 0 {
req.Options["truncate"] = e.truncatePromptTokens
}
// Send request
startTime := time.Now()
resp, err := e.ollamaService.Embeddings(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to get embedding vectors: %w", err)
}
logger.GetLogger(ctx).Debugf("Embedding vector retrieval took: %v", time.Since(startTime))
return resp.Embeddings, nil
}
// GetModelName returns the model name
func (e *OllamaEmbedder) GetModelName() string {
return e.modelName
}
// GetDimensions returns the vector dimensions
func (e *OllamaEmbedder) GetDimensions() int {
return e.dimensions
}
// GetModelID returns the model ID
func (e *OllamaEmbedder) GetModelID() string {
return e.modelID
}
+198
View File
@@ -0,0 +1,198 @@
package embedding
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/Tencent/WeKnora/internal/logger"
)
// OpenAIEmbedder implements text vectorization functionality using OpenAI API
type OpenAIEmbedder struct {
apiKey string
baseURL string
modelName string
truncatePromptTokens int
dimensions int
modelID string
httpClient *http.Client
timeout time.Duration
maxRetries int
EmbedderPooler
}
// OpenAIEmbedRequest represents an OpenAI embedding request
type OpenAIEmbedRequest struct {
Model string `json:"model"`
Input []string `json:"input"`
TruncatePromptTokens int `json:"truncate_prompt_tokens"`
}
// OpenAIEmbedResponse represents an OpenAI embedding response
type OpenAIEmbedResponse struct {
Data []struct {
Embedding []float32 `json:"embedding"`
Index int `json:"index"`
} `json:"data"`
}
// NewOpenAIEmbedder creates a new OpenAI embedder
func NewOpenAIEmbedder(apiKey, baseURL, modelName string,
truncatePromptTokens int, dimensions int, modelID string, pooler EmbedderPooler,
) (*OpenAIEmbedder, error) {
if baseURL == "" {
baseURL = "https://api.openai.com/v1"
}
if modelName == "" {
return nil, fmt.Errorf("model name is required")
}
if truncatePromptTokens == 0 {
truncatePromptTokens = 511
}
timeout := 60 * time.Second
// Create HTTP client
client := &http.Client{
Timeout: timeout,
}
return &OpenAIEmbedder{
apiKey: apiKey,
baseURL: baseURL,
modelName: modelName,
httpClient: client,
truncatePromptTokens: truncatePromptTokens,
EmbedderPooler: pooler,
dimensions: dimensions,
modelID: modelID,
timeout: timeout,
maxRetries: 3, // Maximum retry count
}, nil
}
// Embed converts text to vector
func (e *OpenAIEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
for range 3 {
embeddings, err := e.BatchEmbed(ctx, []string{text})
if err != nil {
return nil, err
}
if len(embeddings) > 0 {
return embeddings[0], nil
}
}
return nil, fmt.Errorf("no embedding returned")
}
func (e *OpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) {
var resp *http.Response
var err error
url := e.baseURL + "/embeddings"
for i := 0; i <= e.maxRetries; i++ {
if i > 0 {
backoffTime := time.Duration(1<<uint(i-1)) * time.Second
if backoffTime > 10*time.Second {
backoffTime = 10 * time.Second
}
logger.GetLogger(ctx).Infof("OpenAIEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
select {
case <-time.After(backoffTime):
case <-ctx.Done():
return nil, ctx.Err()
}
}
// Rebuild request each time to ensure Body is valid
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData))
if err != nil {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder failed to create request: %v", err)
continue
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+e.apiKey)
resp, err = e.httpClient.Do(req)
if err == nil {
return resp, nil
}
logger.GetLogger(ctx).Errorf("OpenAIEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err)
}
return nil, err
}
func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) {
// Create request body
reqBody := OpenAIEmbedRequest{
Model: e.modelName,
Input: texts,
TruncatePromptTokens: e.truncatePromptTokens,
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch marshal request error: %v", err)
return nil, fmt.Errorf("marshal request: %w", err)
}
// Send request (passing jsonData instead of constructing http.Request)
resp, err := e.doRequestWithRetry(ctx, jsonData)
if err != nil {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch send request error: %v", err)
return nil, fmt.Errorf("send request: %w", err)
}
defer resp.Body.Close()
// Read response
body, err := io.ReadAll(resp.Body)
if err != nil {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch read response error: %v", err)
return nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status)
return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status)
}
// Parse response
var response OpenAIEmbedResponse
if err := json.Unmarshal(body, &response); err != nil {
logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch unmarshal response error: %v", err)
return nil, fmt.Errorf("unmarshal response: %w", err)
}
// Extract embedding vectors
embeddings := make([][]float32, 0, len(response.Data))
for _, data := range response.Data {
embeddings = append(embeddings, data.Embedding)
}
return embeddings, nil
}
// GetModelName returns the model name
func (e *OpenAIEmbedder) GetModelName() string {
return e.modelName
}
// GetDimensions returns the vector dimensions
func (e *OpenAIEmbedder) GetDimensions() int {
return e.dimensions
}
// GetModelID returns the model ID
func (e *OpenAIEmbedder) GetModelID() string {
return e.modelID
}