391 lines
14 KiB
Go
391 lines
14 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/Tencent/WeKnora/internal/common"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"github.com/pgvector/pgvector-go"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
)
|
|
|
|
// pgRepository implements PostgreSQL-based retrieval operations
|
|
type pgRepository struct {
|
|
db *gorm.DB // Database connection
|
|
}
|
|
|
|
// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository
|
|
func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
|
|
logger.GetLogger(context.Background()).Info("[Postgres] Initializing PostgreSQL retriever engine repository")
|
|
return &pgRepository{db: db}
|
|
}
|
|
|
|
// EngineType returns the retriever engine type (PostgreSQL)
|
|
func (r *pgRepository) EngineType() types.RetrieverEngineType {
|
|
return types.PostgresRetrieverEngineType
|
|
}
|
|
|
|
// Support returns supported retriever types (keywords and vector)
|
|
func (r *pgRepository) Support() []types.RetrieverType {
|
|
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
|
}
|
|
|
|
// calculateIndexStorageSize calculates storage size for a single index entry
|
|
func (g *pgRepository) calculateIndexStorageSize(embeddingDB *pgVector) int64 {
|
|
// 1. Text content size
|
|
contentSizeBytes := int64(len(embeddingDB.Content))
|
|
|
|
// 2. Vector storage size (2 bytes per dimension for half-precision float)
|
|
var vectorSizeBytes int64 = 0
|
|
if embeddingDB.Dimension > 0 {
|
|
vectorSizeBytes = int64(embeddingDB.Dimension * 2)
|
|
}
|
|
|
|
// 3. Metadata size (fixed overhead for IDs, timestamps etc.)
|
|
metadataSizeBytes := int64(200)
|
|
|
|
// 4. Index overhead (HNSW index is ~2x vector size)
|
|
indexOverheadBytes := vectorSizeBytes * 2
|
|
|
|
// Total size in bytes
|
|
totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes
|
|
|
|
return totalSizeBytes
|
|
}
|
|
|
|
// EstimateStorageSize estimates total storage size for multiple indices
|
|
func (g *pgRepository) EstimateStorageSize(
|
|
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
|
|
) int64 {
|
|
var totalStorageSize int64 = 0
|
|
for _, indexInfo := range indexInfoList {
|
|
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
|
|
totalStorageSize += g.calculateIndexStorageSize(embeddingDB)
|
|
}
|
|
logger.GetLogger(ctx).Infof(
|
|
"[Postgres] Estimated storage size for %d indices: %d bytes",
|
|
len(indexInfoList), totalStorageSize,
|
|
)
|
|
return totalStorageSize
|
|
}
|
|
|
|
// Save stores a single index entry
|
|
func (g *pgRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, additionalParams map[string]any) error {
|
|
logger.GetLogger(ctx).Debugf("[Postgres] Saving index for source ID: %s", indexInfo.SourceID)
|
|
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
|
|
err := g.db.WithContext(ctx).Create(embeddingDB).Error
|
|
if err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Failed to save index: %v", err)
|
|
return err
|
|
}
|
|
logger.GetLogger(ctx).Infof("[Postgres] Successfully saved index for source ID: %s", indexInfo.SourceID)
|
|
return nil
|
|
}
|
|
|
|
// BatchSave stores multiple index entries in batch
|
|
func (g *pgRepository) BatchSave(
|
|
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
|
|
) error {
|
|
logger.GetLogger(ctx).Infof("[Postgres] Batch saving %d indices", len(indexInfoList))
|
|
indexInfoDBList := make([]*pgVector, len(indexInfoList))
|
|
for i := range indexInfoList {
|
|
indexInfoDBList[i] = toDBVectorEmbedding(indexInfoList[i], additionalParams)
|
|
}
|
|
err := g.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(indexInfoDBList).Error
|
|
if err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Batch save failed: %v", err)
|
|
return err
|
|
}
|
|
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch saved %d indices", len(indexInfoList))
|
|
return nil
|
|
}
|
|
|
|
// DeleteByChunkIDList deletes indices by chunk IDs
|
|
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
|
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
|
|
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
|
|
if result.Error != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by chunk IDs: %v", result.Error)
|
|
return result.Error
|
|
}
|
|
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by chunk IDs", result.RowsAffected)
|
|
return nil
|
|
}
|
|
|
|
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
|
|
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error {
|
|
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
|
|
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
|
|
if result.Error != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by knowledge IDs: %v", result.Error)
|
|
return result.Error
|
|
}
|
|
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by knowledge IDs", result.RowsAffected)
|
|
return nil
|
|
}
|
|
|
|
// Retrieve handles retrieval requests and routes to appropriate method
|
|
func (g *pgRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
|
logger.GetLogger(ctx).Debugf("[Postgres] Processing retrieval request of type: %s", params.RetrieverType)
|
|
switch params.RetrieverType {
|
|
case types.KeywordsRetrieverType:
|
|
return g.KeywordsRetrieve(ctx, params)
|
|
case types.VectorRetrieverType:
|
|
return g.VectorRetrieve(ctx, params)
|
|
}
|
|
err := errors.New("invalid retriever type")
|
|
logger.GetLogger(ctx).Errorf("[Postgres] %v: %s", err, params.RetrieverType)
|
|
return nil, err
|
|
}
|
|
|
|
// KeywordsRetrieve performs keyword-based search using PostgreSQL full-text search
|
|
func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
|
params types.RetrieveParams,
|
|
) ([]*types.RetrieveResult, error) {
|
|
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
|
|
conds := make([]clause.Expression, 0)
|
|
if len(params.KnowledgeBaseIDs) > 0 {
|
|
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
|
|
conds = append(conds, clause.Expr{
|
|
SQL: fmt.Sprintf("knowledge_base_id @@@ 'in (%s)'", common.StringSliceJoin(params.KnowledgeBaseIDs)),
|
|
})
|
|
}
|
|
conds = append(conds, clause.Expr{
|
|
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
|
|
Vars: []interface{}{params.Query},
|
|
})
|
|
conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{
|
|
{Column: clause.Column{Name: "score"}, Desc: true},
|
|
}})
|
|
|
|
var embeddingDBList []pgVectorWithScore
|
|
err := g.db.WithContext(ctx).Clauses(conds...).Debug().
|
|
Select([]string{
|
|
"paradedb.score(id) as score",
|
|
"id",
|
|
"content",
|
|
"source_id",
|
|
"source_type",
|
|
"chunk_id",
|
|
"knowledge_id",
|
|
"knowledge_base_id",
|
|
}).
|
|
Limit(int(params.TopK)).
|
|
Find(&embeddingDBList).Error
|
|
|
|
if err == gorm.ErrRecordNotFound {
|
|
logger.GetLogger(ctx).Warnf("[Postgres] No records found for keywords query: %s", params.Query)
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Keywords retrieval failed: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList))
|
|
results := make([]*types.IndexWithScore, len(embeddingDBList))
|
|
for i := range embeddingDBList {
|
|
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords)
|
|
logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
|
|
i, results[i].ChunkID, results[i].Score)
|
|
}
|
|
return []*types.RetrieveResult{
|
|
{
|
|
Results: results,
|
|
RetrieverEngineType: types.PostgresRetrieverEngineType,
|
|
RetrieverType: types.KeywordsRetrieverType,
|
|
Error: nil,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// VectorRetrieve performs vector similarity search using pgvector
|
|
func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
|
params types.RetrieveParams,
|
|
) ([]*types.RetrieveResult, error) {
|
|
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
|
|
len(params.Embedding), params.TopK, params.Threshold)
|
|
|
|
conds := make([]clause.Expression, 0)
|
|
if len(params.KnowledgeBaseIDs) > 0 {
|
|
logger.GetLogger(ctx).Debugf(
|
|
"[Postgres] Filtering vector search by knowledge base IDs: %v",
|
|
params.KnowledgeBaseIDs,
|
|
)
|
|
conds = append(conds, clause.IN{
|
|
Column: "knowledge_base_id",
|
|
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
|
|
})
|
|
}
|
|
// <=> Cosine similarity operator
|
|
// <-> L2 distance operator
|
|
// <#> Inner product operator
|
|
dimension := len(params.Embedding)
|
|
conds = append(conds, clause.Expr{SQL: "dimension = ?", Vars: []interface{}{dimension}})
|
|
conds = append(conds, clause.Expr{
|
|
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec < ?", dimension),
|
|
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding), 1 - params.Threshold},
|
|
})
|
|
conds = append(conds, clause.OrderBy{Expression: clause.Expr{
|
|
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec", dimension),
|
|
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding)},
|
|
}})
|
|
|
|
var embeddingDBList []pgVectorWithScore
|
|
|
|
err := g.db.WithContext(ctx).Clauses(conds...).
|
|
Select(fmt.Sprintf(
|
|
"id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, "+
|
|
"(1 - (embedding::halfvec(%d) <=> ?::halfvec)) as score",
|
|
dimension,
|
|
), pgvector.NewHalfVector(params.Embedding)).
|
|
Limit(int(params.TopK)).
|
|
Find(&embeddingDBList).Error
|
|
|
|
if err == gorm.ErrRecordNotFound {
|
|
logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold)
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Vector retrieval failed: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
|
|
results := make([]*types.IndexWithScore, len(embeddingDBList))
|
|
for i := range embeddingDBList {
|
|
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding)
|
|
logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
|
|
i, results[i].ChunkID, results[i].Score)
|
|
}
|
|
return []*types.RetrieveResult{
|
|
{
|
|
Results: results,
|
|
RetrieverEngineType: types.PostgresRetrieverEngineType,
|
|
RetrieverType: types.VectorRetrieverType,
|
|
Error: nil,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// CopyIndices copies index data
|
|
func (g *pgRepository) CopyIndices(ctx context.Context,
|
|
sourceKnowledgeBaseID string,
|
|
sourceToTargetKBIDMap map[string]string,
|
|
sourceToTargetChunkIDMap map[string]string,
|
|
targetKnowledgeBaseID string,
|
|
dimension int,
|
|
) error {
|
|
logger.GetLogger(ctx).Infof(
|
|
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
|
|
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
|
|
)
|
|
|
|
if len(sourceToTargetChunkIDMap) == 0 {
|
|
logger.GetLogger(ctx).Warnf("[Postgres] Mapping is empty, no need to copy")
|
|
return nil
|
|
}
|
|
|
|
// Batch processing parameters
|
|
batchSize := 500 // Number of records to process per batch
|
|
offset := 0 // Offset for pagination
|
|
totalCopied := 0 // Total number of copied records
|
|
|
|
for {
|
|
// Paginated query for source data
|
|
var sourceVectors []*pgVector
|
|
if err := g.db.WithContext(ctx).
|
|
Where("knowledge_base_id = ?", sourceKnowledgeBaseID).
|
|
Limit(batchSize).
|
|
Offset(offset).
|
|
Find(&sourceVectors).Error; err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Failed to query source index data: %v", err)
|
|
return err
|
|
}
|
|
|
|
// If no more data, exit the loop
|
|
if len(sourceVectors) == 0 {
|
|
if offset == 0 {
|
|
logger.GetLogger(ctx).Warnf("[Postgres] No source index data found")
|
|
}
|
|
break
|
|
}
|
|
|
|
batchCount := len(sourceVectors)
|
|
logger.GetLogger(ctx).Infof(
|
|
"[Postgres] Found %d source index data, batch start position: %d",
|
|
batchCount, offset,
|
|
)
|
|
|
|
// Create target vector index
|
|
targetVectors := make([]*pgVector, 0, batchCount)
|
|
for _, sourceVector := range sourceVectors {
|
|
// Get the mapped target chunk ID
|
|
targetChunkID, ok := sourceToTargetChunkIDMap[sourceVector.ChunkID]
|
|
if !ok {
|
|
logger.GetLogger(ctx).Warnf(
|
|
"[Postgres] Source chunk %s not found in target chunk mapping, skipping",
|
|
sourceVector.ChunkID,
|
|
)
|
|
continue
|
|
}
|
|
|
|
// Get the mapped target knowledge ID
|
|
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceVector.KnowledgeID]
|
|
if !ok {
|
|
logger.GetLogger(ctx).Warnf(
|
|
"[Postgres] Source knowledge %s not found in target knowledge mapping, skipping",
|
|
sourceVector.KnowledgeID,
|
|
)
|
|
continue
|
|
}
|
|
|
|
// Create new vector index, copy the content and vector of the source index
|
|
targetVector := &pgVector{
|
|
Content: sourceVector.Content,
|
|
SourceID: targetChunkID, // Update to target chunk ID
|
|
SourceType: sourceVector.SourceType,
|
|
ChunkID: targetChunkID, // Update to target chunk ID
|
|
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
|
|
KnowledgeBaseID: targetKnowledgeBaseID, // Update to target knowledge base ID
|
|
Dimension: sourceVector.Dimension,
|
|
Embedding: sourceVector.Embedding, // Copy the vector embedding directly, avoid recalculation
|
|
}
|
|
|
|
targetVectors = append(targetVectors, targetVector)
|
|
}
|
|
|
|
// Batch insert target vector index
|
|
if len(targetVectors) > 0 {
|
|
if err := g.db.WithContext(ctx).
|
|
Clauses(clause.OnConflict{DoNothing: true}).Create(targetVectors).Error; err != nil {
|
|
logger.GetLogger(ctx).Errorf("[Postgres] Failed to batch create target index: %v", err)
|
|
return err
|
|
}
|
|
|
|
totalCopied += len(targetVectors)
|
|
logger.GetLogger(ctx).Infof(
|
|
"[Postgres] Successfully copied batch data, batch size: %d, total copied: %d",
|
|
len(targetVectors),
|
|
totalCopied,
|
|
)
|
|
}
|
|
|
|
// Move to the next batch
|
|
offset += batchCount
|
|
|
|
// If the number of returned records is less than the requested size, it means the last page has been reached
|
|
if batchCount < batchSize {
|
|
break
|
|
}
|
|
}
|
|
|
|
logger.GetLogger(ctx).Infof("[Postgres] Index copying completed, total copied: %d", totalCopied)
|
|
return nil
|
|
}
|