Files
Enterprise-Knowledge-Base/internal/application/repository/retriever/postgres/repository.go
T
wizardchen 56eb2bce33 init commit
2025-08-05 15:08:07 +08:00

391 lines
14 KiB
Go

package postgres
import (
"context"
"errors"
"fmt"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/pgvector/pgvector-go"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// pgRepository implements PostgreSQL-based retrieval operations
type pgRepository struct {
db *gorm.DB // Database connection
}
// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository
func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
logger.GetLogger(context.Background()).Info("[Postgres] Initializing PostgreSQL retriever engine repository")
return &pgRepository{db: db}
}
// EngineType returns the retriever engine type (PostgreSQL)
func (r *pgRepository) EngineType() types.RetrieverEngineType {
return types.PostgresRetrieverEngineType
}
// Support returns supported retriever types (keywords and vector)
func (r *pgRepository) Support() []types.RetrieverType {
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
// calculateIndexStorageSize calculates storage size for a single index entry
func (g *pgRepository) calculateIndexStorageSize(embeddingDB *pgVector) int64 {
// 1. Text content size
contentSizeBytes := int64(len(embeddingDB.Content))
// 2. Vector storage size (2 bytes per dimension for half-precision float)
var vectorSizeBytes int64 = 0
if embeddingDB.Dimension > 0 {
vectorSizeBytes = int64(embeddingDB.Dimension * 2)
}
// 3. Metadata size (fixed overhead for IDs, timestamps etc.)
metadataSizeBytes := int64(200)
// 4. Index overhead (HNSW index is ~2x vector size)
indexOverheadBytes := vectorSizeBytes * 2
// Total size in bytes
totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes
return totalSizeBytes
}
// EstimateStorageSize estimates total storage size for multiple indices
func (g *pgRepository) EstimateStorageSize(
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
) int64 {
var totalStorageSize int64 = 0
for _, indexInfo := range indexInfoList {
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
totalStorageSize += g.calculateIndexStorageSize(embeddingDB)
}
logger.GetLogger(ctx).Infof(
"[Postgres] Estimated storage size for %d indices: %d bytes",
len(indexInfoList), totalStorageSize,
)
return totalStorageSize
}
// Save stores a single index entry
func (g *pgRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, additionalParams map[string]any) error {
logger.GetLogger(ctx).Debugf("[Postgres] Saving index for source ID: %s", indexInfo.SourceID)
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
err := g.db.WithContext(ctx).Create(embeddingDB).Error
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to save index: %v", err)
return err
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully saved index for source ID: %s", indexInfo.SourceID)
return nil
}
// BatchSave stores multiple index entries in batch
func (g *pgRepository) BatchSave(
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
) error {
logger.GetLogger(ctx).Infof("[Postgres] Batch saving %d indices", len(indexInfoList))
indexInfoDBList := make([]*pgVector, len(indexInfoList))
for i := range indexInfoList {
indexInfoDBList[i] = toDBVectorEmbedding(indexInfoList[i], additionalParams)
}
err := g.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(indexInfoDBList).Error
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Batch save failed: %v", err)
return err
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch saved %d indices", len(indexInfoList))
return nil
}
// DeleteByChunkIDList deletes indices by chunk IDs
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by chunk IDs: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by chunk IDs", result.RowsAffected)
return nil
}
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by knowledge IDs: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by knowledge IDs", result.RowsAffected)
return nil
}
// Retrieve handles retrieval requests and routes to appropriate method
func (g *pgRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Debugf("[Postgres] Processing retrieval request of type: %s", params.RetrieverType)
switch params.RetrieverType {
case types.KeywordsRetrieverType:
return g.KeywordsRetrieve(ctx, params)
case types.VectorRetrieverType:
return g.VectorRetrieve(ctx, params)
}
err := errors.New("invalid retriever type")
logger.GetLogger(ctx).Errorf("[Postgres] %v: %s", err, params.RetrieverType)
return nil, err
}
// KeywordsRetrieve performs keyword-based search using PostgreSQL full-text search
func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
conds := make([]clause.Expression, 0)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
conds = append(conds, clause.Expr{
SQL: fmt.Sprintf("knowledge_base_id @@@ 'in (%s)'", common.StringSliceJoin(params.KnowledgeBaseIDs)),
})
}
conds = append(conds, clause.Expr{
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
Vars: []interface{}{params.Query},
})
conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{
{Column: clause.Column{Name: "score"}, Desc: true},
}})
var embeddingDBList []pgVectorWithScore
err := g.db.WithContext(ctx).Clauses(conds...).Debug().
Select([]string{
"paradedb.score(id) as score",
"id",
"content",
"source_id",
"source_type",
"chunk_id",
"knowledge_id",
"knowledge_base_id",
}).
Limit(int(params.TopK)).
Find(&embeddingDBList).Error
if err == gorm.ErrRecordNotFound {
logger.GetLogger(ctx).Warnf("[Postgres] No records found for keywords query: %s", params.Query)
return nil, nil
}
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Keywords retrieval failed: %v", err)
return nil, err
}
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords)
logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
i, results[i].ChunkID, results[i].Score)
}
return []*types.RetrieveResult{
{
Results: results,
RetrieverEngineType: types.PostgresRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
Error: nil,
},
}, nil
}
// VectorRetrieve performs vector similarity search using pgvector
func (g *pgRepository) VectorRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
len(params.Embedding), params.TopK, params.Threshold)
conds := make([]clause.Expression, 0)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge base IDs: %v",
params.KnowledgeBaseIDs,
)
conds = append(conds, clause.IN{
Column: "knowledge_base_id",
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
})
}
// <=> Cosine similarity operator
// <-> L2 distance operator
// <#> Inner product operator
dimension := len(params.Embedding)
conds = append(conds, clause.Expr{SQL: "dimension = ?", Vars: []interface{}{dimension}})
conds = append(conds, clause.Expr{
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec < ?", dimension),
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding), 1 - params.Threshold},
})
conds = append(conds, clause.OrderBy{Expression: clause.Expr{
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec", dimension),
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding)},
}})
var embeddingDBList []pgVectorWithScore
err := g.db.WithContext(ctx).Clauses(conds...).
Select(fmt.Sprintf(
"id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, "+
"(1 - (embedding::halfvec(%d) <=> ?::halfvec)) as score",
dimension,
), pgvector.NewHalfVector(params.Embedding)).
Limit(int(params.TopK)).
Find(&embeddingDBList).Error
if err == gorm.ErrRecordNotFound {
logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold)
return nil, nil
}
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Vector retrieval failed: %v", err)
return nil, err
}
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding)
logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
i, results[i].ChunkID, results[i].Score)
}
return []*types.RetrieveResult{
{
Results: results,
RetrieverEngineType: types.PostgresRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
Error: nil,
},
}, nil
}
// CopyIndices copies index data
func (g *pgRepository) CopyIndices(ctx context.Context,
sourceKnowledgeBaseID string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
) error {
logger.GetLogger(ctx).Infof(
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
)
if len(sourceToTargetChunkIDMap) == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] Mapping is empty, no need to copy")
return nil
}
// Batch processing parameters
batchSize := 500 // Number of records to process per batch
offset := 0 // Offset for pagination
totalCopied := 0 // Total number of copied records
for {
// Paginated query for source data
var sourceVectors []*pgVector
if err := g.db.WithContext(ctx).
Where("knowledge_base_id = ?", sourceKnowledgeBaseID).
Limit(batchSize).
Offset(offset).
Find(&sourceVectors).Error; err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to query source index data: %v", err)
return err
}
// If no more data, exit the loop
if len(sourceVectors) == 0 {
if offset == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] No source index data found")
}
break
}
batchCount := len(sourceVectors)
logger.GetLogger(ctx).Infof(
"[Postgres] Found %d source index data, batch start position: %d",
batchCount, offset,
)
// Create target vector index
targetVectors := make([]*pgVector, 0, batchCount)
for _, sourceVector := range sourceVectors {
// Get the mapped target chunk ID
targetChunkID, ok := sourceToTargetChunkIDMap[sourceVector.ChunkID]
if !ok {
logger.GetLogger(ctx).Warnf(
"[Postgres] Source chunk %s not found in target chunk mapping, skipping",
sourceVector.ChunkID,
)
continue
}
// Get the mapped target knowledge ID
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceVector.KnowledgeID]
if !ok {
logger.GetLogger(ctx).Warnf(
"[Postgres] Source knowledge %s not found in target knowledge mapping, skipping",
sourceVector.KnowledgeID,
)
continue
}
// Create new vector index, copy the content and vector of the source index
targetVector := &pgVector{
Content: sourceVector.Content,
SourceID: targetChunkID, // Update to target chunk ID
SourceType: sourceVector.SourceType,
ChunkID: targetChunkID, // Update to target chunk ID
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
KnowledgeBaseID: targetKnowledgeBaseID, // Update to target knowledge base ID
Dimension: sourceVector.Dimension,
Embedding: sourceVector.Embedding, // Copy the vector embedding directly, avoid recalculation
}
targetVectors = append(targetVectors, targetVector)
}
// Batch insert target vector index
if len(targetVectors) > 0 {
if err := g.db.WithContext(ctx).
Clauses(clause.OnConflict{DoNothing: true}).Create(targetVectors).Error; err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to batch create target index: %v", err)
return err
}
totalCopied += len(targetVectors)
logger.GetLogger(ctx).Infof(
"[Postgres] Successfully copied batch data, batch size: %d, total copied: %d",
len(targetVectors),
totalCopied,
)
}
// Move to the next batch
offset += batchCount
// If the number of returned records is less than the requested size, it means the last page has been reached
if batchCount < batchSize {
break
}
}
logger.GetLogger(ctx).Infof("[Postgres] Index copying completed, total copied: %d", totalCopied)
return nil
}