package postgres import ( "context" "errors" "fmt" "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/pgvector/pgvector-go" "gorm.io/gorm" "gorm.io/gorm/clause" ) // pgRepository implements PostgreSQL-based retrieval operations type pgRepository struct { db *gorm.DB // Database connection } // NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository { logger.GetLogger(context.Background()).Info("[Postgres] Initializing PostgreSQL retriever engine repository") return &pgRepository{db: db} } // EngineType returns the retriever engine type (PostgreSQL) func (r *pgRepository) EngineType() types.RetrieverEngineType { return types.PostgresRetrieverEngineType } // Support returns supported retriever types (keywords and vector) func (r *pgRepository) Support() []types.RetrieverType { return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType} } // calculateIndexStorageSize calculates storage size for a single index entry func (g *pgRepository) calculateIndexStorageSize(embeddingDB *pgVector) int64 { // 1. Text content size contentSizeBytes := int64(len(embeddingDB.Content)) // 2. Vector storage size (2 bytes per dimension for half-precision float) var vectorSizeBytes int64 = 0 if embeddingDB.Dimension > 0 { vectorSizeBytes = int64(embeddingDB.Dimension * 2) } // 3. Metadata size (fixed overhead for IDs, timestamps etc.) metadataSizeBytes := int64(200) // 4. Index overhead (HNSW index is ~2x vector size) indexOverheadBytes := vectorSizeBytes * 2 // Total size in bytes totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes return totalSizeBytes } // EstimateStorageSize estimates total storage size for multiple indices func (g *pgRepository) EstimateStorageSize( ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any, ) int64 { var totalStorageSize int64 = 0 for _, indexInfo := range indexInfoList { embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams) totalStorageSize += g.calculateIndexStorageSize(embeddingDB) } logger.GetLogger(ctx).Infof( "[Postgres] Estimated storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize, ) return totalStorageSize } // Save stores a single index entry func (g *pgRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, additionalParams map[string]any) error { logger.GetLogger(ctx).Debugf("[Postgres] Saving index for source ID: %s", indexInfo.SourceID) embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams) err := g.db.WithContext(ctx).Create(embeddingDB).Error if err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Failed to save index: %v", err) return err } logger.GetLogger(ctx).Infof("[Postgres] Successfully saved index for source ID: %s", indexInfo.SourceID) return nil } // BatchSave stores multiple index entries in batch func (g *pgRepository) BatchSave( ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any, ) error { logger.GetLogger(ctx).Infof("[Postgres] Batch saving %d indices", len(indexInfoList)) indexInfoDBList := make([]*pgVector, len(indexInfoList)) for i := range indexInfoList { indexInfoDBList[i] = toDBVectorEmbedding(indexInfoList[i], additionalParams) } err := g.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(indexInfoDBList).Error if err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Batch save failed: %v", err) return err } logger.GetLogger(ctx).Infof("[Postgres] Successfully batch saved %d indices", len(indexInfoList)) return nil } // DeleteByChunkIDList deletes indices by chunk IDs func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{}) if result.Error != nil { logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by chunk IDs: %v", result.Error) return result.Error } logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by chunk IDs", result.RowsAffected) return nil } // DeleteByKnowledgeIDList deletes indices by knowledge IDs func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error { logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{}) if result.Error != nil { logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by knowledge IDs: %v", result.Error) return result.Error } logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by knowledge IDs", result.RowsAffected) return nil } // Retrieve handles retrieval requests and routes to appropriate method func (g *pgRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) { logger.GetLogger(ctx).Debugf("[Postgres] Processing retrieval request of type: %s", params.RetrieverType) switch params.RetrieverType { case types.KeywordsRetrieverType: return g.KeywordsRetrieve(ctx, params) case types.VectorRetrieverType: return g.VectorRetrieve(ctx, params) } err := errors.New("invalid retriever type") logger.GetLogger(ctx).Errorf("[Postgres] %v: %s", err, params.RetrieverType) return nil, err } // KeywordsRetrieve performs keyword-based search using PostgreSQL full-text search func (g *pgRepository) KeywordsRetrieve(ctx context.Context, params types.RetrieveParams, ) ([]*types.RetrieveResult, error) { logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK) conds := make([]clause.Expression, 0) if len(params.KnowledgeBaseIDs) > 0 { logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs) conds = append(conds, clause.Expr{ SQL: fmt.Sprintf("knowledge_base_id @@@ 'in (%s)'", common.StringSliceJoin(params.KnowledgeBaseIDs)), }) } conds = append(conds, clause.Expr{ SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)", Vars: []interface{}{params.Query}, }) conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{ {Column: clause.Column{Name: "score"}, Desc: true}, }}) var embeddingDBList []pgVectorWithScore err := g.db.WithContext(ctx).Clauses(conds...).Debug(). Select([]string{ "paradedb.score(id) as score", "id", "content", "source_id", "source_type", "chunk_id", "knowledge_id", "knowledge_base_id", }). Limit(int(params.TopK)). Find(&embeddingDBList).Error if err == gorm.ErrRecordNotFound { logger.GetLogger(ctx).Warnf("[Postgres] No records found for keywords query: %s", params.Query) return nil, nil } if err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Keywords retrieval failed: %v", err) return nil, err } logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords) logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", i, results[i].ChunkID, results[i].Score) } return []*types.RetrieveResult{ { Results: results, RetrieverEngineType: types.PostgresRetrieverEngineType, RetrieverType: types.KeywordsRetrieverType, Error: nil, }, }, nil } // VectorRetrieve performs vector similarity search using pgvector func (g *pgRepository) VectorRetrieve(ctx context.Context, params types.RetrieveParams, ) ([]*types.RetrieveResult, error) { logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", len(params.Embedding), params.TopK, params.Threshold) conds := make([]clause.Expression, 0) if len(params.KnowledgeBaseIDs) > 0 { logger.GetLogger(ctx).Debugf( "[Postgres] Filtering vector search by knowledge base IDs: %v", params.KnowledgeBaseIDs, ) conds = append(conds, clause.IN{ Column: "knowledge_base_id", Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs), }) } // <=> Cosine similarity operator // <-> L2 distance operator // <#> Inner product operator dimension := len(params.Embedding) conds = append(conds, clause.Expr{SQL: "dimension = ?", Vars: []interface{}{dimension}}) conds = append(conds, clause.Expr{ SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec < ?", dimension), Vars: []interface{}{pgvector.NewHalfVector(params.Embedding), 1 - params.Threshold}, }) conds = append(conds, clause.OrderBy{Expression: clause.Expr{ SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec", dimension), Vars: []interface{}{pgvector.NewHalfVector(params.Embedding)}, }}) var embeddingDBList []pgVectorWithScore err := g.db.WithContext(ctx).Clauses(conds...). Select(fmt.Sprintf( "id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, "+ "(1 - (embedding::halfvec(%d) <=> ?::halfvec)) as score", dimension, ), pgvector.NewHalfVector(params.Embedding)). Limit(int(params.TopK)). Find(&embeddingDBList).Error if err == gorm.ErrRecordNotFound { logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold) return nil, nil } if err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Vector retrieval failed: %v", err) return nil, err } logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList)) results := make([]*types.IndexWithScore, len(embeddingDBList)) for i := range embeddingDBList { results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding) logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", i, results[i].ChunkID, results[i].Score) } return []*types.RetrieveResult{ { Results: results, RetrieverEngineType: types.PostgresRetrieverEngineType, RetrieverType: types.VectorRetrieverType, Error: nil, }, }, nil } // CopyIndices copies index data func (g *pgRepository) CopyIndices(ctx context.Context, sourceKnowledgeBaseID string, sourceToTargetKBIDMap map[string]string, sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, ) error { logger.GetLogger(ctx).Infof( "[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d", sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), ) if len(sourceToTargetChunkIDMap) == 0 { logger.GetLogger(ctx).Warnf("[Postgres] Mapping is empty, no need to copy") return nil } // Batch processing parameters batchSize := 500 // Number of records to process per batch offset := 0 // Offset for pagination totalCopied := 0 // Total number of copied records for { // Paginated query for source data var sourceVectors []*pgVector if err := g.db.WithContext(ctx). Where("knowledge_base_id = ?", sourceKnowledgeBaseID). Limit(batchSize). Offset(offset). Find(&sourceVectors).Error; err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Failed to query source index data: %v", err) return err } // If no more data, exit the loop if len(sourceVectors) == 0 { if offset == 0 { logger.GetLogger(ctx).Warnf("[Postgres] No source index data found") } break } batchCount := len(sourceVectors) logger.GetLogger(ctx).Infof( "[Postgres] Found %d source index data, batch start position: %d", batchCount, offset, ) // Create target vector index targetVectors := make([]*pgVector, 0, batchCount) for _, sourceVector := range sourceVectors { // Get the mapped target chunk ID targetChunkID, ok := sourceToTargetChunkIDMap[sourceVector.ChunkID] if !ok { logger.GetLogger(ctx).Warnf( "[Postgres] Source chunk %s not found in target chunk mapping, skipping", sourceVector.ChunkID, ) continue } // Get the mapped target knowledge ID targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceVector.KnowledgeID] if !ok { logger.GetLogger(ctx).Warnf( "[Postgres] Source knowledge %s not found in target knowledge mapping, skipping", sourceVector.KnowledgeID, ) continue } // Create new vector index, copy the content and vector of the source index targetVector := &pgVector{ Content: sourceVector.Content, SourceID: targetChunkID, // Update to target chunk ID SourceType: sourceVector.SourceType, ChunkID: targetChunkID, // Update to target chunk ID KnowledgeID: targetKnowledgeID, // Update to target knowledge ID KnowledgeBaseID: targetKnowledgeBaseID, // Update to target knowledge base ID Dimension: sourceVector.Dimension, Embedding: sourceVector.Embedding, // Copy the vector embedding directly, avoid recalculation } targetVectors = append(targetVectors, targetVector) } // Batch insert target vector index if len(targetVectors) > 0 { if err := g.db.WithContext(ctx). Clauses(clause.OnConflict{DoNothing: true}).Create(targetVectors).Error; err != nil { logger.GetLogger(ctx).Errorf("[Postgres] Failed to batch create target index: %v", err) return err } totalCopied += len(targetVectors) logger.GetLogger(ctx).Infof( "[Postgres] Successfully copied batch data, batch size: %d, total copied: %d", len(targetVectors), totalCopied, ) } // Move to the next batch offset += batchCount // If the number of returned records is less than the requested size, it means the last page has been reached if batchCount < batchSize { break } } logger.GetLogger(ctx).Infof("[Postgres] Index copying completed, total copied: %d", totalCopied) return nil }