Files
Enterprise-Knowledge-Base/internal/application/repository/retriever/postgres/repository.go
T
wizardchen bc2cb96d70 feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses.
- Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively.
- Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience.
- Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users.

These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
2026-03-12 10:32:26 +08:00

638 lines
23 KiB
Go

package postgres
import (
"context"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/pgvector/pgvector-go"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// pgRepository implements PostgreSQL-based retrieval operations
type pgRepository struct {
db *gorm.DB // Database connection
}
// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository
func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
logger.GetLogger(context.Background()).Info("[Postgres] Initializing PostgreSQL retriever engine repository")
return &pgRepository{db: db}
}
// EngineType returns the retriever engine type (PostgreSQL)
func (r *pgRepository) EngineType() types.RetrieverEngineType {
return types.PostgresRetrieverEngineType
}
// Support returns supported retriever types (keywords and vector)
func (r *pgRepository) Support() []types.RetrieverType {
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
// calculateIndexStorageSize calculates storage size for a single index entry
func (g *pgRepository) calculateIndexStorageSize(embeddingDB *pgVector) int64 {
// 1. Text content size
contentSizeBytes := int64(len(embeddingDB.Content))
// 2. Vector storage size (2 bytes per dimension for half-precision float)
var vectorSizeBytes int64 = 0
if embeddingDB.Dimension > 0 {
vectorSizeBytes = int64(embeddingDB.Dimension * 2)
}
// 3. Metadata size (fixed overhead for IDs, timestamps etc.)
metadataSizeBytes := int64(200)
// 4. Index overhead (HNSW index is ~2x vector size)
indexOverheadBytes := vectorSizeBytes * 2
// Total size in bytes
totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes
return totalSizeBytes
}
// EstimateStorageSize estimates total storage size for multiple indices
func (g *pgRepository) EstimateStorageSize(
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
) int64 {
var totalStorageSize int64 = 0
for _, indexInfo := range indexInfoList {
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
totalStorageSize += g.calculateIndexStorageSize(embeddingDB)
}
logger.GetLogger(ctx).Infof(
"[Postgres] Estimated storage size for %d indices: %d bytes",
len(indexInfoList), totalStorageSize,
)
return totalStorageSize
}
// Save stores a single index entry
func (g *pgRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, additionalParams map[string]any) error {
logger.GetLogger(ctx).Debugf("[Postgres] Saving index for source ID: %s", indexInfo.SourceID)
embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams)
err := g.db.WithContext(ctx).Create(embeddingDB).Error
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to save index: %v", err)
return err
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully saved index for source ID: %s", indexInfo.SourceID)
return nil
}
// BatchSave stores multiple index entries in batch
func (g *pgRepository) BatchSave(
ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any,
) error {
logger.GetLogger(ctx).Infof("[Postgres] Batch saving %d indices", len(indexInfoList))
indexInfoDBList := make([]*pgVector, len(indexInfoList))
for i := range indexInfoList {
indexInfoDBList[i] = toDBVectorEmbedding(indexInfoList[i], additionalParams)
}
err := g.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(indexInfoDBList).Error
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Batch save failed: %v", err)
return err
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch saved %d indices", len(indexInfoList))
return nil
}
// DeleteByChunkIDList deletes indices by chunk IDs
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by chunk IDs: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by chunk IDs", result.RowsAffected)
return nil
}
// DeleteBySourceIDList deletes indices by source IDs
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
if len(sourceIDList) == 0 {
return nil
}
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by source IDs, count: %d", len(sourceIDList))
result := g.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&pgVector{})
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by source IDs: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by source IDs", result.RowsAffected)
return nil
}
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by knowledge IDs: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by knowledge IDs", result.RowsAffected)
return nil
}
// Retrieve handles retrieval requests and routes to appropriate method
func (g *pgRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Debugf("[Postgres] Processing retrieval request of type: %s", params.RetrieverType)
switch params.RetrieverType {
case types.KeywordsRetrieverType:
return g.KeywordsRetrieve(ctx, params)
case types.VectorRetrieverType:
return g.VectorRetrieve(ctx, params)
}
err := errors.New("invalid retriever type")
logger.GetLogger(ctx).Errorf("[Postgres] %v: %s", err, params.RetrieverType)
return nil, err
}
// KeywordsRetrieve performs keyword-based search using PostgreSQL full-text search
func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
conds := make([]clause.Expression, 0)
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
conds = append(conds, clause.IN{
Column: "knowledge_base_id",
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
})
}
if len(params.KnowledgeIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge IDs: %v", params.KnowledgeIDs)
conds = append(conds, clause.IN{
Column: "knowledge_id",
Values: common.ToInterfaceSlice(params.KnowledgeIDs),
})
}
// Filter by tag IDs if specified
if len(params.TagIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by tag IDs: %v", params.TagIDs)
conds = append(conds, clause.IN{
Column: "tag_id",
Values: common.ToInterfaceSlice(params.TagIDs),
})
}
conds = append(conds, clause.Expr{
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
Vars: []interface{}{params.Query},
})
// Filter by is_enabled = true or NULL (NULL means enabled for historical data)
conds = append(conds, clause.Expr{
SQL: "(is_enabled IS NULL OR is_enabled = ?)",
Vars: []interface{}{true},
})
conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{
{Column: clause.Column{Name: "score"}, Desc: true},
}})
var embeddingDBList []pgVectorWithScore
err := g.db.WithContext(ctx).Clauses(conds...).Debug().
Select([]string{
"paradedb.score(id) as score",
"id",
"content",
"source_id",
"source_type",
"chunk_id",
"knowledge_id",
"knowledge_base_id",
"tag_id",
}).
Limit(int(params.TopK)).
Find(&embeddingDBList).Error
if err == gorm.ErrRecordNotFound {
logger.GetLogger(ctx).Warnf("[Postgres] No records found for keywords query: %s", params.Query)
return nil, nil
}
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Keywords retrieval failed: %v", err)
return nil, err
}
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
const maxKeywordResultLog = 8
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords)
if i < maxKeywordResultLog {
logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
i, results[i].ChunkID, results[i].Score)
}
}
if len(results) > maxKeywordResultLog {
logger.GetLogger(ctx).Debugf(
"[Postgres] Keywords result summary: total=%d logged=%d truncated=%d",
len(results), maxKeywordResultLog, len(results)-maxKeywordResultLog,
)
}
return []*types.RetrieveResult{
{
Results: results,
RetrieverEngineType: types.PostgresRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
Error: nil,
},
}, nil
}
// VectorRetrieve performs vector similarity search using pgvector
// Optimized to use HNSW index efficiently and avoid recalculating vector distance
func (g *pgRepository) VectorRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
len(params.Embedding), params.TopK, params.Threshold)
dimension := len(params.Embedding)
queryVector := pgvector.NewHalfVector(params.Embedding)
// Build WHERE conditions for filtering
whereParts := make([]string, 0)
allVars := make([]interface{}, 0)
// Add query vector first (used in ORDER BY for HNSW index)
allVars = append(allVars, queryVector)
// Dimension filter (required for HNSW index WHERE clause)
whereParts = append(whereParts, fmt.Sprintf("dimension = $%d", len(allVars)+1))
allVars = append(allVars, dimension)
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge base IDs: %v",
params.KnowledgeBaseIDs,
)
placeholders := make([]string, len(params.KnowledgeBaseIDs))
paramStart := len(allVars) + 1
for i := range params.KnowledgeBaseIDs {
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
allVars = append(allVars, params.KnowledgeBaseIDs[i])
}
whereParts = append(whereParts, fmt.Sprintf("knowledge_base_id IN (%s)",
strings.Join(placeholders, ", ")))
}
if len(params.KnowledgeIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge IDs: %v",
params.KnowledgeIDs,
)
placeholders := make([]string, len(params.KnowledgeIDs))
paramStart := len(allVars) + 1
for i := range params.KnowledgeIDs {
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
allVars = append(allVars, params.KnowledgeIDs[i])
}
whereParts = append(whereParts, fmt.Sprintf("knowledge_id IN (%s)",
strings.Join(placeholders, ", ")))
}
// Filter by tag IDs if specified
if len(params.TagIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by tag IDs: %v",
params.TagIDs,
)
placeholders := make([]string, len(params.TagIDs))
paramStart := len(allVars) + 1
for i := range params.TagIDs {
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
allVars = append(allVars, params.TagIDs[i])
}
whereParts = append(whereParts, fmt.Sprintf("tag_id IN (%s)",
strings.Join(placeholders, ", ")))
}
// is_enabled filter
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
allVars = append(allVars, true)
// Build WHERE clause string
whereClause := ""
if len(whereParts) > 0 {
whereClause = "WHERE " + strings.Join(whereParts, " AND ")
}
// Expand TopK to get more candidates before threshold filtering
expandedTopK := params.TopK * 2
if expandedTopK < 100 {
expandedTopK = 100 // Minimum 100 candidates
}
if expandedTopK > 1000 {
expandedTopK = 1000 // Maximum 1000 candidates
}
// Optimized query: Use subquery to calculate distance once
// Strategy: Use ORDER BY with vector distance to leverage HNSW index,
// then filter by threshold in outer query
// This allows PostgreSQL to use HNSW index efficiently
subqueryLimitParam := len(allVars) + 1
thresholdParam := len(allVars) + 2
finalLimitParam := len(allVars) + 3
querySQL := fmt.Sprintf(`
SELECT
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id,
(1 - distance) as score
FROM (
SELECT
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id,
embedding::halfvec(%d) <=> $1::halfvec as distance
FROM embeddings
%s
ORDER BY embedding::halfvec(%d) <=> $1::halfvec
LIMIT $%d
) AS candidates
WHERE distance <= $%d
ORDER BY distance ASC
LIMIT $%d
`, dimension, whereClause, dimension, subqueryLimitParam, thresholdParam, finalLimitParam)
allVars = append(allVars, expandedTopK) // LIMIT in subquery
allVars = append(allVars, 1-params.Threshold) // Distance threshold
allVars = append(allVars, params.TopK) // Final LIMIT
var embeddingDBList []pgVectorWithScore
err := g.db.WithContext(ctx).Raw(querySQL, allVars...).Scan(&embeddingDBList).Error
if err == gorm.ErrRecordNotFound {
logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold)
return nil, nil
}
if err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Vector retrieval failed: %v", err)
return nil, err
}
// Apply final TopK limit (in case we got more results than needed)
if len(embeddingDBList) > int(params.TopK) {
embeddingDBList = embeddingDBList[:params.TopK]
}
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
const maxVectorResultLog = 8
for i := range embeddingDBList {
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding)
if i < maxVectorResultLog {
logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
i, results[i].ChunkID, results[i].Score)
}
}
if len(results) > maxVectorResultLog {
logger.GetLogger(ctx).Debugf(
"[Postgres] Vector search result summary: total=%d logged=%d truncated=%d",
len(results), maxVectorResultLog, len(results)-maxVectorResultLog,
)
}
return []*types.RetrieveResult{
{
Results: results,
RetrieverEngineType: types.PostgresRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
Error: nil,
},
}, nil
}
// CopyIndices copies index data
func (g *pgRepository) CopyIndices(ctx context.Context,
sourceKnowledgeBaseID string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
logger.GetLogger(ctx).Infof(
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
)
if len(sourceToTargetChunkIDMap) == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] Mapping is empty, no need to copy")
return nil
}
// Batch processing parameters
batchSize := 500 // Number of records to process per batch
offset := 0 // Offset for pagination
totalCopied := 0 // Total number of copied records
for {
// Paginated query for source data
var sourceVectors []*pgVector
if err := g.db.WithContext(ctx).
Where("knowledge_base_id = ?", sourceKnowledgeBaseID).
Limit(batchSize).
Offset(offset).
Find(&sourceVectors).Error; err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to query source index data: %v", err)
return err
}
// If no more data, exit the loop
if len(sourceVectors) == 0 {
if offset == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] No source index data found")
}
break
}
batchCount := len(sourceVectors)
logger.GetLogger(ctx).Infof(
"[Postgres] Found %d source index data, batch start position: %d",
batchCount, offset,
)
// Create target vector index
targetVectors := make([]*pgVector, 0, batchCount)
for _, sourceVector := range sourceVectors {
// Get the mapped target chunk ID
targetChunkID, ok := sourceToTargetChunkIDMap[sourceVector.ChunkID]
if !ok {
logger.GetLogger(ctx).Warnf(
"[Postgres] Source chunk %s not found in target chunk mapping, skipping",
sourceVector.ChunkID,
)
continue
}
// Get the mapped target knowledge ID
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceVector.KnowledgeID]
if !ok {
logger.GetLogger(ctx).Warnf(
"[Postgres] Source knowledge %s not found in target knowledge mapping, skipping",
sourceVector.KnowledgeID,
)
continue
}
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
var targetSourceID string
if sourceVector.SourceID == sourceVector.ChunkID {
// Regular chunk, use targetChunkID as SourceID
targetSourceID = targetChunkID
} else if strings.HasPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-") {
// This is a generated question, preserve the questionID part
questionID := strings.TrimPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-")
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
} else {
// For other complex scenarios, generate new unique SourceID
targetSourceID = uuid.New().String()
}
// Create new vector index, copy the content and vector of the source index
targetVector := &pgVector{
Content: sourceVector.Content,
SourceID: targetSourceID, // Handle SourceID transformation properly
SourceType: sourceVector.SourceType,
ChunkID: targetChunkID, // Update to target chunk ID
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
KnowledgeBaseID: targetKnowledgeBaseID, // Update to target knowledge base ID
Dimension: sourceVector.Dimension,
Embedding: sourceVector.Embedding, // Copy the vector embedding directly, avoid recalculation
}
targetVectors = append(targetVectors, targetVector)
}
// Batch insert target vector index
if len(targetVectors) > 0 {
if err := g.db.WithContext(ctx).
Clauses(clause.OnConflict{DoNothing: true}).Create(targetVectors).Error; err != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to batch create target index: %v", err)
return err
}
totalCopied += len(targetVectors)
logger.GetLogger(ctx).Infof(
"[Postgres] Successfully copied batch data, batch size: %d, total copied: %d",
len(targetVectors),
totalCopied,
)
}
// Move to the next batch
offset += batchCount
// If the number of returned records is less than the requested size, it means the last page has been reached
if batchCount < batchSize {
break
}
}
logger.GetLogger(ctx).Infof("[Postgres] Index copying completed, total copied: %d", totalCopied)
return nil
}
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
if len(chunkStatusMap) == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] Chunk status map is empty, skipping update")
return nil
}
logger.GetLogger(ctx).Infof("[Postgres] Batch updating chunk enabled status, count: %d", len(chunkStatusMap))
// Group chunks by enabled status for batch updates
enabledChunkIDs := make([]string, 0)
disabledChunkIDs := make([]string, 0)
for chunkID, enabled := range chunkStatusMap {
if enabled {
enabledChunkIDs = append(enabledChunkIDs, chunkID)
} else {
disabledChunkIDs = append(disabledChunkIDs, chunkID)
}
}
// Batch update enabled chunks
if len(enabledChunkIDs) > 0 {
result := g.db.WithContext(ctx).Model(&pgVector{}).
Where("chunk_id IN ?", enabledChunkIDs).
Update("is_enabled", true)
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update enabled chunks: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).
Infof("[Postgres] Updated %d chunks to enabled, rows affected: %d", len(enabledChunkIDs), result.RowsAffected)
}
// Batch update disabled chunks
if len(disabledChunkIDs) > 0 {
result := g.db.WithContext(ctx).Model(&pgVector{}).
Where("chunk_id IN ?", disabledChunkIDs).
Update("is_enabled", false)
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update disabled chunks: %v", result.Error)
return result.Error
}
logger.GetLogger(ctx).
Infof("[Postgres] Updated %d chunks to disabled, rows affected: %d", len(disabledChunkIDs), result.RowsAffected)
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk enabled status")
return nil
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
if len(chunkTagMap) == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] Chunk tag map is empty, skipping update")
return nil
}
logger.GetLogger(ctx).Infof("[Postgres] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
// Group chunks by tag ID for batch updates
tagGroups := make(map[string][]string)
for chunkID, tagID := range chunkTagMap {
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
}
// Batch update chunks for each tag ID
for tagID, chunkIDs := range tagGroups {
result := g.db.WithContext(ctx).Model(&pgVector{}).
Where("chunk_id IN ?", chunkIDs).
Update("tag_id", tagID)
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update chunks with tag_id %s: %v", tagID, result.Error)
return result.Error
}
logger.GetLogger(ctx).
Infof("[Postgres] Updated %d chunks to tag_id=%s, rows affected: %d", len(chunkIDs), tagID, result.RowsAffected)
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID")
return nil
}