diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 1497a9bc..60c6747f 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -1036,11 +1036,28 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context, // Extract basic content content, _ := sourceObj["content"].(string) + originalSourceID, _ := sourceObj["source_id"].(string) sourceType := 0 if st, ok := sourceObj["source_type"].(float64); ok { sourceType = int(st) } + // Handle SourceID transformation for generated questions + // Generated questions have SourceID format: {chunkID}-{questionID} + // Regular chunks have SourceID == ChunkID + var targetSourceID string + if originalSourceID == sourceChunkID { + // Regular chunk, use targetChunkID as SourceID + targetSourceID = targetChunkID + } else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") { + // This is a generated question, preserve the questionID part + questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-") + targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID) + } else { + // For other complex scenarios, generate new unique SourceID + targetSourceID = uuid.New().String() + } + // Extract embedding vector (if exists) var embedding []float32 if embeddingInterface, ok := sourceObj["embedding"].([]interface{}); ok { @@ -1057,7 +1074,7 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context, // Create IndexInfo object indexInfo := &typesLocal.IndexInfo{ ChunkID: targetChunkID, - SourceID: targetChunkID, + SourceID: targetSourceID, KnowledgeID: targetKnowledgeID, KnowledgeBaseID: targetKnowledgeBaseID, Content: content, diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index acd305c3..a73986d1 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "strings" elasticsearchRetriever "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch" "github.com/Tencent/WeKnora/internal/config" @@ -15,6 +16,7 @@ import ( "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" "github.com/elastic/go-elasticsearch/v8/typedapi/types" "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/scriptlanguage" + "github.com/google/uuid" ) // elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8 @@ -527,10 +529,26 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context, embeddingMap[targetChunkID] = sourceDoc.Embedding } + // Handle SourceID transformation for generated questions + // Generated questions have SourceID format: {chunkID}-{questionID} + // Regular chunks have SourceID == ChunkID + var targetSourceID string + if sourceDoc.SourceID == sourceDoc.ChunkID { + // Regular chunk, use targetChunkID as SourceID + targetSourceID = targetChunkID + } else if strings.HasPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-") { + // This is a generated question, preserve the questionID part + questionID := strings.TrimPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-") + targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID) + } else { + // For other complex scenarios, generate new unique SourceID + targetSourceID = uuid.New().String() + } + // Create new index information indexInfo := &typesLocal.IndexInfo{ Content: sourceDoc.Content, - SourceID: targetChunkID, + SourceID: targetSourceID, SourceType: typesLocal.SourceType(sourceDoc.SourceType), ChunkID: targetChunkID, KnowledgeID: targetKnowledgeID, diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index 3c493714..773203f0 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" "github.com/pgvector/pgvector-go" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -423,10 +424,26 @@ func (g *pgRepository) CopyIndices(ctx context.Context, continue } + // Handle SourceID transformation for generated questions + // Generated questions have SourceID format: {chunkID}-{questionID} + // Regular chunks have SourceID == ChunkID + var targetSourceID string + if sourceVector.SourceID == sourceVector.ChunkID { + // Regular chunk, use targetChunkID as SourceID + targetSourceID = targetChunkID + } else if strings.HasPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-") { + // This is a generated question, preserve the questionID part + questionID := strings.TrimPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-") + targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID) + } else { + // For other complex scenarios, generate new unique SourceID + targetSourceID = uuid.New().String() + } + // Create new vector index, copy the content and vector of the source index targetVector := &pgVector{ Content: sourceVector.Content, - SourceID: targetChunkID, // Update to target chunk ID + SourceID: targetSourceID, // Handle SourceID transformation properly SourceType: sourceVector.SourceType, ChunkID: targetChunkID, // Update to target chunk ID KnowledgeID: targetKnowledgeID, // Update to target knowledge ID diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index c971a1c4..6d660d5e 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -699,6 +699,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, sourceChunkID := payload[fieldChunkID].GetStringValue() sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue() + originalSourceID := payload[fieldSourceID].GetStringValue() targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID] if !ok { @@ -712,9 +713,25 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, continue } + // Handle SourceID transformation for generated questions + // Generated questions have SourceID format: {chunkID}-{questionID} + // Regular chunks have SourceID == ChunkID + var targetSourceID string + if originalSourceID == sourceChunkID { + // Regular chunk, use targetChunkID as SourceID + targetSourceID = targetChunkID + } else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") { + // This is a generated question, preserve the questionID part + questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-") + targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID) + } else { + // For other complex scenarios, generate new unique SourceID + targetSourceID = uuid.New().String() + } + newPayload := qdrant.NewValueMap(map[string]any{ fieldContent: payload[fieldContent].GetStringValue(), - fieldSourceID: targetChunkID, + fieldSourceID: targetSourceID, fieldSourceType: payload[fieldSourceType].GetIntegerValue(), fieldChunkID: targetChunkID, fieldKnowledgeID: targetKnowledgeID, diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 42ed3b5b..ec53f49d 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2531,6 +2531,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl chunkPage := 1 chunkPageSize := 100 srcTodst := map[string]string{} + tagIDMapping := map[string]string{} // srcTagID -> dstTagID targetChunks := make([]*types.Chunk, 0, 10) chunkType := []types.ChunkType{ types.ChunkTypeText, types.ChunkTypeSummary, @@ -2555,22 +2556,41 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl if len(sourceChunks) == 0 { break } + now := time.Now() for _, sourceChunk := range sourceChunks { + // Map TagID to target knowledge base + targetTagID := "" + if sourceChunk.TagID != "" { + if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok { + targetTagID = mappedTagID + } else { + // Try to find or create the tag in target knowledge base + targetTagID = s.getOrCreateTagInTarget(ctx, src, dst, sourceChunk.TagID, tagIDMapping) + } + } + targetChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: dst.TenantID, KnowledgeID: dst.ID, KnowledgeBaseID: dst.KnowledgeBaseID, + TagID: targetTagID, Content: sourceChunk.Content, ChunkIndex: sourceChunk.ChunkIndex, IsEnabled: sourceChunk.IsEnabled, + Flags: sourceChunk.Flags, + Status: sourceChunk.Status, StartAt: sourceChunk.StartAt, EndAt: sourceChunk.EndAt, PreChunkID: sourceChunk.PreChunkID, NextChunkID: sourceChunk.NextChunkID, ChunkType: sourceChunk.ChunkType, ParentChunkID: sourceChunk.ParentChunkID, + Metadata: sourceChunk.Metadata, + ContentHash: sourceChunk.ContentHash, ImageInfo: sourceChunk.ImageInfo, + CreatedAt: now, + UpdatedAt: now, } targetChunks = append(targetChunks, targetChunk) srcTodst[sourceChunk.ID] = targetChunk.ID @@ -5618,3 +5638,50 @@ func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string } return &progress, nil } + +// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag. +// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB. +// If not found, it creates a new tag with the same properties. +// The mapping is cached in tagIDMapping for subsequent lookups. +func (s *knowledgeService) getOrCreateTagInTarget( + ctx context.Context, + src, dst *types.Knowledge, + srcTagID string, + tagIDMapping map[string]string, +) string { + // Get source tag + srcTag, err := s.tagRepo.GetByID(ctx, src.TenantID, srcTagID) + if err != nil || srcTag == nil { + logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err) + tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups + return "" + } + + // Try to find existing tag with same name in target KB + dstTag, err := s.tagRepo.GetByName(ctx, dst.TenantID, dst.KnowledgeBaseID, srcTag.Name) + if err == nil && dstTag != nil { + tagIDMapping[srcTagID] = dstTag.ID + return dstTag.ID + } + + // Create new tag in target KB + newTag := &types.KnowledgeTag{ + ID: uuid.New().String(), + TenantID: dst.TenantID, + KnowledgeBaseID: dst.KnowledgeBaseID, + Name: srcTag.Name, + Color: srcTag.Color, + SortOrder: srcTag.SortOrder, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + if err := s.tagRepo.Create(ctx, newTag); err != nil { + logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err) + tagIDMapping[srcTagID] = "" // Cache empty result + return "" + } + + tagIDMapping[srcTagID] = newTag.ID + logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dst.KnowledgeBaseID) + return newTag.ID +}