feat: 完善知识库复制时生成问题的SourceID转换逻辑和标签映射功能
This commit is contained in:
@@ -1036,11 +1036,28 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
|
||||
|
||||
// Extract basic content
|
||||
content, _ := sourceObj["content"].(string)
|
||||
originalSourceID, _ := sourceObj["source_id"].(string)
|
||||
sourceType := 0
|
||||
if st, ok := sourceObj["source_type"].(float64); ok {
|
||||
sourceType = int(st)
|
||||
}
|
||||
|
||||
// Handle SourceID transformation for generated questions
|
||||
// Generated questions have SourceID format: {chunkID}-{questionID}
|
||||
// Regular chunks have SourceID == ChunkID
|
||||
var targetSourceID string
|
||||
if originalSourceID == sourceChunkID {
|
||||
// Regular chunk, use targetChunkID as SourceID
|
||||
targetSourceID = targetChunkID
|
||||
} else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") {
|
||||
// This is a generated question, preserve the questionID part
|
||||
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
|
||||
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
|
||||
} else {
|
||||
// For other complex scenarios, generate new unique SourceID
|
||||
targetSourceID = uuid.New().String()
|
||||
}
|
||||
|
||||
// Extract embedding vector (if exists)
|
||||
var embedding []float32
|
||||
if embeddingInterface, ok := sourceObj["embedding"].([]interface{}); ok {
|
||||
@@ -1057,7 +1074,7 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
|
||||
// Create IndexInfo object
|
||||
indexInfo := &typesLocal.IndexInfo{
|
||||
ChunkID: targetChunkID,
|
||||
SourceID: targetChunkID,
|
||||
SourceID: targetSourceID,
|
||||
KnowledgeID: targetKnowledgeID,
|
||||
KnowledgeBaseID: targetKnowledgeBaseID,
|
||||
Content: content,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
elasticsearchRetriever "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
|
||||
"github.com/elastic/go-elasticsearch/v8/typedapi/types"
|
||||
"github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/scriptlanguage"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8
|
||||
@@ -527,10 +529,26 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
|
||||
embeddingMap[targetChunkID] = sourceDoc.Embedding
|
||||
}
|
||||
|
||||
// Handle SourceID transformation for generated questions
|
||||
// Generated questions have SourceID format: {chunkID}-{questionID}
|
||||
// Regular chunks have SourceID == ChunkID
|
||||
var targetSourceID string
|
||||
if sourceDoc.SourceID == sourceDoc.ChunkID {
|
||||
// Regular chunk, use targetChunkID as SourceID
|
||||
targetSourceID = targetChunkID
|
||||
} else if strings.HasPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-") {
|
||||
// This is a generated question, preserve the questionID part
|
||||
questionID := strings.TrimPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-")
|
||||
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
|
||||
} else {
|
||||
// For other complex scenarios, generate new unique SourceID
|
||||
targetSourceID = uuid.New().String()
|
||||
}
|
||||
|
||||
// Create new index information
|
||||
indexInfo := &typesLocal.IndexInfo{
|
||||
Content: sourceDoc.Content,
|
||||
SourceID: targetChunkID,
|
||||
SourceID: targetSourceID,
|
||||
SourceType: typesLocal.SourceType(sourceDoc.SourceType),
|
||||
ChunkID: targetChunkID,
|
||||
KnowledgeID: targetKnowledgeID,
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pgvector/pgvector-go"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
@@ -423,10 +424,26 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle SourceID transformation for generated questions
|
||||
// Generated questions have SourceID format: {chunkID}-{questionID}
|
||||
// Regular chunks have SourceID == ChunkID
|
||||
var targetSourceID string
|
||||
if sourceVector.SourceID == sourceVector.ChunkID {
|
||||
// Regular chunk, use targetChunkID as SourceID
|
||||
targetSourceID = targetChunkID
|
||||
} else if strings.HasPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-") {
|
||||
// This is a generated question, preserve the questionID part
|
||||
questionID := strings.TrimPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-")
|
||||
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
|
||||
} else {
|
||||
// For other complex scenarios, generate new unique SourceID
|
||||
targetSourceID = uuid.New().String()
|
||||
}
|
||||
|
||||
// Create new vector index, copy the content and vector of the source index
|
||||
targetVector := &pgVector{
|
||||
Content: sourceVector.Content,
|
||||
SourceID: targetChunkID, // Update to target chunk ID
|
||||
SourceID: targetSourceID, // Handle SourceID transformation properly
|
||||
SourceType: sourceVector.SourceType,
|
||||
ChunkID: targetChunkID, // Update to target chunk ID
|
||||
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
|
||||
|
||||
@@ -699,6 +699,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
|
||||
sourceChunkID := payload[fieldChunkID].GetStringValue()
|
||||
sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue()
|
||||
originalSourceID := payload[fieldSourceID].GetStringValue()
|
||||
|
||||
targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID]
|
||||
if !ok {
|
||||
@@ -712,9 +713,25 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle SourceID transformation for generated questions
|
||||
// Generated questions have SourceID format: {chunkID}-{questionID}
|
||||
// Regular chunks have SourceID == ChunkID
|
||||
var targetSourceID string
|
||||
if originalSourceID == sourceChunkID {
|
||||
// Regular chunk, use targetChunkID as SourceID
|
||||
targetSourceID = targetChunkID
|
||||
} else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") {
|
||||
// This is a generated question, preserve the questionID part
|
||||
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
|
||||
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
|
||||
} else {
|
||||
// For other complex scenarios, generate new unique SourceID
|
||||
targetSourceID = uuid.New().String()
|
||||
}
|
||||
|
||||
newPayload := qdrant.NewValueMap(map[string]any{
|
||||
fieldContent: payload[fieldContent].GetStringValue(),
|
||||
fieldSourceID: targetChunkID,
|
||||
fieldSourceID: targetSourceID,
|
||||
fieldSourceType: payload[fieldSourceType].GetIntegerValue(),
|
||||
fieldChunkID: targetChunkID,
|
||||
fieldKnowledgeID: targetKnowledgeID,
|
||||
|
||||
@@ -2531,6 +2531,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
chunkPage := 1
|
||||
chunkPageSize := 100
|
||||
srcTodst := map[string]string{}
|
||||
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
|
||||
targetChunks := make([]*types.Chunk, 0, 10)
|
||||
chunkType := []types.ChunkType{
|
||||
types.ChunkTypeText, types.ChunkTypeSummary,
|
||||
@@ -2555,22 +2556,41 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
if len(sourceChunks) == 0 {
|
||||
break
|
||||
}
|
||||
now := time.Now()
|
||||
for _, sourceChunk := range sourceChunks {
|
||||
// Map TagID to target knowledge base
|
||||
targetTagID := ""
|
||||
if sourceChunk.TagID != "" {
|
||||
if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok {
|
||||
targetTagID = mappedTagID
|
||||
} else {
|
||||
// Try to find or create the tag in target knowledge base
|
||||
targetTagID = s.getOrCreateTagInTarget(ctx, src, dst, sourceChunk.TagID, tagIDMapping)
|
||||
}
|
||||
}
|
||||
|
||||
targetChunk := &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: dst.TenantID,
|
||||
KnowledgeID: dst.ID,
|
||||
KnowledgeBaseID: dst.KnowledgeBaseID,
|
||||
TagID: targetTagID,
|
||||
Content: sourceChunk.Content,
|
||||
ChunkIndex: sourceChunk.ChunkIndex,
|
||||
IsEnabled: sourceChunk.IsEnabled,
|
||||
Flags: sourceChunk.Flags,
|
||||
Status: sourceChunk.Status,
|
||||
StartAt: sourceChunk.StartAt,
|
||||
EndAt: sourceChunk.EndAt,
|
||||
PreChunkID: sourceChunk.PreChunkID,
|
||||
NextChunkID: sourceChunk.NextChunkID,
|
||||
ChunkType: sourceChunk.ChunkType,
|
||||
ParentChunkID: sourceChunk.ParentChunkID,
|
||||
Metadata: sourceChunk.Metadata,
|
||||
ContentHash: sourceChunk.ContentHash,
|
||||
ImageInfo: sourceChunk.ImageInfo,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
targetChunks = append(targetChunks, targetChunk)
|
||||
srcTodst[sourceChunk.ID] = targetChunk.ID
|
||||
@@ -5618,3 +5638,50 @@ func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string
|
||||
}
|
||||
return &progress, nil
|
||||
}
|
||||
|
||||
// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag.
|
||||
// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB.
|
||||
// If not found, it creates a new tag with the same properties.
|
||||
// The mapping is cached in tagIDMapping for subsequent lookups.
|
||||
func (s *knowledgeService) getOrCreateTagInTarget(
|
||||
ctx context.Context,
|
||||
src, dst *types.Knowledge,
|
||||
srcTagID string,
|
||||
tagIDMapping map[string]string,
|
||||
) string {
|
||||
// Get source tag
|
||||
srcTag, err := s.tagRepo.GetByID(ctx, src.TenantID, srcTagID)
|
||||
if err != nil || srcTag == nil {
|
||||
logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err)
|
||||
tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups
|
||||
return ""
|
||||
}
|
||||
|
||||
// Try to find existing tag with same name in target KB
|
||||
dstTag, err := s.tagRepo.GetByName(ctx, dst.TenantID, dst.KnowledgeBaseID, srcTag.Name)
|
||||
if err == nil && dstTag != nil {
|
||||
tagIDMapping[srcTagID] = dstTag.ID
|
||||
return dstTag.ID
|
||||
}
|
||||
|
||||
// Create new tag in target KB
|
||||
newTag := &types.KnowledgeTag{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: dst.TenantID,
|
||||
KnowledgeBaseID: dst.KnowledgeBaseID,
|
||||
Name: srcTag.Name,
|
||||
Color: srcTag.Color,
|
||||
SortOrder: srcTag.SortOrder,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := s.tagRepo.Create(ctx, newTag); err != nil {
|
||||
logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err)
|
||||
tagIDMapping[srcTagID] = "" // Cache empty result
|
||||
return ""
|
||||
}
|
||||
|
||||
tagIDMapping[srcTagID] = newTag.ID
|
||||
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dst.KnowledgeBaseID)
|
||||
return newTag.ID
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user