feat: 完善知识库复制时生成问题的SourceID转换逻辑和标签映射功能

This commit is contained in:
wizardchen
2025-12-17 11:30:59 +08:00
committed by lyingbug
parent b70450ddd8
commit 3977d90181
5 changed files with 140 additions and 4 deletions
@@ -1036,11 +1036,28 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
// Extract basic content
content, _ := sourceObj["content"].(string)
originalSourceID, _ := sourceObj["source_id"].(string)
sourceType := 0
if st, ok := sourceObj["source_type"].(float64); ok {
sourceType = int(st)
}
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
var targetSourceID string
if originalSourceID == sourceChunkID {
// Regular chunk, use targetChunkID as SourceID
targetSourceID = targetChunkID
} else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") {
// This is a generated question, preserve the questionID part
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
} else {
// For other complex scenarios, generate new unique SourceID
targetSourceID = uuid.New().String()
}
// Extract embedding vector (if exists)
var embedding []float32
if embeddingInterface, ok := sourceObj["embedding"].([]interface{}); ok {
@@ -1057,7 +1074,7 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
// Create IndexInfo object
indexInfo := &typesLocal.IndexInfo{
ChunkID: targetChunkID,
SourceID: targetChunkID,
SourceID: targetSourceID,
KnowledgeID: targetKnowledgeID,
KnowledgeBaseID: targetKnowledgeBaseID,
Content: content,
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"strings"
elasticsearchRetriever "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch"
"github.com/Tencent/WeKnora/internal/config"
@@ -15,6 +16,7 @@ import (
"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
"github.com/elastic/go-elasticsearch/v8/typedapi/types"
"github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/scriptlanguage"
"github.com/google/uuid"
)
// elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8
@@ -527,10 +529,26 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
embeddingMap[targetChunkID] = sourceDoc.Embedding
}
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
var targetSourceID string
if sourceDoc.SourceID == sourceDoc.ChunkID {
// Regular chunk, use targetChunkID as SourceID
targetSourceID = targetChunkID
} else if strings.HasPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-") {
// This is a generated question, preserve the questionID part
questionID := strings.TrimPrefix(sourceDoc.SourceID, sourceDoc.ChunkID+"-")
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
} else {
// For other complex scenarios, generate new unique SourceID
targetSourceID = uuid.New().String()
}
// Create new index information
indexInfo := &typesLocal.IndexInfo{
Content: sourceDoc.Content,
SourceID: targetChunkID,
SourceID: targetSourceID,
SourceType: typesLocal.SourceType(sourceDoc.SourceType),
ChunkID: targetChunkID,
KnowledgeID: targetKnowledgeID,
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/pgvector/pgvector-go"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -423,10 +424,26 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
continue
}
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
var targetSourceID string
if sourceVector.SourceID == sourceVector.ChunkID {
// Regular chunk, use targetChunkID as SourceID
targetSourceID = targetChunkID
} else if strings.HasPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-") {
// This is a generated question, preserve the questionID part
questionID := strings.TrimPrefix(sourceVector.SourceID, sourceVector.ChunkID+"-")
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
} else {
// For other complex scenarios, generate new unique SourceID
targetSourceID = uuid.New().String()
}
// Create new vector index, copy the content and vector of the source index
targetVector := &pgVector{
Content: sourceVector.Content,
SourceID: targetChunkID, // Update to target chunk ID
SourceID: targetSourceID, // Handle SourceID transformation properly
SourceType: sourceVector.SourceType,
ChunkID: targetChunkID, // Update to target chunk ID
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
@@ -699,6 +699,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
sourceChunkID := payload[fieldChunkID].GetStringValue()
sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue()
originalSourceID := payload[fieldSourceID].GetStringValue()
targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID]
if !ok {
@@ -712,9 +713,25 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
continue
}
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
var targetSourceID string
if originalSourceID == sourceChunkID {
// Regular chunk, use targetChunkID as SourceID
targetSourceID = targetChunkID
} else if strings.HasPrefix(originalSourceID, sourceChunkID+"-") {
// This is a generated question, preserve the questionID part
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
targetSourceID = fmt.Sprintf("%s-%s", targetChunkID, questionID)
} else {
// For other complex scenarios, generate new unique SourceID
targetSourceID = uuid.New().String()
}
newPayload := qdrant.NewValueMap(map[string]any{
fieldContent: payload[fieldContent].GetStringValue(),
fieldSourceID: targetChunkID,
fieldSourceID: targetSourceID,
fieldSourceType: payload[fieldSourceType].GetIntegerValue(),
fieldChunkID: targetChunkID,
fieldKnowledgeID: targetKnowledgeID,
+67
View File
@@ -2531,6 +2531,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
chunkPage := 1
chunkPageSize := 100
srcTodst := map[string]string{}
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
targetChunks := make([]*types.Chunk, 0, 10)
chunkType := []types.ChunkType{
types.ChunkTypeText, types.ChunkTypeSummary,
@@ -2555,22 +2556,41 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
if len(sourceChunks) == 0 {
break
}
now := time.Now()
for _, sourceChunk := range sourceChunks {
// Map TagID to target knowledge base
targetTagID := ""
if sourceChunk.TagID != "" {
if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok {
targetTagID = mappedTagID
} else {
// Try to find or create the tag in target knowledge base
targetTagID = s.getOrCreateTagInTarget(ctx, src, dst, sourceChunk.TagID, tagIDMapping)
}
}
targetChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: dst.TenantID,
KnowledgeID: dst.ID,
KnowledgeBaseID: dst.KnowledgeBaseID,
TagID: targetTagID,
Content: sourceChunk.Content,
ChunkIndex: sourceChunk.ChunkIndex,
IsEnabled: sourceChunk.IsEnabled,
Flags: sourceChunk.Flags,
Status: sourceChunk.Status,
StartAt: sourceChunk.StartAt,
EndAt: sourceChunk.EndAt,
PreChunkID: sourceChunk.PreChunkID,
NextChunkID: sourceChunk.NextChunkID,
ChunkType: sourceChunk.ChunkType,
ParentChunkID: sourceChunk.ParentChunkID,
Metadata: sourceChunk.Metadata,
ContentHash: sourceChunk.ContentHash,
ImageInfo: sourceChunk.ImageInfo,
CreatedAt: now,
UpdatedAt: now,
}
targetChunks = append(targetChunks, targetChunk)
srcTodst[sourceChunk.ID] = targetChunk.ID
@@ -5618,3 +5638,50 @@ func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string
}
return &progress, nil
}
// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag.
// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB.
// If not found, it creates a new tag with the same properties.
// The mapping is cached in tagIDMapping for subsequent lookups.
func (s *knowledgeService) getOrCreateTagInTarget(
ctx context.Context,
src, dst *types.Knowledge,
srcTagID string,
tagIDMapping map[string]string,
) string {
// Get source tag
srcTag, err := s.tagRepo.GetByID(ctx, src.TenantID, srcTagID)
if err != nil || srcTag == nil {
logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err)
tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups
return ""
}
// Try to find existing tag with same name in target KB
dstTag, err := s.tagRepo.GetByName(ctx, dst.TenantID, dst.KnowledgeBaseID, srcTag.Name)
if err == nil && dstTag != nil {
tagIDMapping[srcTagID] = dstTag.ID
return dstTag.ID
}
// Create new tag in target KB
newTag := &types.KnowledgeTag{
ID: uuid.New().String(),
TenantID: dst.TenantID,
KnowledgeBaseID: dst.KnowledgeBaseID,
Name: srcTag.Name,
Color: srcTag.Color,
SortOrder: srcTag.SortOrder,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
if err := s.tagRepo.Create(ctx, newTag); err != nil {
logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err)
tagIDMapping[srcTagID] = "" // Cache empty result
return ""
}
tagIDMapping[srcTagID] = newTag.ID
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dst.KnowledgeBaseID)
return newTag.ID
}