feat(repository): enhance document indexing with isEnabled and isRecommended fields
- Added `IsEnabled` and `IsRecommended` fields to the document indexing process in both ElasticFaiss and Elasticsearch repositories, improving the flexibility and functionality of document retrieval. - Updated the data extraction process to include these new fields, ensuring they are correctly populated from the source data. - Adjusted the indexing logic in the knowledge service to reflect these changes, maintaining consistency across the application.
This commit is contained in:
@@ -1063,6 +1063,21 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
|
||||
sourceType = int(st)
|
||||
}
|
||||
|
||||
// Extract is_enabled (default true for backward compatibility)
|
||||
isEnabled := true
|
||||
if v, ok := sourceObj["is_enabled"].(bool); ok {
|
||||
isEnabled = v
|
||||
}
|
||||
|
||||
// Extract is_recommended
|
||||
isRecommended := false
|
||||
if v, ok := sourceObj["is_recommended"].(bool); ok {
|
||||
isRecommended = v
|
||||
}
|
||||
|
||||
// Extract tag_id
|
||||
tagID, _ := sourceObj["tag_id"].(string)
|
||||
|
||||
// Handle SourceID transformation for generated questions
|
||||
// Generated questions have SourceID format: {chunkID}-{questionID}
|
||||
// Regular chunks have SourceID == ChunkID
|
||||
@@ -1100,6 +1115,9 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
|
||||
KnowledgeBaseID: targetKnowledgeBaseID,
|
||||
Content: content,
|
||||
SourceType: typesLocal.SourceType(sourceType),
|
||||
IsEnabled: isEnabled,
|
||||
IsRecommended: isRecommended,
|
||||
TagID: tagID,
|
||||
}
|
||||
|
||||
return indexInfo, embedding, nil
|
||||
|
||||
@@ -487,6 +487,7 @@ func (s *DataTableSummaryService) indexToVectorDB(
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
IsEnabled: true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1509,6 +1509,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
IsEnabled: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1996,6 +1997,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
ChunkID: summaryChunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
IsEnabled: true,
|
||||
}}
|
||||
|
||||
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
|
||||
@@ -2163,6 +2165,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
IsEnabled: true,
|
||||
})
|
||||
}
|
||||
logger.Debugf(ctx, "Generated %d questions for chunk %s", len(questions), chunk.ID)
|
||||
@@ -2819,6 +2822,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
IsEnabled: true,
|
||||
})
|
||||
ids = append(ids, chunk.ID)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user