feat: 支持FAQ索引类型分离,新增is_enabled字段过滤和批量更新优化

This commit is contained in:
wizardchen
2025-12-16 21:17:55 +08:00
committed by lyingbug
parent 509582a4d5
commit b70450ddd8
12 changed files with 101 additions and 59 deletions
@@ -287,18 +287,18 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
}
// DeleteByChunkIDList Delete indices by chunk ID list
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
}
// DeleteBySourceIDList Delete indices by source ID list
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
}
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
}
@@ -816,6 +816,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(
@@ -163,7 +163,7 @@ func (e *elasticsearchRepository) BatchSave(ctx context.Context,
// DeleteByChunkIDList removes documents from the index based on chunk IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(chunkIDList) == 0 {
log.Warn("[Elasticsearch] Empty chunk ID list provided for deletion, skipping")
@@ -186,7 +186,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
// DeleteBySourceIDList removes documents from the index based on source IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(sourceIDList) == 0 {
log.Warn("[Elasticsearch] Empty source ID list provided for deletion, skipping")
@@ -210,7 +210,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
// DeleteByKnowledgeIDList removes documents from the index based on knowledge IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(knowledgeIDList) == 0 {
@@ -447,6 +447,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(
@@ -107,7 +107,7 @@ func (g *pgRepository) BatchSave(
}
// DeleteByChunkIDList deletes indices by chunk IDs
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
if result.Error != nil {
@@ -119,7 +119,7 @@ func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []st
}
// DeleteBySourceIDList deletes indices by source IDs
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
if len(sourceIDList) == 0 {
return nil
}
@@ -134,7 +134,7 @@ func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []
}
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error {
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
if result.Error != nil {
@@ -357,6 +357,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
logger.GetLogger(ctx).Infof(
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
@@ -264,7 +264,7 @@ func (q *qdrantRepository) BatchSave(ctx context.Context,
}
// DeleteByChunkIDList removes points from the collection based on chunk IDs
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(chunkIDList) == 0 {
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
@@ -293,7 +293,7 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(knowledgeIDList) == 0 {
@@ -323,7 +323,7 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
// DeleteBySourceIDList removes points from the collection based on source IDs
func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context,
sourceIDList []string, dimension int,
sourceIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(sourceIDList) == 0 {
@@ -644,6 +644,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(