Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(elasticsearch): remove ".keyword" suffix from ID fields in Elasticsearch queries

Browse Source 
- Updated Elasticsearch repository methods to use ID fields without the ".keyword" suffix, aligning with the new index mapping requirements.
- Enhanced documentation for the VectorEmbedding struct to clarify expected index mapping and usage guidelines for ID fields.
- Ensured consistency across all repository methods that interact with ID fields, improving query performance and maintainability.
This commit is contained in:
wizardchen
2026-03-24 21:49:19 +08:00
committed by lyingbug
parent 164c39444d
commit 3daaabfc4a
3 changed files with 56 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/application/repository/retriever/elasticsearch/structs.go
+34 -1
View File
@@ -7,7 +7,40 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// VectorEmbedding defines the Elasticsearch document structure for vector embeddings
// VectorEmbedding defines the Elasticsearch document structure for vector embeddings.
//
// Expected index mapping (all ID fields must be "keyword" type, not "text"):
//
// {
// "settings": {
// "analysis": {
// "analyzer": {
// "ik_max_word": { "type": "custom", "tokenizer": "ik_max_word" },
// "ik_smart": { "type": "custom", "tokenizer": "ik_smart" }
// }
// }
// },
// "mappings": {
// "dynamic_templates": [
// { "strings": { "match_mapping_type": "string", "mapping": { "type": "keyword" } } }
// ],
// "properties": {
// "content": { "type": "text", "analyzer": "ik_max_word", "search_analyzer": "ik_smart" },
// "chunk_id": { "type": "keyword" },
// "knowledge_id": { "type": "keyword" },
// "knowledge_base_id": { "type": "keyword" },
// "source_id": { "type": "keyword" },
// "tag_id": { "type": "keyword" },
// "source_type": { "type": "long" },
// "is_enabled": { "type": "boolean" },
// "is_recommended": { "type": "boolean" },
// "embedding": { "type": "dense_vector", "dims": "<model_dims>" }
// }
// }
// }
//
// IMPORTANT: Do NOT use ".keyword" suffix in queries for ID fields (e.g. use "chunk_id", not
// "chunk_id.keyword"), because these fields are already "keyword" type without a ".keyword" sub-field.
type VectorEmbedding struct {
Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk
SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document
internal/application/repository/retriever/elasticsearch/v7/repository.go
+11 -11
View File
@@ -288,19 +288,19 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
// DeleteByChunkIDList Delete indices by chunk ID list
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
return e.deleteByFieldList(ctx, "chunk_id", chunkIDList)
}
// DeleteBySourceIDList Delete indices by source ID list
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
return e.deleteByFieldList(ctx, "source_id", sourceIDList)
}
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
return e.deleteByFieldList(ctx, "knowledge_id", knowledgeIDList)
}
// deleteByFieldList Delete documents by field value list
@@ -368,14 +368,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
"knowledge_base_id": params.KnowledgeBaseIDs,
},
})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_id.keyword": params.KnowledgeIDs,
"knowledge_id": params.KnowledgeIDs,
},
})
}
@@ -383,7 +383,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.TagIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"tag_id.keyword": params.TagIDs,
"tag_id": params.TagIDs,
},
})
}
@@ -400,14 +400,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_id.keyword": params.ExcludeKnowledgeIDs,
"knowledge_id": params.ExcludeKnowledgeIDs,
},
})
}
if len(params.ExcludeChunkIDs) > 0 {
mustNot = append(mustNot, map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": params.ExcludeChunkIDs,
"chunk_id": params.ExcludeChunkIDs,
},
})
}
@@ -1185,7 +1185,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": enabledChunkIDs,
"chunk_id": enabledChunkIDs,
},
},
"script": map[string]interface{}{
@@ -1220,7 +1220,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": disabledChunkIDs,
"chunk_id": disabledChunkIDs,
},
},
"script": map[string]interface{}{
@@ -1278,7 +1278,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": chunkIDs,
"chunk_id": chunkIDs,
},
},
"script": map[string]interface{}{
internal/application/repository/retriever/elasticsearch/v8/repository.go
+11 -11
View File
@@ -175,7 +175,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
// Use DeleteByQuery to delete all documents matching the chunk IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id": chunkIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err)
@@ -198,7 +198,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
log.Infof("[Elasticsearch] Deleting indices by source IDs, count: %d", len(sourceIDList))
// Use DeleteByQuery to delete all documents matching the source IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id.keyword": sourceIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id": sourceIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by source IDs: %v", err)
@@ -223,7 +223,7 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
// Use DeleteByQuery to delete all documents matching the knowledge IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id": knowledgeIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err)
@@ -247,14 +247,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
"knowledge_base_id": params.KnowledgeBaseIDs,
},
}})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"knowledge_id.keyword": params.KnowledgeIDs,
"knowledge_id": params.KnowledgeIDs,
},
}})
}
@@ -262,7 +262,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.TagIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"tag_id.keyword": params.TagIDs,
"tag_id": params.TagIDs,
},
}})
}
@@ -275,12 +275,12 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
}})
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs},
TermsQuery: map[string]types.TermsQueryField{"knowledge_id": params.ExcludeKnowledgeIDs},
}})
}
if len(params.ExcludeChunkIDs) > 0 {
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs},
TermsQuery: map[string]types.TermsQueryField{"chunk_id": params.ExcludeChunkIDs},
}})
}
return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}}
@@ -642,7 +642,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": enabledChunkIDs,
"chunk_id": enabledChunkIDs,
},
}},
},
@@ -668,7 +668,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": disabledChunkIDs,
"chunk_id": disabledChunkIDs,
},
}},
},
@@ -717,7 +717,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": chunkIDs,
"chunk_id": chunkIDs,
},
}},
},