From 3daaabfc4a8bf5f48d89f3c3bf261e5fd4a4cb14 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 24 Mar 2026 21:49:19 +0800 Subject: [PATCH] refactor(elasticsearch): remove ".keyword" suffix from ID fields in Elasticsearch queries - Updated Elasticsearch repository methods to use ID fields without the ".keyword" suffix, aligning with the new index mapping requirements. - Enhanced documentation for the VectorEmbedding struct to clarify expected index mapping and usage guidelines for ID fields. - Ensured consistency across all repository methods that interact with ID fields, improving query performance and maintainability. --- .../retriever/elasticsearch/structs.go | 35 ++++++++++++++++++- .../retriever/elasticsearch/v7/repository.go | 22 ++++++------ .../retriever/elasticsearch/v8/repository.go | 22 ++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/internal/application/repository/retriever/elasticsearch/structs.go b/internal/application/repository/retriever/elasticsearch/structs.go index f001d0d2..200bf354 100644 --- a/internal/application/repository/retriever/elasticsearch/structs.go +++ b/internal/application/repository/retriever/elasticsearch/structs.go @@ -7,7 +7,40 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// VectorEmbedding defines the Elasticsearch document structure for vector embeddings +// VectorEmbedding defines the Elasticsearch document structure for vector embeddings. +// +// Expected index mapping (all ID fields must be "keyword" type, not "text"): +// +// { +// "settings": { +// "analysis": { +// "analyzer": { +// "ik_max_word": { "type": "custom", "tokenizer": "ik_max_word" }, +// "ik_smart": { "type": "custom", "tokenizer": "ik_smart" } +// } +// } +// }, +// "mappings": { +// "dynamic_templates": [ +// { "strings": { "match_mapping_type": "string", "mapping": { "type": "keyword" } } } +// ], +// "properties": { +// "content": { "type": "text", "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, +// "chunk_id": { "type": "keyword" }, +// "knowledge_id": { "type": "keyword" }, +// "knowledge_base_id": { "type": "keyword" }, +// "source_id": { "type": "keyword" }, +// "tag_id": { "type": "keyword" }, +// "source_type": { "type": "long" }, +// "is_enabled": { "type": "boolean" }, +// "is_recommended": { "type": "boolean" }, +// "embedding": { "type": "dense_vector", "dims": "" } +// } +// } +// } +// +// IMPORTANT: Do NOT use ".keyword" suffix in queries for ID fields (e.g. use "chunk_id", not +// "chunk_id.keyword"), because these fields are already "keyword" type without a ".keyword" sub-field. type VectorEmbedding struct { Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 56373ba7..06ec5da9 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -288,19 +288,19 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context, // DeleteByChunkIDList Delete indices by chunk ID list func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { - return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList) + return e.deleteByFieldList(ctx, "chunk_id", chunkIDList) } // DeleteBySourceIDList Delete indices by source ID list func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error { - return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList) + return e.deleteByFieldList(ctx, "source_id", sourceIDList) } // DeleteByKnowledgeIDList Delete indices by knowledge ID list func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string, ) error { - return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList) + return e.deleteByFieldList(ctx, "knowledge_id", knowledgeIDList) } // deleteByFieldList Delete documents by field value list @@ -368,14 +368,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.KnowledgeBaseIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_base_id.keyword": params.KnowledgeBaseIDs, + "knowledge_base_id": params.KnowledgeBaseIDs, }, }) } if len(params.KnowledgeIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_id.keyword": params.KnowledgeIDs, + "knowledge_id": params.KnowledgeIDs, }, }) } @@ -383,7 +383,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.TagIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "tag_id.keyword": params.TagIDs, + "tag_id": params.TagIDs, }, }) } @@ -400,14 +400,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.ExcludeKnowledgeIDs) > 0 { mustNot = append(mustNot, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_id.keyword": params.ExcludeKnowledgeIDs, + "knowledge_id": params.ExcludeKnowledgeIDs, }, }) } if len(params.ExcludeChunkIDs) > 0 { mustNot = append(mustNot, map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": params.ExcludeChunkIDs, + "chunk_id": params.ExcludeChunkIDs, }, }) } @@ -1185,7 +1185,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": enabledChunkIDs, + "chunk_id": enabledChunkIDs, }, }, "script": map[string]interface{}{ @@ -1220,7 +1220,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": disabledChunkIDs, + "chunk_id": disabledChunkIDs, }, }, "script": map[string]interface{}{ @@ -1278,7 +1278,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": chunkIDs, + "chunk_id": chunkIDs, }, }, "script": map[string]interface{}{ diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index 364ab983..def9c0ed 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -175,7 +175,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) // Use DeleteByQuery to delete all documents matching the chunk IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id": chunkIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err) @@ -198,7 +198,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour log.Infof("[Elasticsearch] Deleting indices by source IDs, count: %d", len(sourceIDList)) // Use DeleteByQuery to delete all documents matching the source IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id.keyword": sourceIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id": sourceIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by source IDs: %v", err) @@ -223,7 +223,7 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) // Use DeleteByQuery to delete all documents matching the knowledge IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id": knowledgeIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err) @@ -247,14 +247,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.KnowledgeBaseIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "knowledge_base_id.keyword": params.KnowledgeBaseIDs, + "knowledge_base_id": params.KnowledgeBaseIDs, }, }}) } if len(params.KnowledgeIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "knowledge_id.keyword": params.KnowledgeIDs, + "knowledge_id": params.KnowledgeIDs, }, }}) } @@ -262,7 +262,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.TagIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "tag_id.keyword": params.TagIDs, + "tag_id": params.TagIDs, }, }}) } @@ -275,12 +275,12 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) }}) if len(params.ExcludeKnowledgeIDs) > 0 { mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs}, + TermsQuery: map[string]types.TermsQueryField{"knowledge_id": params.ExcludeKnowledgeIDs}, }}) } if len(params.ExcludeChunkIDs) > 0 { mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs}, + TermsQuery: map[string]types.TermsQueryField{"chunk_id": params.ExcludeChunkIDs}, }}) } return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}} @@ -642,7 +642,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": enabledChunkIDs, + "chunk_id": enabledChunkIDs, }, }}, }, @@ -668,7 +668,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": disabledChunkIDs, + "chunk_id": disabledChunkIDs, }, }}, }, @@ -717,7 +717,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": chunkIDs, + "chunk_id": chunkIDs, }, }}, },