diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 56373ba7..9feabb1d 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -26,8 +26,9 @@ import ( ) type elasticsearchRepository struct { - client *elasticsearch.Client - index string + client *elasticsearch.Client + index string + useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries } func NewElasticsearchEngineRepository(client *elasticsearch.Client, @@ -44,9 +45,79 @@ func NewElasticsearchEngineRepository(client *elasticsearch.Client, log.Infof("[ElasticsearchV7] Using index: %s", indexName) res := &elasticsearchRepository{client: client, index: indexName} + res.detectFieldTypes(context.Background()) return res } +// idField returns the query field name for an ID field, appending ".keyword" +// suffix when the index uses text-type mappings with keyword sub-fields. +func (e *elasticsearchRepository) idField(name string) string { + if e.useKeywordSuffix { + return name + ".keyword" + } + return name +} + +// detectFieldTypes inspects the index mapping to determine whether ID fields +// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix). +func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) { + log := logger.GetLogger(ctx) + + resp, err := e.client.Indices.GetMapping( + e.client.Indices.GetMapping.WithIndex(e.index), + e.client.Indices.GetMapping.WithContext(ctx), + ) + if err != nil { + log.Warnf("[ElasticsearchV7] Failed to get index mapping, defaulting to .keyword suffix: %v", err) + e.useKeywordSuffix = true + return + } + defer resp.Body.Close() + + if resp.IsError() { + log.Warnf("[ElasticsearchV7] GetMapping returned error: %s, defaulting to .keyword suffix", resp.String()) + e.useKeywordSuffix = true + return + } + + var result map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + log.Warnf("[ElasticsearchV7] Failed to parse mapping response: %v, defaulting to .keyword suffix", err) + e.useKeywordSuffix = true + return + } + + indexData, _ := result[e.index].(map[string]interface{}) + if indexData == nil { + log.Warnf("[ElasticsearchV7] Index %s not found in mapping response, defaulting to .keyword suffix", e.index) + e.useKeywordSuffix = true + return + } + mappings, _ := indexData["mappings"].(map[string]interface{}) + if mappings == nil { + e.useKeywordSuffix = true + return + } + properties, _ := mappings["properties"].(map[string]interface{}) + if properties == nil { + log.Infof("[ElasticsearchV7] No mapping detected for ID fields (empty index?), defaulting to .keyword suffix") + e.useKeywordSuffix = true + return + } + chunkIDProp, _ := properties["chunk_id"].(map[string]interface{}) + if chunkIDProp == nil { + e.useKeywordSuffix = true + return + } + if fieldType, _ := chunkIDProp["type"].(string); fieldType == "keyword" { + e.useKeywordSuffix = false + log.Infof("[ElasticsearchV7] Detected keyword type for ID fields, querying without .keyword suffix") + } else { + e.useKeywordSuffix = true + log.Infof("[ElasticsearchV7] Detected %s type for ID fields, querying with .keyword suffix", fieldType) + } +} + func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType { return typesLocal.ElasticsearchRetrieverEngineType } @@ -288,19 +359,19 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context, // DeleteByChunkIDList Delete indices by chunk ID list func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { - return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList) + return e.deleteByFieldList(ctx, e.idField("chunk_id"), chunkIDList) } // DeleteBySourceIDList Delete indices by source ID list func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error { - return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList) + return e.deleteByFieldList(ctx, e.idField("source_id"), sourceIDList) } // DeleteByKnowledgeIDList Delete indices by knowledge ID list func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string, ) error { - return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList) + return e.deleteByFieldList(ctx, e.idField("knowledge_id"), knowledgeIDList) } // deleteByFieldList Delete documents by field value list @@ -368,14 +439,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.KnowledgeBaseIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_base_id.keyword": params.KnowledgeBaseIDs, + e.idField("knowledge_base_id"): params.KnowledgeBaseIDs, }, }) } if len(params.KnowledgeIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_id.keyword": params.KnowledgeIDs, + e.idField("knowledge_id"): params.KnowledgeIDs, }, }) } @@ -383,7 +454,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.TagIDs) > 0 { must = append(must, map[string]interface{}{ "terms": map[string]interface{}{ - "tag_id.keyword": params.TagIDs, + e.idField("tag_id"): params.TagIDs, }, }) } @@ -400,14 +471,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.ExcludeKnowledgeIDs) > 0 { mustNot = append(mustNot, map[string]interface{}{ "terms": map[string]interface{}{ - "knowledge_id.keyword": params.ExcludeKnowledgeIDs, + e.idField("knowledge_id"): params.ExcludeKnowledgeIDs, }, }) } if len(params.ExcludeChunkIDs) > 0 { mustNot = append(mustNot, map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": params.ExcludeChunkIDs, + e.idField("chunk_id"): params.ExcludeChunkIDs, }, }) } @@ -1185,7 +1256,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": enabledChunkIDs, + e.idField("chunk_id"): enabledChunkIDs, }, }, "script": map[string]interface{}{ @@ -1220,7 +1291,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": disabledChunkIDs, + e.idField("chunk_id"): disabledChunkIDs, }, }, "script": map[string]interface{}{ @@ -1278,7 +1349,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID( query := map[string]interface{}{ "query": map[string]interface{}{ "terms": map[string]interface{}{ - "chunk_id.keyword": chunkIDs, + e.idField("chunk_id"): chunkIDs, }, }, "script": map[string]interface{}{ diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index 364ab983..ed25f084 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -21,8 +21,9 @@ import ( // elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8 type elasticsearchRepository struct { - client *elasticsearch.TypedClient // Elasticsearch client instance - index string // Name of the Elasticsearch index to use + client *elasticsearch.TypedClient // Elasticsearch client instance + index string // Name of the Elasticsearch index to use + useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries } // NewElasticsearchEngineRepository creates and initializes a new Elasticsearch v8 repository @@ -47,9 +48,59 @@ func NewElasticsearchEngineRepository(client *elasticsearch.TypedClient, } else { log.Info("[Elasticsearch] Successfully initialized repository") } + res.detectFieldTypes(context.Background()) return res } +// idField returns the query field name for an ID field, appending ".keyword" +// suffix when the index uses text-type mappings with keyword sub-fields. +func (e *elasticsearchRepository) idField(name string) string { + if e.useKeywordSuffix { + return name + ".keyword" + } + return name +} + +// detectFieldTypes inspects the index mapping to determine whether ID fields +// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix). +func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) { + log := logger.GetLogger(ctx) + + mappingResp, err := e.client.Indices.GetMapping().Index(e.index).Do(ctx) + if err != nil { + log.Warnf("[Elasticsearch] Failed to get index mapping, defaulting to .keyword suffix: %v", err) + e.useKeywordSuffix = true + return + } + + indexMapping, ok := mappingResp[e.index] + if !ok { + log.Warnf("[Elasticsearch] Index %s not found in mapping response, defaulting to .keyword suffix", e.index) + e.useKeywordSuffix = true + return + } + + if prop, ok := indexMapping.Mappings.Properties["chunk_id"]; ok { + propBytes, err := json.Marshal(prop) + if err == nil { + var propInfo struct { + Type string `json:"type"` + } + if json.Unmarshal(propBytes, &propInfo) == nil && propInfo.Type == "keyword" { + e.useKeywordSuffix = false + log.Infof("[Elasticsearch] Detected keyword type for ID fields, querying without .keyword suffix") + return + } + } + e.useKeywordSuffix = true + log.Infof("[Elasticsearch] ID fields are not keyword type, querying with .keyword suffix") + return + } + + log.Infof("[Elasticsearch] No mapping detected for chunk_id (empty index?), defaulting to .keyword suffix") + e.useKeywordSuffix = true +} + // EngineType returns the type of retriever engine (Elasticsearch) func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType { return typesLocal.ElasticsearchRetrieverEngineType @@ -175,7 +226,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) // Use DeleteByQuery to delete all documents matching the chunk IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): chunkIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err) @@ -198,7 +249,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour log.Infof("[Elasticsearch] Deleting indices by source IDs, count: %d", len(sourceIDList)) // Use DeleteByQuery to delete all documents matching the source IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id.keyword": sourceIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("source_id"): sourceIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by source IDs: %v", err) @@ -223,7 +274,7 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) // Use DeleteByQuery to delete all documents matching the knowledge IDs _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ - Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}}, + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): knowledgeIDList}}, }).Do(ctx) if err != nil { log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err) @@ -247,14 +298,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.KnowledgeBaseIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "knowledge_base_id.keyword": params.KnowledgeBaseIDs, + e.idField("knowledge_base_id"): params.KnowledgeBaseIDs, }, }}) } if len(params.KnowledgeIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "knowledge_id.keyword": params.KnowledgeIDs, + e.idField("knowledge_id"): params.KnowledgeIDs, }, }}) } @@ -262,7 +313,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) if len(params.TagIDs) > 0 { must = append(must, types.Query{Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "tag_id.keyword": params.TagIDs, + e.idField("tag_id"): params.TagIDs, }, }}) } @@ -275,12 +326,12 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) }}) if len(params.ExcludeKnowledgeIDs) > 0 { mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs}, + TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): params.ExcludeKnowledgeIDs}, }}) } if len(params.ExcludeChunkIDs) > 0 { mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs}, + TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): params.ExcludeChunkIDs}, }}) } return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}} @@ -642,7 +693,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": enabledChunkIDs, + e.idField("chunk_id"): enabledChunkIDs, }, }}, }, @@ -668,7 +719,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": disabledChunkIDs, + e.idField("chunk_id"): disabledChunkIDs, }, }}, }, @@ -717,7 +768,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID( Must: []types.Query{ {Terms: &types.TermsQuery{ TermsQuery: map[string]types.TermsQueryField{ - "chunk_id.keyword": chunkIDs, + e.idField("chunk_id"): chunkIDs, }, }}, },