feat(elasticsearch): enhance ID field handling with dynamic suffix detection

- Added logic to determine whether to append the ".keyword" suffix to ID fields based on index mapping.
- Introduced `detectFieldTypes` method to inspect index mappings and set the `useKeywordSuffix` flag accordingly.
- Updated methods to utilize the new `idField` function for consistent ID field querying across the repository.
- Improved logging for better visibility into mapping detection and suffix usage.
This commit is contained in:
wizardchen
2026-04-02 12:27:57 +08:00
committed by lyingbug
parent 33e919cc80
commit 9d021a29b7
2 changed files with 148 additions and 26 deletions
@@ -26,8 +26,9 @@ import (
)
type elasticsearchRepository struct {
client *elasticsearch.Client
index string
client *elasticsearch.Client
index string
useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries
}
func NewElasticsearchEngineRepository(client *elasticsearch.Client,
@@ -44,9 +45,79 @@ func NewElasticsearchEngineRepository(client *elasticsearch.Client,
log.Infof("[ElasticsearchV7] Using index: %s", indexName)
res := &elasticsearchRepository{client: client, index: indexName}
res.detectFieldTypes(context.Background())
return res
}
// idField returns the query field name for an ID field, appending ".keyword"
// suffix when the index uses text-type mappings with keyword sub-fields.
func (e *elasticsearchRepository) idField(name string) string {
if e.useKeywordSuffix {
return name + ".keyword"
}
return name
}
// detectFieldTypes inspects the index mapping to determine whether ID fields
// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix).
func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) {
log := logger.GetLogger(ctx)
resp, err := e.client.Indices.GetMapping(
e.client.Indices.GetMapping.WithIndex(e.index),
e.client.Indices.GetMapping.WithContext(ctx),
)
if err != nil {
log.Warnf("[ElasticsearchV7] Failed to get index mapping, defaulting to .keyword suffix: %v", err)
e.useKeywordSuffix = true
return
}
defer resp.Body.Close()
if resp.IsError() {
log.Warnf("[ElasticsearchV7] GetMapping returned error: %s, defaulting to .keyword suffix", resp.String())
e.useKeywordSuffix = true
return
}
var result map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
log.Warnf("[ElasticsearchV7] Failed to parse mapping response: %v, defaulting to .keyword suffix", err)
e.useKeywordSuffix = true
return
}
indexData, _ := result[e.index].(map[string]interface{})
if indexData == nil {
log.Warnf("[ElasticsearchV7] Index %s not found in mapping response, defaulting to .keyword suffix", e.index)
e.useKeywordSuffix = true
return
}
mappings, _ := indexData["mappings"].(map[string]interface{})
if mappings == nil {
e.useKeywordSuffix = true
return
}
properties, _ := mappings["properties"].(map[string]interface{})
if properties == nil {
log.Infof("[ElasticsearchV7] No mapping detected for ID fields (empty index?), defaulting to .keyword suffix")
e.useKeywordSuffix = true
return
}
chunkIDProp, _ := properties["chunk_id"].(map[string]interface{})
if chunkIDProp == nil {
e.useKeywordSuffix = true
return
}
if fieldType, _ := chunkIDProp["type"].(string); fieldType == "keyword" {
e.useKeywordSuffix = false
log.Infof("[ElasticsearchV7] Detected keyword type for ID fields, querying without .keyword suffix")
} else {
e.useKeywordSuffix = true
log.Infof("[ElasticsearchV7] Detected %s type for ID fields, querying with .keyword suffix", fieldType)
}
}
func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType {
return typesLocal.ElasticsearchRetrieverEngineType
}
@@ -288,19 +359,19 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
// DeleteByChunkIDList Delete indices by chunk ID list
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
return e.deleteByFieldList(ctx, e.idField("chunk_id"), chunkIDList)
}
// DeleteBySourceIDList Delete indices by source ID list
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
return e.deleteByFieldList(ctx, e.idField("source_id"), sourceIDList)
}
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
return e.deleteByFieldList(ctx, e.idField("knowledge_id"), knowledgeIDList)
}
// deleteByFieldList Delete documents by field value list
@@ -368,14 +439,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
e.idField("knowledge_base_id"): params.KnowledgeBaseIDs,
},
})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_id.keyword": params.KnowledgeIDs,
e.idField("knowledge_id"): params.KnowledgeIDs,
},
})
}
@@ -383,7 +454,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.TagIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"tag_id.keyword": params.TagIDs,
e.idField("tag_id"): params.TagIDs,
},
})
}
@@ -400,14 +471,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_id.keyword": params.ExcludeKnowledgeIDs,
e.idField("knowledge_id"): params.ExcludeKnowledgeIDs,
},
})
}
if len(params.ExcludeChunkIDs) > 0 {
mustNot = append(mustNot, map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": params.ExcludeChunkIDs,
e.idField("chunk_id"): params.ExcludeChunkIDs,
},
})
}
@@ -1185,7 +1256,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": enabledChunkIDs,
e.idField("chunk_id"): enabledChunkIDs,
},
},
"script": map[string]interface{}{
@@ -1220,7 +1291,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": disabledChunkIDs,
e.idField("chunk_id"): disabledChunkIDs,
},
},
"script": map[string]interface{}{
@@ -1278,7 +1349,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": chunkIDs,
e.idField("chunk_id"): chunkIDs,
},
},
"script": map[string]interface{}{
@@ -21,8 +21,9 @@ import (
// elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8
type elasticsearchRepository struct {
client *elasticsearch.TypedClient // Elasticsearch client instance
index string // Name of the Elasticsearch index to use
client *elasticsearch.TypedClient // Elasticsearch client instance
index string // Name of the Elasticsearch index to use
useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries
}
// NewElasticsearchEngineRepository creates and initializes a new Elasticsearch v8 repository
@@ -47,9 +48,59 @@ func NewElasticsearchEngineRepository(client *elasticsearch.TypedClient,
} else {
log.Info("[Elasticsearch] Successfully initialized repository")
}
res.detectFieldTypes(context.Background())
return res
}
// idField returns the query field name for an ID field, appending ".keyword"
// suffix when the index uses text-type mappings with keyword sub-fields.
func (e *elasticsearchRepository) idField(name string) string {
if e.useKeywordSuffix {
return name + ".keyword"
}
return name
}
// detectFieldTypes inspects the index mapping to determine whether ID fields
// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix).
func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) {
log := logger.GetLogger(ctx)
mappingResp, err := e.client.Indices.GetMapping().Index(e.index).Do(ctx)
if err != nil {
log.Warnf("[Elasticsearch] Failed to get index mapping, defaulting to .keyword suffix: %v", err)
e.useKeywordSuffix = true
return
}
indexMapping, ok := mappingResp[e.index]
if !ok {
log.Warnf("[Elasticsearch] Index %s not found in mapping response, defaulting to .keyword suffix", e.index)
e.useKeywordSuffix = true
return
}
if prop, ok := indexMapping.Mappings.Properties["chunk_id"]; ok {
propBytes, err := json.Marshal(prop)
if err == nil {
var propInfo struct {
Type string `json:"type"`
}
if json.Unmarshal(propBytes, &propInfo) == nil && propInfo.Type == "keyword" {
e.useKeywordSuffix = false
log.Infof("[Elasticsearch] Detected keyword type for ID fields, querying without .keyword suffix")
return
}
}
e.useKeywordSuffix = true
log.Infof("[Elasticsearch] ID fields are not keyword type, querying with .keyword suffix")
return
}
log.Infof("[Elasticsearch] No mapping detected for chunk_id (empty index?), defaulting to .keyword suffix")
e.useKeywordSuffix = true
}
// EngineType returns the type of retriever engine (Elasticsearch)
func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType {
return typesLocal.ElasticsearchRetrieverEngineType
@@ -175,7 +226,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
// Use DeleteByQuery to delete all documents matching the chunk IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): chunkIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err)
@@ -198,7 +249,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
log.Infof("[Elasticsearch] Deleting indices by source IDs, count: %d", len(sourceIDList))
// Use DeleteByQuery to delete all documents matching the source IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id.keyword": sourceIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("source_id"): sourceIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by source IDs: %v", err)
@@ -223,7 +274,7 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
// Use DeleteByQuery to delete all documents matching the knowledge IDs
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}},
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): knowledgeIDList}},
}).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err)
@@ -247,14 +298,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
e.idField("knowledge_base_id"): params.KnowledgeBaseIDs,
},
}})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"knowledge_id.keyword": params.KnowledgeIDs,
e.idField("knowledge_id"): params.KnowledgeIDs,
},
}})
}
@@ -262,7 +313,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
if len(params.TagIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"tag_id.keyword": params.TagIDs,
e.idField("tag_id"): params.TagIDs,
},
}})
}
@@ -275,12 +326,12 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
}})
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs},
TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): params.ExcludeKnowledgeIDs},
}})
}
if len(params.ExcludeChunkIDs) > 0 {
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs},
TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): params.ExcludeChunkIDs},
}})
}
return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}}
@@ -642,7 +693,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": enabledChunkIDs,
e.idField("chunk_id"): enabledChunkIDs,
},
}},
},
@@ -668,7 +719,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": disabledChunkIDs,
e.idField("chunk_id"): disabledChunkIDs,
},
}},
},
@@ -717,7 +768,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": chunkIDs,
e.idField("chunk_id"): chunkIDs,
},
}},
},