feat(elasticsearch): enhance ID field handling with dynamic suffix detection
- Added logic to determine whether to append the ".keyword" suffix to ID fields based on index mapping. - Introduced `detectFieldTypes` method to inspect index mappings and set the `useKeywordSuffix` flag accordingly. - Updated methods to utilize the new `idField` function for consistent ID field querying across the repository. - Improved logging for better visibility into mapping detection and suffix usage.
This commit is contained in:
@@ -26,8 +26,9 @@ import (
|
||||
)
|
||||
|
||||
type elasticsearchRepository struct {
|
||||
client *elasticsearch.Client
|
||||
index string
|
||||
client *elasticsearch.Client
|
||||
index string
|
||||
useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries
|
||||
}
|
||||
|
||||
func NewElasticsearchEngineRepository(client *elasticsearch.Client,
|
||||
@@ -44,9 +45,79 @@ func NewElasticsearchEngineRepository(client *elasticsearch.Client,
|
||||
|
||||
log.Infof("[ElasticsearchV7] Using index: %s", indexName)
|
||||
res := &elasticsearchRepository{client: client, index: indexName}
|
||||
res.detectFieldTypes(context.Background())
|
||||
return res
|
||||
}
|
||||
|
||||
// idField returns the query field name for an ID field, appending ".keyword"
|
||||
// suffix when the index uses text-type mappings with keyword sub-fields.
|
||||
func (e *elasticsearchRepository) idField(name string) string {
|
||||
if e.useKeywordSuffix {
|
||||
return name + ".keyword"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// detectFieldTypes inspects the index mapping to determine whether ID fields
|
||||
// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix).
|
||||
func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) {
|
||||
log := logger.GetLogger(ctx)
|
||||
|
||||
resp, err := e.client.Indices.GetMapping(
|
||||
e.client.Indices.GetMapping.WithIndex(e.index),
|
||||
e.client.Indices.GetMapping.WithContext(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
log.Warnf("[ElasticsearchV7] Failed to get index mapping, defaulting to .keyword suffix: %v", err)
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.IsError() {
|
||||
log.Warnf("[ElasticsearchV7] GetMapping returned error: %s, defaulting to .keyword suffix", resp.String())
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
log.Warnf("[ElasticsearchV7] Failed to parse mapping response: %v, defaulting to .keyword suffix", err)
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
|
||||
indexData, _ := result[e.index].(map[string]interface{})
|
||||
if indexData == nil {
|
||||
log.Warnf("[ElasticsearchV7] Index %s not found in mapping response, defaulting to .keyword suffix", e.index)
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
mappings, _ := indexData["mappings"].(map[string]interface{})
|
||||
if mappings == nil {
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
properties, _ := mappings["properties"].(map[string]interface{})
|
||||
if properties == nil {
|
||||
log.Infof("[ElasticsearchV7] No mapping detected for ID fields (empty index?), defaulting to .keyword suffix")
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
chunkIDProp, _ := properties["chunk_id"].(map[string]interface{})
|
||||
if chunkIDProp == nil {
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
if fieldType, _ := chunkIDProp["type"].(string); fieldType == "keyword" {
|
||||
e.useKeywordSuffix = false
|
||||
log.Infof("[ElasticsearchV7] Detected keyword type for ID fields, querying without .keyword suffix")
|
||||
} else {
|
||||
e.useKeywordSuffix = true
|
||||
log.Infof("[ElasticsearchV7] Detected %s type for ID fields, querying with .keyword suffix", fieldType)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType {
|
||||
return typesLocal.ElasticsearchRetrieverEngineType
|
||||
}
|
||||
@@ -288,19 +359,19 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
|
||||
|
||||
// DeleteByChunkIDList Delete indices by chunk ID list
|
||||
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
|
||||
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
|
||||
return e.deleteByFieldList(ctx, e.idField("chunk_id"), chunkIDList)
|
||||
}
|
||||
|
||||
// DeleteBySourceIDList Delete indices by source ID list
|
||||
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
|
||||
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
|
||||
return e.deleteByFieldList(ctx, e.idField("source_id"), sourceIDList)
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
|
||||
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
|
||||
return e.deleteByFieldList(ctx, e.idField("knowledge_id"), knowledgeIDList)
|
||||
}
|
||||
|
||||
// deleteByFieldList Delete documents by field value list
|
||||
@@ -368,14 +439,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
|
||||
e.idField("knowledge_base_id"): params.KnowledgeBaseIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_id.keyword": params.KnowledgeIDs,
|
||||
e.idField("knowledge_id"): params.KnowledgeIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -383,7 +454,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
if len(params.TagIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"tag_id.keyword": params.TagIDs,
|
||||
e.idField("tag_id"): params.TagIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -400,14 +471,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_id.keyword": params.ExcludeKnowledgeIDs,
|
||||
e.idField("knowledge_id"): params.ExcludeKnowledgeIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
mustNot = append(mustNot, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": params.ExcludeChunkIDs,
|
||||
e.idField("chunk_id"): params.ExcludeChunkIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1185,7 +1256,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
query := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": enabledChunkIDs,
|
||||
e.idField("chunk_id"): enabledChunkIDs,
|
||||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
@@ -1220,7 +1291,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
query := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": disabledChunkIDs,
|
||||
e.idField("chunk_id"): disabledChunkIDs,
|
||||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
@@ -1278,7 +1349,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
|
||||
query := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": chunkIDs,
|
||||
e.idField("chunk_id"): chunkIDs,
|
||||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
|
||||
@@ -21,8 +21,9 @@ import (
|
||||
|
||||
// elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8
|
||||
type elasticsearchRepository struct {
|
||||
client *elasticsearch.TypedClient // Elasticsearch client instance
|
||||
index string // Name of the Elasticsearch index to use
|
||||
client *elasticsearch.TypedClient // Elasticsearch client instance
|
||||
index string // Name of the Elasticsearch index to use
|
||||
useKeywordSuffix bool // Whether to append .keyword suffix to ID field names in queries
|
||||
}
|
||||
|
||||
// NewElasticsearchEngineRepository creates and initializes a new Elasticsearch v8 repository
|
||||
@@ -47,9 +48,59 @@ func NewElasticsearchEngineRepository(client *elasticsearch.TypedClient,
|
||||
} else {
|
||||
log.Info("[Elasticsearch] Successfully initialized repository")
|
||||
}
|
||||
res.detectFieldTypes(context.Background())
|
||||
return res
|
||||
}
|
||||
|
||||
// idField returns the query field name for an ID field, appending ".keyword"
|
||||
// suffix when the index uses text-type mappings with keyword sub-fields.
|
||||
func (e *elasticsearchRepository) idField(name string) string {
|
||||
if e.useKeywordSuffix {
|
||||
return name + ".keyword"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// detectFieldTypes inspects the index mapping to determine whether ID fields
|
||||
// are mapped as "keyword" (no suffix needed) or "text" (needs ".keyword" suffix).
|
||||
func (e *elasticsearchRepository) detectFieldTypes(ctx context.Context) {
|
||||
log := logger.GetLogger(ctx)
|
||||
|
||||
mappingResp, err := e.client.Indices.GetMapping().Index(e.index).Do(ctx)
|
||||
if err != nil {
|
||||
log.Warnf("[Elasticsearch] Failed to get index mapping, defaulting to .keyword suffix: %v", err)
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
|
||||
indexMapping, ok := mappingResp[e.index]
|
||||
if !ok {
|
||||
log.Warnf("[Elasticsearch] Index %s not found in mapping response, defaulting to .keyword suffix", e.index)
|
||||
e.useKeywordSuffix = true
|
||||
return
|
||||
}
|
||||
|
||||
if prop, ok := indexMapping.Mappings.Properties["chunk_id"]; ok {
|
||||
propBytes, err := json.Marshal(prop)
|
||||
if err == nil {
|
||||
var propInfo struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
if json.Unmarshal(propBytes, &propInfo) == nil && propInfo.Type == "keyword" {
|
||||
e.useKeywordSuffix = false
|
||||
log.Infof("[Elasticsearch] Detected keyword type for ID fields, querying without .keyword suffix")
|
||||
return
|
||||
}
|
||||
}
|
||||
e.useKeywordSuffix = true
|
||||
log.Infof("[Elasticsearch] ID fields are not keyword type, querying with .keyword suffix")
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("[Elasticsearch] No mapping detected for chunk_id (empty index?), defaulting to .keyword suffix")
|
||||
e.useKeywordSuffix = true
|
||||
}
|
||||
|
||||
// EngineType returns the type of retriever engine (Elasticsearch)
|
||||
func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType {
|
||||
return typesLocal.ElasticsearchRetrieverEngineType
|
||||
@@ -175,7 +226,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
|
||||
log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
|
||||
// Use DeleteByQuery to delete all documents matching the chunk IDs
|
||||
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}},
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): chunkIDList}},
|
||||
}).Do(ctx)
|
||||
if err != nil {
|
||||
log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err)
|
||||
@@ -198,7 +249,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
|
||||
log.Infof("[Elasticsearch] Deleting indices by source IDs, count: %d", len(sourceIDList))
|
||||
// Use DeleteByQuery to delete all documents matching the source IDs
|
||||
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"source_id.keyword": sourceIDList}},
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("source_id"): sourceIDList}},
|
||||
}).Do(ctx)
|
||||
if err != nil {
|
||||
log.Errorf("[Elasticsearch] Failed to delete by source IDs: %v", err)
|
||||
@@ -223,7 +274,7 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
|
||||
// Use DeleteByQuery to delete all documents matching the knowledge IDs
|
||||
_, err := e.client.DeleteByQuery(e.index).Query(&types.Query{
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}},
|
||||
Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): knowledgeIDList}},
|
||||
}).Do(ctx)
|
||||
if err != nil {
|
||||
log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err)
|
||||
@@ -247,14 +298,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
|
||||
e.idField("knowledge_base_id"): params.KnowledgeBaseIDs,
|
||||
},
|
||||
}})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"knowledge_id.keyword": params.KnowledgeIDs,
|
||||
e.idField("knowledge_id"): params.KnowledgeIDs,
|
||||
},
|
||||
}})
|
||||
}
|
||||
@@ -262,7 +313,7 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
if len(params.TagIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"tag_id.keyword": params.TagIDs,
|
||||
e.idField("tag_id"): params.TagIDs,
|
||||
},
|
||||
}})
|
||||
}
|
||||
@@ -275,12 +326,12 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
}})
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs},
|
||||
TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): params.ExcludeKnowledgeIDs},
|
||||
}})
|
||||
}
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs},
|
||||
TermsQuery: map[string]types.TermsQueryField{e.idField("chunk_id"): params.ExcludeChunkIDs},
|
||||
}})
|
||||
}
|
||||
return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}}
|
||||
@@ -642,7 +693,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
Must: []types.Query{
|
||||
{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"chunk_id.keyword": enabledChunkIDs,
|
||||
e.idField("chunk_id"): enabledChunkIDs,
|
||||
},
|
||||
}},
|
||||
},
|
||||
@@ -668,7 +719,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
Must: []types.Query{
|
||||
{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"chunk_id.keyword": disabledChunkIDs,
|
||||
e.idField("chunk_id"): disabledChunkIDs,
|
||||
},
|
||||
}},
|
||||
},
|
||||
@@ -717,7 +768,7 @@ func (e *elasticsearchRepository) BatchUpdateChunkTagID(
|
||||
Must: []types.Query{
|
||||
{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"chunk_id.keyword": chunkIDs,
|
||||
e.idField("chunk_id"): chunkIDs,
|
||||
},
|
||||
}},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user