Files
Enterprise-Knowledge-Base/internal/application/repository/retriever/elasticsearch/structs.go
T
wizardchen 3daaabfc4a refactor(elasticsearch): remove ".keyword" suffix from ID fields in Elasticsearch queries
- Updated Elasticsearch repository methods to use ID fields without the ".keyword" suffix, aligning with the new index mapping requirements.
- Enhanced documentation for the VectorEmbedding struct to clarify expected index mapping and usage guidelines for ID fields.
- Ensured consistency across all repository methods that interact with ID fields, improving query performance and maintainability.
2026-03-25 22:08:29 +08:00

112 lines
4.7 KiB
Go

package elasticsearch
import (
"maps"
"slices"
"github.com/Tencent/WeKnora/internal/types"
)
// VectorEmbedding defines the Elasticsearch document structure for vector embeddings.
//
// Expected index mapping (all ID fields must be "keyword" type, not "text"):
//
// {
// "settings": {
// "analysis": {
// "analyzer": {
// "ik_max_word": { "type": "custom", "tokenizer": "ik_max_word" },
// "ik_smart": { "type": "custom", "tokenizer": "ik_smart" }
// }
// }
// },
// "mappings": {
// "dynamic_templates": [
// { "strings": { "match_mapping_type": "string", "mapping": { "type": "keyword" } } }
// ],
// "properties": {
// "content": { "type": "text", "analyzer": "ik_max_word", "search_analyzer": "ik_smart" },
// "chunk_id": { "type": "keyword" },
// "knowledge_id": { "type": "keyword" },
// "knowledge_base_id": { "type": "keyword" },
// "source_id": { "type": "keyword" },
// "tag_id": { "type": "keyword" },
// "source_type": { "type": "long" },
// "is_enabled": { "type": "boolean" },
// "is_recommended": { "type": "boolean" },
// "embedding": { "type": "dense_vector", "dims": "<model_dims>" }
// }
// }
// }
//
// IMPORTANT: Do NOT use ".keyword" suffix in queries for ID fields (e.g. use "chunk_id", not
// "chunk_id.keyword"), because these fields are already "keyword" type without a ".keyword" sub-field.
type VectorEmbedding struct {
Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk
SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document
SourceType int `json:"source_type" gorm:"column:source_type;not null"` // Type of the source document
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` // Unique ID of the text chunk
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` // ID of the knowledge item
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` // ID of the knowledge base
TagID string `json:"tag_id" gorm:"column:tag_id"` // Tag ID for categorization
Embedding []float32 `json:"embedding" gorm:"column:embedding;not null"` // Vector embedding of the content
IsEnabled bool `json:"is_enabled"` // Whether the chunk is enabled
IsRecommended bool `json:"is_recommended"` // Whether the chunk is recommended
}
// VectorEmbeddingWithScore extends VectorEmbedding with similarity score
type VectorEmbeddingWithScore struct {
VectorEmbedding
Score float64 // Similarity score from vector search
}
// ToDBVectorEmbedding converts IndexInfo to Elasticsearch document format
func ToDBVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *VectorEmbedding {
vector := &VectorEmbedding{
Content: embedding.Content,
SourceID: embedding.SourceID,
SourceType: int(embedding.SourceType),
ChunkID: embedding.ChunkID,
KnowledgeID: embedding.KnowledgeID,
KnowledgeBaseID: embedding.KnowledgeBaseID,
TagID: embedding.TagID,
IsEnabled: embedding.IsEnabled,
IsRecommended: embedding.IsRecommended,
}
// Add embedding data if available in additionalParams
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), "embedding") {
if embeddingMap, ok := additionalParams["embedding"].(map[string][]float32); ok {
vector.Embedding = embeddingMap[embedding.SourceID]
}
}
// Get is_enabled from additionalParams if available
if additionalParams != nil {
if chunkEnabledMap, ok := additionalParams["chunk_enabled"].(map[string]bool); ok {
if enabled, exists := chunkEnabledMap[embedding.ChunkID]; exists {
vector.IsEnabled = enabled
}
}
}
return vector
}
// FromDBVectorEmbeddingWithScore converts Elasticsearch document to IndexWithScore domain model
func FromDBVectorEmbeddingWithScore(id string,
embedding *VectorEmbeddingWithScore,
matchType types.MatchType,
) *types.IndexWithScore {
return &types.IndexWithScore{
ID: id,
SourceID: embedding.SourceID,
SourceType: types.SourceType(embedding.SourceType),
ChunkID: embedding.ChunkID,
KnowledgeID: embedding.KnowledgeID,
KnowledgeBaseID: embedding.KnowledgeBaseID,
TagID: embedding.TagID,
Content: embedding.Content,
Score: embedding.Score,
MatchType: matchType,
IsEnabled: embedding.IsEnabled,
}
}