refactor: Improve Elasticsearch query construction and error handling
This commit is contained in:
@@ -353,38 +353,70 @@ func (e *elasticsearchRepository) deleteByFieldList(ctx context.Context, field s
|
||||
// Returns a JSON string representing the query conditions
|
||||
func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) string {
|
||||
// Build MUST conditions (positive filters)
|
||||
must := make([]string, 0)
|
||||
must := make([]map[string]interface{}, 0)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
ids, _ := json.Marshal(params.KnowledgeBaseIDs)
|
||||
must = append(must, fmt.Sprintf(`{"terms": {"knowledge_base_id.keyword": %s}}`, ids))
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_base_id.keyword": params.KnowledgeBaseIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Build MUST_NOT conditions (negative filters)
|
||||
mustNot := make([]string, 0)
|
||||
mustNot := make([]map[string]interface{}, 0)
|
||||
// Exclude disabled chunks (is_enabled = false)
|
||||
// Note: Historical data without is_enabled field will be included (not matching must_not)
|
||||
mustNot = append(mustNot, `{"term": {"is_enabled": false}}`)
|
||||
mustNot = append(mustNot, map[string]interface{}{
|
||||
"term": map[string]interface{}{
|
||||
"is_enabled": false,
|
||||
},
|
||||
})
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
ids, _ := json.Marshal(params.ExcludeKnowledgeIDs)
|
||||
mustNot = append(mustNot, fmt.Sprintf(`{"terms": {"knowledge_id.keyword": %s}}`, ids))
|
||||
mustNot = append(mustNot, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_id.keyword": params.ExcludeKnowledgeIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
ids, _ := json.Marshal(params.ExcludeChunkIDs)
|
||||
mustNot = append(mustNot, fmt.Sprintf(`{"terms": {"chunk_id.keyword": %s}}`, ids))
|
||||
mustNot = append(mustNot, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": params.ExcludeChunkIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Combine conditions based on presence
|
||||
var query map[string]interface{}
|
||||
if len(must) == 0 && len(mustNot) == 0 {
|
||||
return "{}" // Empty query if no conditions
|
||||
query = map[string]interface{}{}
|
||||
} else if len(must) == 0 {
|
||||
query = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must_not": mustNot,
|
||||
},
|
||||
}
|
||||
} else if len(mustNot) == 0 {
|
||||
query = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must": must,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
query = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must": must,
|
||||
"must_not": mustNot,
|
||||
},
|
||||
}
|
||||
}
|
||||
if len(must) == 0 {
|
||||
return fmt.Sprintf(`{"bool": {"must_not": [%s]}}`, strings.Join(mustNot, ","))
|
||||
|
||||
// Marshal to JSON string
|
||||
jsonBytes, err := json.Marshal(query)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
if len(mustNot) == 0 {
|
||||
return fmt.Sprintf(`{"bool": {"must": [%s]}}`, strings.Join(must, ","))
|
||||
}
|
||||
return fmt.Sprintf(`{"bool": {"must": [%s], "must_not": [%s]}}`,
|
||||
strings.Join(must, ","), strings.Join(mustNot, ","))
|
||||
return string(jsonBytes)
|
||||
}
|
||||
|
||||
func (e *elasticsearchRepository) Retrieve(ctx context.Context,
|
||||
@@ -439,26 +471,43 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
|
||||
) (string, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
|
||||
filter := e.getBaseConds(params)
|
||||
|
||||
// Serialize the query vector
|
||||
queryVectorJSON, err := json.Marshal(params.Embedding)
|
||||
if err != nil {
|
||||
log.Errorf("[ElasticsearchV7] Failed to marshal query vector: %v", err)
|
||||
return "", fmt.Errorf("failed to marshal query embedding: %w", err)
|
||||
// Parse filter conditions
|
||||
var filterQuery map[string]interface{}
|
||||
filterJSON := e.getBaseConds(params)
|
||||
if err := json.Unmarshal([]byte(filterJSON), &filterQuery); err != nil {
|
||||
log.Errorf("[ElasticsearchV7] Failed to unmarshal filter: %v", err)
|
||||
filterQuery = map[string]interface{}{}
|
||||
}
|
||||
|
||||
// Construct the script_score query
|
||||
query := fmt.Sprintf(
|
||||
`{"query":{"script_score":{"query":{"bool":{"filter":[%s]}},
|
||||
"script":{"source":"cosineSimilarity(params.query_vector,'embedding')",
|
||||
"params":{"query_vector":%s}},"min_score":%f}},"size":%d}`,
|
||||
filter,
|
||||
string(queryVectorJSON),
|
||||
params.Threshold,
|
||||
params.TopK,
|
||||
)
|
||||
// Construct the script_score query using structured objects
|
||||
queryObj := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"script_score": map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"filter": []interface{}{filterQuery},
|
||||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
"source": "cosineSimilarity(params.query_vector,'embedding')",
|
||||
"params": map[string]interface{}{
|
||||
"query_vector": params.Embedding,
|
||||
},
|
||||
},
|
||||
"min_score": params.Threshold,
|
||||
},
|
||||
},
|
||||
"size": params.TopK,
|
||||
}
|
||||
|
||||
// Marshal to JSON string
|
||||
queryBytes, err := json.Marshal(queryObj)
|
||||
if err != nil {
|
||||
log.Errorf("[ElasticsearchV7] Failed to marshal query: %v", err)
|
||||
return "", fmt.Errorf("failed to marshal query: %w", err)
|
||||
}
|
||||
|
||||
query := string(queryBytes)
|
||||
log.Debugf("[ElasticsearchV7] Executing vector search with query: %s", query)
|
||||
return query, nil
|
||||
}
|
||||
|
||||
@@ -145,6 +145,9 @@ func (b *graphBuilder) extractEntities(ctx context.Context, chunk *types.Chunk)
|
||||
defer b.mutex.Unlock()
|
||||
|
||||
for _, entity := range extractedEntities {
|
||||
if entity == nil {
|
||||
continue
|
||||
}
|
||||
if entity.Title == "" || entity.Description == "" {
|
||||
log.WithField("entity", entity).Warn("Invalid entity with empty title or description")
|
||||
continue
|
||||
@@ -272,6 +275,10 @@ func (b *graphBuilder) extractRelationships(ctx context.Context,
|
||||
relationship.Source, relationship.Target, relationship.ID)
|
||||
} else {
|
||||
// This relationship already exists, update its properties
|
||||
if existingRel == nil {
|
||||
log.Warnf("existingRel is nil, skip update")
|
||||
continue
|
||||
}
|
||||
chunkIDsAdded := 0
|
||||
for _, chunkID := range relationChunkIDs {
|
||||
if !slices.Contains(existingRel.ChunkIDs, chunkID) {
|
||||
|
||||
@@ -433,7 +433,6 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
logger.Infof(ctx, "Hybrid search parameters, knowledge base ID: %s, query text: %s", id, params.QueryText)
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
logger.Infof(ctx, "Creating composite retrieval engine, tenant ID: %d", tenantInfo.ID)
|
||||
|
||||
// Create a composite retrieval engine with tenant's configured retrievers
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
|
||||
@@ -44,7 +44,6 @@ func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types
|
||||
return fmt.Errorf("failed to create MCP service: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("MCP service created: %s (ID: %s)", service.Name, service.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -587,15 +587,13 @@ func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, sess
|
||||
// 3. Session's SummaryModelID (if not Remote)
|
||||
// 4. First knowledge base's SummaryModelID
|
||||
func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) {
|
||||
chatModelID := ""
|
||||
|
||||
// First, check if session has a SummaryModelID and if it's a Remote model
|
||||
if session.SummaryModelID != "" {
|
||||
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
|
||||
if err == nil && model != nil && model.Source == types.ModelSourceRemote {
|
||||
chatModelID = session.SummaryModelID
|
||||
logger.Infof(ctx, "Using session's Remote summary model: %s", chatModelID)
|
||||
return chatModelID, nil
|
||||
logger.Infof(ctx, "Using session's Remote summary model: %s", session.SummaryModelID)
|
||||
return session.SummaryModelID, nil
|
||||
} else if err == nil && model != nil {
|
||||
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
@@ -603,7 +601,7 @@ func (s *sessionService) selectChatModelID(ctx context.Context, session *types.S
|
||||
}
|
||||
|
||||
// If no Remote model found from session, check knowledge bases for Remote models
|
||||
if chatModelID == "" && len(knowledgeBaseIDs) > 0 {
|
||||
if len(knowledgeBaseIDs) > 0 {
|
||||
// Try to find a knowledge base with Remote model
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, kbID)
|
||||
@@ -614,44 +612,35 @@ func (s *sessionService) selectChatModelID(ctx context.Context, session *types.S
|
||||
if kb != nil && kb.SummaryModelID != "" {
|
||||
model, err := s.modelService.GetModelByID(ctx, kb.SummaryModelID)
|
||||
if err == nil && model != nil && model.Source == types.ModelSourceRemote {
|
||||
chatModelID = kb.SummaryModelID
|
||||
logger.Infof(ctx, "Using Remote summary model from knowledge base %s: %s", kbID, chatModelID)
|
||||
return chatModelID, nil
|
||||
logger.Infof(ctx, "Using Remote summary model from knowledge base %s: %s", kbID, kb.SummaryModelID)
|
||||
return kb.SummaryModelID, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If still no Remote model found, use session's SummaryModelID if available
|
||||
if chatModelID == "" && session.SummaryModelID != "" {
|
||||
chatModelID = session.SummaryModelID
|
||||
logger.Infof(ctx, "No Remote model found, using session's summary model: %s", chatModelID)
|
||||
return chatModelID, nil
|
||||
if session.SummaryModelID != "" {
|
||||
logger.Infof(ctx, "No Remote model found, using session's summary model: %s", session.SummaryModelID)
|
||||
return session.SummaryModelID, nil
|
||||
}
|
||||
|
||||
// If still empty, use first knowledge base's model
|
||||
if chatModelID == "" {
|
||||
kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledgeBaseIDs[0])
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base for model ID: %v", err)
|
||||
return "", fmt.Errorf("failed to get knowledge base %s: %w", knowledgeBaseIDs[0], err)
|
||||
}
|
||||
if kb != nil && kb.SummaryModelID != "" {
|
||||
chatModelID = kb.SummaryModelID
|
||||
logger.Infof(ctx, "Using summary model from first knowledge base %s: %s", knowledgeBaseIDs[0], chatModelID)
|
||||
return chatModelID, nil
|
||||
} else {
|
||||
logger.Errorf(ctx, "Knowledge base %s has no summary model ID", knowledgeBaseIDs[0])
|
||||
return "", fmt.Errorf("knowledge base %s has no summary model configured", knowledgeBaseIDs[0])
|
||||
}
|
||||
kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledgeBaseIDs[0])
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base for model ID: %v", err)
|
||||
return "", fmt.Errorf("failed to get knowledge base %s: %w", knowledgeBaseIDs[0], err)
|
||||
}
|
||||
if kb != nil && kb.SummaryModelID != "" {
|
||||
logger.Infof(ctx, "Using summary model from first knowledge base %s: %s", knowledgeBaseIDs[0], kb.SummaryModelID)
|
||||
return kb.SummaryModelID, nil
|
||||
} else {
|
||||
logger.Errorf(ctx, "Knowledge base %s has no summary model ID", knowledgeBaseIDs[0])
|
||||
return "", fmt.Errorf("knowledge base %s has no summary model configured", knowledgeBaseIDs[0])
|
||||
}
|
||||
}
|
||||
|
||||
if chatModelID == "" {
|
||||
logger.Error(ctx, "No chat model ID available")
|
||||
return "", errors.New("no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID")
|
||||
}
|
||||
|
||||
return chatModelID, nil
|
||||
logger.Error(ctx, "No chat model ID available")
|
||||
return "", errors.New("no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID")
|
||||
}
|
||||
|
||||
// KnowledgeQAByEvent processes knowledge QA through a series of events in the pipeline
|
||||
|
||||
@@ -18,7 +18,14 @@ import (
|
||||
)
|
||||
|
||||
// JWT secret key - in production this should be from environment variable
|
||||
var jwtSecret = []byte("your-secret-key")
|
||||
|
||||
func getJwtSecret() string {
|
||||
jwtSecret := os.Getenv("JWT_SECRET")
|
||||
if jwtSecret != "" {
|
||||
return jwtSecret
|
||||
}
|
||||
return "your-secret-key"
|
||||
}
|
||||
|
||||
// userService implements the UserService interface
|
||||
type userService struct {
|
||||
@@ -282,7 +289,7 @@ func (s *userService) GenerateTokens(ctx context.Context, user *types.User) (acc
|
||||
}
|
||||
|
||||
accessTokenObj := jwt.NewWithClaims(jwt.SigningMethodHS256, accessClaims)
|
||||
accessToken, err = accessTokenObj.SignedString(jwtSecret)
|
||||
accessToken, err = accessTokenObj.SignedString([]byte(getJwtSecret()))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
@@ -296,7 +303,7 @@ func (s *userService) GenerateTokens(ctx context.Context, user *types.User) (acc
|
||||
}
|
||||
|
||||
refreshTokenObj := jwt.NewWithClaims(jwt.SigningMethodHS256, refreshClaims)
|
||||
refreshToken, err = refreshTokenObj.SignedString(jwtSecret)
|
||||
refreshToken, err = refreshTokenObj.SignedString([]byte(getJwtSecret()))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
@@ -334,7 +341,7 @@ func (s *userService) ValidateToken(ctx context.Context, tokenString string) (*t
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
|
||||
}
|
||||
return jwtSecret, nil
|
||||
return []byte(getJwtSecret()), nil
|
||||
})
|
||||
|
||||
if err != nil || !token.Valid {
|
||||
@@ -366,7 +373,7 @@ func (s *userService) RefreshToken(ctx context.Context, refreshTokenString strin
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
|
||||
}
|
||||
return jwtSecret, nil
|
||||
return []byte(getJwtSecret()), nil
|
||||
})
|
||||
|
||||
if err != nil || !token.Valid {
|
||||
|
||||
Reference in New Issue
Block a user